@omnitron-dev/omnitron 0.1.3 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/BUILD.json +5 -0
- package/dist/cli/omnitron.js +378 -25
- package/dist/cli/omnitron.js.map +1 -1
- package/dist/cluster/cluster.rpc-service.d.ts +40 -0
- package/dist/cluster/cluster.rpc-service.d.ts.map +1 -1
- package/dist/cluster/cluster.rpc-service.js +49 -3
- package/dist/cluster/cluster.rpc-service.js.map +1 -1
- package/dist/cluster/config-sync.d.ts.map +1 -1
- package/dist/cluster/config-sync.js.map +1 -1
- package/dist/cluster/leader-election.d.ts +90 -2
- package/dist/cluster/leader-election.d.ts.map +1 -1
- package/dist/cluster/leader-election.js +176 -24
- package/dist/cluster/leader-election.js.map +1 -1
- package/dist/cluster/mesh-link.d.ts +136 -0
- package/dist/cluster/mesh-link.d.ts.map +1 -0
- package/dist/cluster/mesh-link.js +212 -0
- package/dist/cluster/mesh-link.js.map +1 -0
- package/dist/cluster/mesh.d.ts +59 -0
- package/dist/cluster/mesh.d.ts.map +1 -0
- package/dist/cluster/mesh.js +117 -0
- package/dist/cluster/mesh.js.map +1 -0
- package/dist/cluster/slave-connector.d.ts +193 -9
- package/dist/cluster/slave-connector.d.ts.map +1 -1
- package/dist/cluster/slave-connector.js +409 -37
- package/dist/cluster/slave-connector.js.map +1 -1
- package/dist/commands/audit.d.ts +65 -0
- package/dist/commands/audit.d.ts.map +1 -0
- package/dist/commands/audit.js +233 -0
- package/dist/commands/audit.js.map +1 -0
- package/dist/commands/backup.d.ts +4 -0
- package/dist/commands/backup.d.ts.map +1 -1
- package/dist/commands/backup.js +255 -33
- package/dist/commands/backup.js.map +1 -1
- package/dist/commands/cluster.d.ts.map +1 -1
- package/dist/commands/cluster.js +78 -47
- package/dist/commands/cluster.js.map +1 -1
- package/dist/commands/daemon-cmd.d.ts +7 -1
- package/dist/commands/daemon-cmd.d.ts.map +1 -1
- package/dist/commands/daemon-cmd.js +87 -21
- package/dist/commands/daemon-cmd.js.map +1 -1
- package/dist/commands/daemon-required.d.ts +34 -0
- package/dist/commands/daemon-required.d.ts.map +1 -0
- package/dist/commands/daemon-required.js +92 -0
- package/dist/commands/daemon-required.js.map +1 -0
- package/dist/commands/deploy.d.ts +70 -8
- package/dist/commands/deploy.d.ts.map +1 -1
- package/dist/commands/deploy.js +99 -64
- package/dist/commands/deploy.js.map +1 -1
- package/dist/commands/discover.d.ts +25 -4
- package/dist/commands/discover.d.ts.map +1 -1
- package/dist/commands/discover.js +155 -102
- package/dist/commands/discover.js.map +1 -1
- package/dist/commands/doctor.d.ts +293 -0
- package/dist/commands/doctor.d.ts.map +1 -0
- package/dist/commands/doctor.js +1706 -0
- package/dist/commands/doctor.js.map +1 -0
- package/dist/commands/env.d.ts +8 -1
- package/dist/commands/env.d.ts.map +1 -1
- package/dist/commands/env.js +18 -5
- package/dist/commands/env.js.map +1 -1
- package/dist/commands/exec.d.ts.map +1 -1
- package/dist/commands/exec.js +3 -2
- package/dist/commands/exec.js.map +1 -1
- package/dist/commands/fleet-asking.d.ts +87 -0
- package/dist/commands/fleet-asking.d.ts.map +1 -0
- package/dist/commands/fleet-asking.js +176 -0
- package/dist/commands/fleet-asking.js.map +1 -0
- package/dist/commands/fleet.d.ts +18 -0
- package/dist/commands/fleet.d.ts.map +1 -1
- package/dist/commands/fleet.js +314 -74
- package/dist/commands/fleet.js.map +1 -1
- package/dist/commands/health-check.d.ts +8 -6
- package/dist/commands/health-check.d.ts.map +1 -1
- package/dist/commands/health-check.js +26 -23
- package/dist/commands/health-check.js.map +1 -1
- package/dist/commands/health.d.ts.map +1 -1
- package/dist/commands/health.js +26 -4
- package/dist/commands/health.js.map +1 -1
- package/dist/commands/infra.d.ts +42 -1
- package/dist/commands/infra.d.ts.map +1 -1
- package/dist/commands/infra.js +418 -49
- package/dist/commands/infra.js.map +1 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +21 -36
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/inspect-graph.d.ts +20 -0
- package/dist/commands/inspect-graph.d.ts.map +1 -0
- package/dist/commands/inspect-graph.js +74 -0
- package/dist/commands/inspect-graph.js.map +1 -0
- package/dist/commands/inspect.d.ts.map +1 -1
- package/dist/commands/inspect.js +115 -11
- package/dist/commands/inspect.js.map +1 -1
- package/dist/commands/k8s.d.ts.map +1 -1
- package/dist/commands/k8s.js +23 -18
- package/dist/commands/k8s.js.map +1 -1
- package/dist/commands/kb.d.ts.map +1 -1
- package/dist/commands/kb.js +88 -29
- package/dist/commands/kb.js.map +1 -1
- package/dist/commands/list.d.ts.map +1 -1
- package/dist/commands/list.js +25 -3
- package/dist/commands/list.js.map +1 -1
- package/dist/commands/log-health.d.ts +120 -0
- package/dist/commands/log-health.d.ts.map +1 -0
- package/dist/commands/log-health.js +212 -0
- package/dist/commands/log-health.js.map +1 -0
- package/dist/commands/logs.d.ts +96 -2
- package/dist/commands/logs.d.ts.map +1 -1
- package/dist/commands/logs.js +532 -192
- package/dist/commands/logs.js.map +1 -1
- package/dist/commands/metrics.d.ts.map +1 -1
- package/dist/commands/metrics.js +40 -14
- package/dist/commands/metrics.js.map +1 -1
- package/dist/commands/monit.d.ts.map +1 -1
- package/dist/commands/monit.js +12 -2
- package/dist/commands/monit.js.map +1 -1
- package/dist/commands/node.d.ts +68 -0
- package/dist/commands/node.d.ts.map +1 -1
- package/dist/commands/node.js +285 -68
- package/dist/commands/node.js.map +1 -1
- package/dist/commands/output.d.ts +45 -0
- package/dist/commands/output.d.ts.map +1 -0
- package/dist/commands/output.js +103 -0
- package/dist/commands/output.js.map +1 -0
- package/dist/commands/pipeline.d.ts.map +1 -1
- package/dist/commands/pipeline.js +26 -19
- package/dist/commands/pipeline.js.map +1 -1
- package/dist/commands/project.d.ts +19 -0
- package/dist/commands/project.d.ts.map +1 -1
- package/dist/commands/project.js +125 -27
- package/dist/commands/project.js.map +1 -1
- package/dist/commands/release-store.d.ts +28 -0
- package/dist/commands/release-store.d.ts.map +1 -0
- package/dist/commands/release-store.js +158 -0
- package/dist/commands/release-store.js.map +1 -0
- package/dist/commands/release.d.ts +61 -0
- package/dist/commands/release.d.ts.map +1 -0
- package/dist/commands/release.js +555 -0
- package/dist/commands/release.js.map +1 -0
- package/dist/commands/reload.d.ts +16 -0
- package/dist/commands/reload.d.ts.map +1 -1
- package/dist/commands/reload.js +21 -5
- package/dist/commands/reload.js.map +1 -1
- package/dist/commands/remote.d.ts +40 -0
- package/dist/commands/remote.d.ts.map +1 -1
- package/dist/commands/remote.js +146 -35
- package/dist/commands/remote.js.map +1 -1
- package/dist/commands/restart.d.ts +16 -0
- package/dist/commands/restart.d.ts.map +1 -1
- package/dist/commands/restart.js +21 -5
- package/dist/commands/restart.js.map +1 -1
- package/dist/commands/scale.d.ts.map +1 -1
- package/dist/commands/scale.js +4 -3
- package/dist/commands/scale.js.map +1 -1
- package/dist/commands/secret.d.ts +54 -0
- package/dist/commands/secret.d.ts.map +1 -1
- package/dist/commands/secret.js +236 -34
- package/dist/commands/secret.js.map +1 -1
- package/dist/commands/service-units.d.ts +181 -0
- package/dist/commands/service-units.d.ts.map +1 -0
- package/dist/commands/service-units.js +291 -0
- package/dist/commands/service-units.js.map +1 -0
- package/dist/commands/service.d.ts +39 -0
- package/dist/commands/service.d.ts.map +1 -0
- package/dist/commands/service.js +370 -0
- package/dist/commands/service.js.map +1 -0
- package/dist/commands/spinner.d.ts +29 -0
- package/dist/commands/spinner.d.ts.map +1 -0
- package/dist/commands/spinner.js +44 -0
- package/dist/commands/spinner.js.map +1 -0
- package/dist/commands/stack.d.ts +49 -1
- package/dist/commands/stack.d.ts.map +1 -1
- package/dist/commands/stack.js +341 -13
- package/dist/commands/stack.js.map +1 -1
- package/dist/commands/start.d.ts +16 -0
- package/dist/commands/start.d.ts.map +1 -1
- package/dist/commands/start.js +20 -3
- package/dist/commands/start.js.map +1 -1
- package/dist/commands/status.d.ts +9 -0
- package/dist/commands/status.d.ts.map +1 -1
- package/dist/commands/status.js +62 -29
- package/dist/commands/status.js.map +1 -1
- package/dist/commands/stop.d.ts +16 -0
- package/dist/commands/stop.d.ts.map +1 -1
- package/dist/commands/stop.js +34 -6
- package/dist/commands/stop.js.map +1 -1
- package/dist/commands/tor.d.ts +20 -0
- package/dist/commands/tor.d.ts.map +1 -0
- package/dist/commands/tor.js +180 -0
- package/dist/commands/tor.js.map +1 -0
- package/dist/commands/up.d.ts +85 -1
- package/dist/commands/up.d.ts.map +1 -1
- package/dist/commands/up.js +181 -33
- package/dist/commands/up.js.map +1 -1
- package/dist/commands/webapp.d.ts.map +1 -1
- package/dist/commands/webapp.js +2 -1
- package/dist/commands/webapp.js.map +1 -1
- package/dist/config/defaults.d.ts +16 -0
- package/dist/config/defaults.d.ts.map +1 -1
- package/dist/config/defaults.js +26 -3
- package/dist/config/defaults.js.map +1 -1
- package/dist/config/define-ecosystem.d.ts.map +1 -1
- package/dist/config/define-ecosystem.js +2 -0
- package/dist/config/define-ecosystem.js.map +1 -1
- package/dist/config/index.d.ts +1 -1
- package/dist/config/index.d.ts.map +1 -1
- package/dist/config/index.js +1 -1
- package/dist/config/index.js.map +1 -1
- package/dist/config/loader.d.ts +45 -1
- package/dist/config/loader.d.ts.map +1 -1
- package/dist/config/loader.js +93 -11
- package/dist/config/loader.js.map +1 -1
- package/dist/config/omnitron-config.schema.d.ts.map +1 -1
- package/dist/config/omnitron-config.schema.js +16 -0
- package/dist/config/omnitron-config.schema.js.map +1 -1
- package/dist/config/types.d.ts +425 -42
- package/dist/config/types.d.ts.map +1 -1
- package/dist/daemon/config-reload.d.ts +27 -0
- package/dist/daemon/config-reload.d.ts.map +1 -0
- package/dist/daemon/config-reload.js +42 -0
- package/dist/daemon/config-reload.js.map +1 -0
- package/dist/daemon/daemon-client.d.ts +150 -2
- package/dist/daemon/daemon-client.d.ts.map +1 -1
- package/dist/daemon/daemon-client.js +311 -21
- package/dist/daemon/daemon-client.js.map +1 -1
- package/dist/daemon/daemon-entry.js +92 -38
- package/dist/daemon/daemon-entry.js.map +1 -1
- package/dist/daemon/daemon-scheduler.d.ts +4 -0
- package/dist/daemon/daemon-scheduler.d.ts.map +1 -1
- package/dist/daemon/daemon-scheduler.js +137 -7
- package/dist/daemon/daemon-scheduler.js.map +1 -1
- package/dist/daemon/daemon-state-store.service.d.ts +263 -0
- package/dist/daemon/daemon-state-store.service.d.ts.map +1 -0
- package/dist/daemon/daemon-state-store.service.js +446 -0
- package/dist/daemon/daemon-state-store.service.js.map +1 -0
- package/dist/daemon/daemon.d.ts +207 -1
- package/dist/daemon/daemon.d.ts.map +1 -1
- package/dist/daemon/daemon.js +1254 -143
- package/dist/daemon/daemon.js.map +1 -1
- package/dist/daemon/daemon.module.d.ts.map +1 -1
- package/dist/daemon/daemon.module.js +186 -79
- package/dist/daemon/daemon.module.js.map +1 -1
- package/dist/daemon/daemon.rpc-service.d.ts +109 -1
- package/dist/daemon/daemon.rpc-service.d.ts.map +1 -1
- package/dist/daemon/daemon.rpc-service.js +451 -95
- package/dist/daemon/daemon.rpc-service.js.map +1 -1
- package/dist/daemon/pid-manager.d.ts +69 -5
- package/dist/daemon/pid-manager.d.ts.map +1 -1
- package/dist/daemon/pid-manager.js +151 -17
- package/dist/daemon/pid-manager.js.map +1 -1
- package/dist/daemon/spawn-daemon.d.ts +73 -0
- package/dist/daemon/spawn-daemon.d.ts.map +1 -0
- package/dist/daemon/spawn-daemon.js +101 -0
- package/dist/daemon/spawn-daemon.js.map +1 -0
- package/dist/daemon/state-store.d.ts +65 -3
- package/dist/daemon/state-store.d.ts.map +1 -1
- package/dist/daemon/state-store.js +103 -19
- package/dist/daemon/state-store.js.map +1 -1
- package/dist/database/connection.d.ts +71 -0
- package/dist/database/connection.d.ts.map +1 -0
- package/dist/database/connection.js +129 -0
- package/dist/database/connection.js.map +1 -0
- package/dist/database/dialect.d.ts +60 -0
- package/dist/database/dialect.d.ts.map +1 -0
- package/dist/database/dialect.js +80 -0
- package/dist/database/dialect.js.map +1 -0
- package/dist/database/migrate.d.ts +5 -2
- package/dist/database/migrate.d.ts.map +1 -1
- package/dist/database/migrate.js +23 -33
- package/dist/database/migrate.js.map +1 -1
- package/dist/database/migration-runner.d.ts +50 -0
- package/dist/database/migration-runner.d.ts.map +1 -0
- package/dist/database/migration-runner.js +143 -0
- package/dist/database/migration-runner.js.map +1 -0
- package/dist/database/migrations/006_login_throttle.d.ts +17 -0
- package/dist/database/migrations/006_login_throttle.d.ts.map +1 -0
- package/dist/database/migrations/006_login_throttle.js +33 -0
- package/dist/database/migrations/006_login_throttle.js.map +1 -0
- package/dist/database/migrations/007_sync_ingested.d.ts +21 -0
- package/dist/database/migrations/007_sync_ingested.d.ts.map +1 -0
- package/dist/database/migrations/007_sync_ingested.js +39 -0
- package/dist/database/migrations/007_sync_ingested.js.map +1 -0
- package/dist/database/migrations/008_node_health_version_text.d.ts +24 -0
- package/dist/database/migrations/008_node_health_version_text.d.ts.map +1 -0
- package/dist/database/migrations/008_node_health_version_text.js +35 -0
- package/dist/database/migrations/008_node_health_version_text.js.map +1 -0
- package/dist/database/migrations/009_node_health_unmeasured.d.ts +40 -0
- package/dist/database/migrations/009_node_health_unmeasured.d.ts.map +1 -0
- package/dist/database/migrations/009_node_health_unmeasured.js +63 -0
- package/dist/database/migrations/009_node_health_unmeasured.js.map +1 -0
- package/dist/database/migrations/010_metrics_raw_had_no_reader.d.ts +22 -0
- package/dist/database/migrations/010_metrics_raw_had_no_reader.d.ts.map +1 -0
- package/dist/database/migrations/010_metrics_raw_had_no_reader.js +26 -0
- package/dist/database/migrations/010_metrics_raw_had_no_reader.js.map +1 -0
- package/dist/database/migrations/011_alerts_nobody_had_to_write.d.ts +56 -0
- package/dist/database/migrations/011_alerts_nobody_had_to_write.d.ts.map +1 -0
- package/dist/database/migrations/011_alerts_nobody_had_to_write.js +79 -0
- package/dist/database/migrations/011_alerts_nobody_had_to_write.js.map +1 -0
- package/dist/database/migrations/012_a_disk_nobody_watched.d.ts +26 -0
- package/dist/database/migrations/012_a_disk_nobody_watched.d.ts.map +1 -0
- package/dist/database/migrations/012_a_disk_nobody_watched.js +40 -0
- package/dist/database/migrations/012_a_disk_nobody_watched.js.map +1 -0
- package/dist/database/migrations/013_an_alarm_delivered.d.ts +14 -0
- package/dist/database/migrations/013_an_alarm_delivered.d.ts.map +1 -0
- package/dist/database/migrations/013_an_alarm_delivered.js +28 -0
- package/dist/database/migrations/013_an_alarm_delivered.js.map +1 -0
- package/dist/database/migrations/index.d.ts +21 -0
- package/dist/database/migrations/index.d.ts.map +1 -0
- package/dist/database/migrations/index.js +47 -0
- package/dist/database/migrations/index.js.map +1 -0
- package/dist/database/schema.d.ts +27 -11
- package/dist/database/schema.d.ts.map +1 -1
- package/dist/database/sqlite-date-binding.d.ts +71 -0
- package/dist/database/sqlite-date-binding.d.ts.map +1 -0
- package/dist/database/sqlite-date-binding.js +174 -0
- package/dist/database/sqlite-date-binding.js.map +1 -0
- package/dist/execution/data-channel.d.ts +48 -0
- package/dist/execution/data-channel.d.ts.map +1 -0
- package/dist/execution/data-channel.js +55 -0
- package/dist/execution/data-channel.js.map +1 -0
- package/dist/execution/execution.service.d.ts +139 -0
- package/dist/execution/execution.service.d.ts.map +1 -1
- package/dist/execution/execution.service.js +273 -10
- package/dist/execution/execution.service.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/infrastructure/bare-metal-plan.d.ts +208 -0
- package/dist/infrastructure/bare-metal-plan.d.ts.map +1 -0
- package/dist/infrastructure/bare-metal-plan.js +363 -0
- package/dist/infrastructure/bare-metal-plan.js.map +1 -0
- package/dist/infrastructure/bare-metal-runner.d.ts +65 -0
- package/dist/infrastructure/bare-metal-runner.d.ts.map +1 -0
- package/dist/infrastructure/bare-metal-runner.js +354 -0
- package/dist/infrastructure/bare-metal-runner.js.map +1 -0
- package/dist/infrastructure/config-payload.d.ts +149 -0
- package/dist/infrastructure/config-payload.d.ts.map +1 -0
- package/dist/infrastructure/config-payload.js +235 -0
- package/dist/infrastructure/config-payload.js.map +1 -0
- package/dist/infrastructure/container-runtime.d.ts +179 -1
- package/dist/infrastructure/container-runtime.d.ts.map +1 -1
- package/dist/infrastructure/container-runtime.js +612 -45
- package/dist/infrastructure/container-runtime.js.map +1 -1
- package/dist/infrastructure/gateway-reachability.d.ts +85 -0
- package/dist/infrastructure/gateway-reachability.d.ts.map +1 -0
- package/dist/infrastructure/gateway-reachability.js +116 -0
- package/dist/infrastructure/gateway-reachability.js.map +1 -0
- package/dist/infrastructure/gateway-redis-db.d.ts +29 -0
- package/dist/infrastructure/gateway-redis-db.d.ts.map +1 -0
- package/dist/infrastructure/gateway-redis-db.js +32 -0
- package/dist/infrastructure/gateway-redis-db.js.map +1 -0
- package/dist/infrastructure/host-inspection.d.ts +223 -0
- package/dist/infrastructure/host-inspection.d.ts.map +1 -0
- package/dist/infrastructure/host-inspection.js +501 -0
- package/dist/infrastructure/host-inspection.js.map +1 -0
- package/dist/infrastructure/infrastructure-gate.d.ts.map +1 -1
- package/dist/infrastructure/infrastructure.service.d.ts +247 -6
- package/dist/infrastructure/infrastructure.service.d.ts.map +1 -1
- package/dist/infrastructure/infrastructure.service.js +571 -96
- package/dist/infrastructure/infrastructure.service.js.map +1 -1
- package/dist/infrastructure/known-machines.d.ts +76 -0
- package/dist/infrastructure/known-machines.d.ts.map +1 -0
- package/dist/infrastructure/known-machines.js +94 -0
- package/dist/infrastructure/known-machines.js.map +1 -0
- package/dist/infrastructure/node-credentials.d.ts +45 -0
- package/dist/infrastructure/node-credentials.d.ts.map +1 -0
- package/dist/infrastructure/node-credentials.js +80 -0
- package/dist/infrastructure/node-credentials.js.map +1 -0
- package/dist/infrastructure/orphan-containers.d.ts +99 -0
- package/dist/infrastructure/orphan-containers.d.ts.map +1 -0
- package/dist/infrastructure/orphan-containers.js +106 -0
- package/dist/infrastructure/orphan-containers.js.map +1 -0
- package/dist/infrastructure/phantom-endpoint-janitor.d.ts +92 -0
- package/dist/infrastructure/phantom-endpoint-janitor.d.ts.map +1 -0
- package/dist/infrastructure/phantom-endpoint-janitor.js +199 -0
- package/dist/infrastructure/phantom-endpoint-janitor.js.map +1 -0
- package/dist/infrastructure/presets/gateway-openresty.d.ts.map +1 -1
- package/dist/infrastructure/presets/gateway-openresty.js +29 -3
- package/dist/infrastructure/presets/gateway-openresty.js.map +1 -1
- package/dist/infrastructure/presets/index.d.ts.map +1 -1
- package/dist/infrastructure/presets/index.js +2 -0
- package/dist/infrastructure/presets/index.js.map +1 -1
- package/dist/infrastructure/presets/minio.d.ts.map +1 -1
- package/dist/infrastructure/presets/minio.js +36 -3
- package/dist/infrastructure/presets/minio.js.map +1 -1
- package/dist/infrastructure/presets/postgres.d.ts.map +1 -1
- package/dist/infrastructure/presets/postgres.js +78 -5
- package/dist/infrastructure/presets/postgres.js.map +1 -1
- package/dist/infrastructure/presets/redis.d.ts.map +1 -1
- package/dist/infrastructure/presets/redis.js +23 -4
- package/dist/infrastructure/presets/redis.js.map +1 -1
- package/dist/infrastructure/presets/registry.d.ts.map +1 -1
- package/dist/infrastructure/presets/registry.js +85 -4
- package/dist/infrastructure/presets/registry.js.map +1 -1
- package/dist/infrastructure/presets/tor.d.ts +112 -0
- package/dist/infrastructure/presets/tor.d.ts.map +1 -0
- package/dist/infrastructure/presets/tor.js +185 -0
- package/dist/infrastructure/presets/tor.js.map +1 -0
- package/dist/infrastructure/presets/torrc.d.ts +54 -0
- package/dist/infrastructure/presets/torrc.d.ts.map +1 -0
- package/dist/infrastructure/presets/torrc.js +153 -0
- package/dist/infrastructure/presets/torrc.js.map +1 -0
- package/dist/infrastructure/presets/types.d.ts +17 -0
- package/dist/infrastructure/presets/types.d.ts.map +1 -1
- package/dist/infrastructure/provisioning-outcome.d.ts +85 -0
- package/dist/infrastructure/provisioning-outcome.d.ts.map +1 -0
- package/dist/infrastructure/provisioning-outcome.js +122 -0
- package/dist/infrastructure/provisioning-outcome.js.map +1 -0
- package/dist/infrastructure/server-registry.d.ts +27 -0
- package/dist/infrastructure/server-registry.d.ts.map +1 -1
- package/dist/infrastructure/server-registry.js +60 -5
- package/dist/infrastructure/server-registry.js.map +1 -1
- package/dist/infrastructure/service-binding.d.ts +60 -0
- package/dist/infrastructure/service-binding.d.ts.map +1 -0
- package/dist/infrastructure/service-binding.js +90 -0
- package/dist/infrastructure/service-binding.js.map +1 -0
- package/dist/infrastructure/service-credentials.d.ts +105 -0
- package/dist/infrastructure/service-credentials.d.ts.map +1 -0
- package/dist/infrastructure/service-credentials.js +141 -0
- package/dist/infrastructure/service-credentials.js.map +1 -0
- package/dist/infrastructure/service-resolver.d.ts +191 -12
- package/dist/infrastructure/service-resolver.d.ts.map +1 -1
- package/dist/infrastructure/service-resolver.js +521 -60
- package/dist/infrastructure/service-resolver.js.map +1 -1
- package/dist/infrastructure/shipped-config.d.ts +43 -0
- package/dist/infrastructure/shipped-config.d.ts.map +1 -0
- package/dist/infrastructure/shipped-config.js +159 -0
- package/dist/infrastructure/shipped-config.js.map +1 -0
- package/dist/infrastructure/stack-infra-manager.d.ts +7 -1
- package/dist/infrastructure/stack-infra-manager.d.ts.map +1 -1
- package/dist/infrastructure/stack-infra-manager.js +32 -14
- package/dist/infrastructure/stack-infra-manager.js.map +1 -1
- package/dist/infrastructure/types.d.ts +278 -6
- package/dist/infrastructure/types.d.ts.map +1 -1
- package/dist/mcp/mcp-bridge.d.ts.map +1 -1
- package/dist/mcp/mcp-bridge.js.map +1 -1
- package/dist/mcp/tool-groups/apps.tools.d.ts +2 -1
- package/dist/mcp/tool-groups/apps.tools.d.ts.map +1 -1
- package/dist/mcp/tool-groups/apps.tools.js +2 -2
- package/dist/mcp/tool-groups/apps.tools.js.map +1 -1
- package/dist/mcp/tool-groups/infra.tools.d.ts +16 -2
- package/dist/mcp/tool-groups/infra.tools.d.ts.map +1 -1
- package/dist/mcp/tool-groups/infra.tools.js +51 -41
- package/dist/mcp/tool-groups/infra.tools.js.map +1 -1
- package/dist/mcp/tool-groups/kb.tools.d.ts.map +1 -1
- package/dist/mcp/tool-groups/kb.tools.js +4 -6
- package/dist/mcp/tool-groups/kb.tools.js.map +1 -1
- package/dist/mcp/tool-groups/management.tools.d.ts +20 -3
- package/dist/mcp/tool-groups/management.tools.d.ts.map +1 -1
- package/dist/mcp/tool-groups/management.tools.js +158 -125
- package/dist/mcp/tool-groups/management.tools.js.map +1 -1
- package/dist/mcp/tool-groups/monitoring.tools.d.ts +10 -1
- package/dist/mcp/tool-groups/monitoring.tools.d.ts.map +1 -1
- package/dist/mcp/tool-groups/monitoring.tools.js +65 -18
- package/dist/mcp/tool-groups/monitoring.tools.js.map +1 -1
- package/dist/mcp/unavailable-tools.d.ts +35 -0
- package/dist/mcp/unavailable-tools.d.ts.map +1 -0
- package/dist/mcp/unavailable-tools.js +59 -0
- package/dist/mcp/unavailable-tools.js.map +1 -0
- package/dist/monitoring/app-health.indicator.d.ts.map +1 -1
- package/dist/monitoring/app-health.indicator.js +12 -0
- package/dist/monitoring/app-health.indicator.js.map +1 -1
- package/dist/monitoring/desktop-notifier.d.ts +26 -0
- package/dist/monitoring/desktop-notifier.d.ts.map +1 -0
- package/dist/monitoring/desktop-notifier.js +49 -0
- package/dist/monitoring/desktop-notifier.js.map +1 -0
- package/dist/monitoring/docker-health.indicator.d.ts +17 -1
- package/dist/monitoring/docker-health.indicator.d.ts.map +1 -1
- package/dist/monitoring/docker-health.indicator.js +38 -8
- package/dist/monitoring/docker-health.indicator.js.map +1 -1
- package/dist/monitoring/event-loop-stall.indicator.d.ts +26 -0
- package/dist/monitoring/event-loop-stall.indicator.d.ts.map +1 -0
- package/dist/monitoring/event-loop-stall.indicator.js +47 -0
- package/dist/monitoring/event-loop-stall.indicator.js.map +1 -0
- package/dist/monitoring/event-loop-watch.d.ts +124 -0
- package/dist/monitoring/event-loop-watch.d.ts.map +1 -0
- package/dist/monitoring/event-loop-watch.js +179 -0
- package/dist/monitoring/event-loop-watch.js.map +1 -0
- package/dist/monitoring/host-disk.d.ts +6 -0
- package/dist/monitoring/host-disk.d.ts.map +1 -0
- package/dist/monitoring/host-disk.js +26 -0
- package/dist/monitoring/host-disk.js.map +1 -0
- package/dist/monitoring/log-line.d.ts +188 -0
- package/dist/monitoring/log-line.d.ts.map +1 -0
- package/dist/monitoring/log-line.js +458 -0
- package/dist/monitoring/log-line.js.map +1 -0
- package/dist/monitoring/log-manager.d.ts +89 -8
- package/dist/monitoring/log-manager.d.ts.map +1 -1
- package/dist/monitoring/log-manager.js +246 -77
- package/dist/monitoring/log-manager.js.map +1 -1
- package/dist/observability/index.d.ts +4 -0
- package/dist/observability/index.d.ts.map +1 -0
- package/dist/observability/index.js +3 -0
- package/dist/observability/index.js.map +1 -0
- package/dist/observability/metrics-bridge.d.ts +87 -0
- package/dist/observability/metrics-bridge.d.ts.map +1 -0
- package/dist/observability/metrics-bridge.js +295 -0
- package/dist/observability/metrics-bridge.js.map +1 -0
- package/dist/observability/metrics-server.d.ts +68 -0
- package/dist/observability/metrics-server.d.ts.map +1 -0
- package/dist/observability/metrics-server.js +146 -0
- package/dist/observability/metrics-server.js.map +1 -0
- package/dist/orchestrator/app-handle.d.ts +170 -3
- package/dist/orchestrator/app-handle.d.ts.map +1 -1
- package/dist/orchestrator/app-handle.js +177 -3
- package/dist/orchestrator/app-handle.js.map +1 -1
- package/dist/orchestrator/app-health.d.ts +33 -0
- package/dist/orchestrator/app-health.d.ts.map +1 -0
- package/dist/orchestrator/app-health.js +74 -0
- package/dist/orchestrator/app-health.js.map +1 -0
- package/dist/orchestrator/app-traffic.d.ts +14 -0
- package/dist/orchestrator/app-traffic.d.ts.map +1 -0
- package/dist/orchestrator/app-traffic.js +33 -0
- package/dist/orchestrator/app-traffic.js.map +1 -0
- package/dist/orchestrator/bootstrap-loader-child.d.ts +19 -0
- package/dist/orchestrator/bootstrap-loader-child.d.ts.map +1 -0
- package/dist/orchestrator/bootstrap-loader-child.js +43 -0
- package/dist/orchestrator/bootstrap-loader-child.js.map +1 -0
- package/dist/orchestrator/bootstrap-loader.d.ts +72 -0
- package/dist/orchestrator/bootstrap-loader.d.ts.map +1 -1
- package/dist/orchestrator/bootstrap-loader.js +350 -5
- package/dist/orchestrator/bootstrap-loader.js.map +1 -1
- package/dist/orchestrator/bootstrap-process.d.ts +148 -4
- package/dist/orchestrator/bootstrap-process.d.ts.map +1 -1
- package/dist/orchestrator/bootstrap-process.js +496 -18
- package/dist/orchestrator/bootstrap-process.js.map +1 -1
- package/dist/orchestrator/build-service.d.ts +50 -3
- package/dist/orchestrator/build-service.d.ts.map +1 -1
- package/dist/orchestrator/build-service.js +357 -85
- package/dist/orchestrator/build-service.js.map +1 -1
- package/dist/orchestrator/call-exposed-service.d.ts +7 -0
- package/dist/orchestrator/call-exposed-service.d.ts.map +1 -0
- package/dist/orchestrator/call-exposed-service.js +50 -0
- package/dist/orchestrator/call-exposed-service.js.map +1 -0
- package/dist/orchestrator/classic-launcher.d.ts +1 -0
- package/dist/orchestrator/classic-launcher.d.ts.map +1 -1
- package/dist/orchestrator/classic-launcher.js +34 -3
- package/dist/orchestrator/classic-launcher.js.map +1 -1
- package/dist/orchestrator/file-watcher.d.ts +78 -1
- package/dist/orchestrator/file-watcher.d.ts.map +1 -1
- package/dist/orchestrator/file-watcher.js +240 -43
- package/dist/orchestrator/file-watcher.js.map +1 -1
- package/dist/orchestrator/module-worker-process.d.ts.map +1 -1
- package/dist/orchestrator/module-worker-process.js +10 -0
- package/dist/orchestrator/module-worker-process.js.map +1 -1
- package/dist/orchestrator/orchestrator.service.d.ts +540 -7
- package/dist/orchestrator/orchestrator.service.d.ts.map +1 -1
- package/dist/orchestrator/orchestrator.service.js +2106 -242
- package/dist/orchestrator/orchestrator.service.js.map +1 -1
- package/dist/orchestrator/owned-pids.d.ts +83 -0
- package/dist/orchestrator/owned-pids.d.ts.map +1 -0
- package/dist/orchestrator/owned-pids.js +93 -0
- package/dist/orchestrator/owned-pids.js.map +1 -0
- package/dist/orchestrator/port-conflict.d.ts +102 -0
- package/dist/orchestrator/port-conflict.d.ts.map +1 -0
- package/dist/orchestrator/port-conflict.js +131 -0
- package/dist/orchestrator/port-conflict.js.map +1 -0
- package/dist/orchestrator/process-janitor.d.ts +227 -0
- package/dist/orchestrator/process-janitor.d.ts.map +1 -0
- package/dist/orchestrator/process-janitor.js +495 -0
- package/dist/orchestrator/process-janitor.js.map +1 -0
- package/dist/orchestrator/ps-metrics.d.ts +37 -0
- package/dist/orchestrator/ps-metrics.d.ts.map +1 -0
- package/dist/orchestrator/ps-metrics.js +60 -0
- package/dist/orchestrator/ps-metrics.js.map +1 -0
- package/dist/orchestrator/service-router.d.ts +58 -2
- package/dist/orchestrator/service-router.d.ts.map +1 -1
- package/dist/orchestrator/service-router.js +157 -25
- package/dist/orchestrator/service-router.js.map +1 -1
- package/dist/orchestrator/topology-query.d.ts +49 -0
- package/dist/orchestrator/topology-query.d.ts.map +1 -0
- package/dist/orchestrator/topology-query.js +54 -0
- package/dist/orchestrator/topology-query.js.map +1 -0
- package/dist/project/artifact-builder.d.ts +174 -3
- package/dist/project/artifact-builder.d.ts.map +1 -1
- package/dist/project/artifact-builder.js +454 -42
- package/dist/project/artifact-builder.js.map +1 -1
- package/dist/project/build-decision.d.ts +58 -0
- package/dist/project/build-decision.d.ts.map +1 -0
- package/dist/project/build-decision.js +59 -0
- package/dist/project/build-decision.js.map +1 -0
- package/dist/project/config-resolver.d.ts +6 -4
- package/dist/project/config-resolver.d.ts.map +1 -1
- package/dist/project/config-resolver.js +158 -78
- package/dist/project/config-resolver.js.map +1 -1
- package/dist/project/declared-config.d.ts +42 -0
- package/dist/project/declared-config.d.ts.map +1 -0
- package/dist/project/declared-config.js +60 -0
- package/dist/project/declared-config.js.map +1 -0
- package/dist/project/dependency-analyzer.d.ts.map +1 -1
- package/dist/project/dependency-analyzer.js.map +1 -1
- package/dist/project/deploy-phases.d.ts +35 -0
- package/dist/project/deploy-phases.d.ts.map +1 -0
- package/dist/project/deploy-phases.js +75 -0
- package/dist/project/deploy-phases.js.map +1 -0
- package/dist/project/exec-in-group.d.ts +49 -0
- package/dist/project/exec-in-group.d.ts.map +1 -0
- package/dist/project/exec-in-group.js +100 -0
- package/dist/project/exec-in-group.js.map +1 -0
- package/dist/project/node-app-config.d.ts +101 -0
- package/dist/project/node-app-config.d.ts.map +1 -0
- package/dist/project/node-app-config.js +134 -0
- package/dist/project/node-app-config.js.map +1 -0
- package/dist/project/node-app-health.d.ts +76 -0
- package/dist/project/node-app-health.d.ts.map +1 -0
- package/dist/project/node-app-health.js +119 -0
- package/dist/project/node-app-health.js.map +1 -0
- package/dist/project/operator-account.d.ts +287 -0
- package/dist/project/operator-account.d.ts.map +1 -0
- package/dist/project/operator-account.js +373 -0
- package/dist/project/operator-account.js.map +1 -0
- package/dist/project/registry.d.ts +46 -8
- package/dist/project/registry.d.ts.map +1 -1
- package/dist/project/registry.js +238 -40
- package/dist/project/registry.js.map +1 -1
- package/dist/project/requirements-scanner.d.ts +1 -0
- package/dist/project/requirements-scanner.d.ts.map +1 -1
- package/dist/project/requirements-scanner.js +28 -3
- package/dist/project/requirements-scanner.js.map +1 -1
- package/dist/project/stack-env.d.ts +37 -0
- package/dist/project/stack-env.d.ts.map +1 -0
- package/dist/project/stack-env.js +85 -0
- package/dist/project/stack-env.js.map +1 -0
- package/dist/project/stand-containers.d.ts +4 -0
- package/dist/project/stand-containers.d.ts.map +1 -0
- package/dist/project/stand-containers.js +27 -0
- package/dist/project/stand-containers.js.map +1 -0
- package/dist/project/working-tree.d.ts +62 -0
- package/dist/project/working-tree.d.ts.map +1 -0
- package/dist/project/working-tree.js +101 -0
- package/dist/project/working-tree.js.map +1 -0
- package/dist/release/artifact-store.d.ts +360 -0
- package/dist/release/artifact-store.d.ts.map +1 -0
- package/dist/release/artifact-store.js +1040 -0
- package/dist/release/artifact-store.js.map +1 -0
- package/dist/release/attest-on-node.d.ts +92 -0
- package/dist/release/attest-on-node.d.ts.map +1 -0
- package/dist/release/attest-on-node.js +207 -0
- package/dist/release/attest-on-node.js.map +1 -0
- package/dist/release/attest.d.ts +135 -0
- package/dist/release/attest.d.ts.map +1 -0
- package/dist/release/attest.js +199 -0
- package/dist/release/attest.js.map +1 -0
- package/dist/release/build-run.d.ts +151 -0
- package/dist/release/build-run.d.ts.map +1 -0
- package/dist/release/build-run.js +609 -0
- package/dist/release/build-run.js.map +1 -0
- package/dist/release/builder.d.ts +57 -0
- package/dist/release/builder.d.ts.map +1 -0
- package/dist/release/builder.js +120 -0
- package/dist/release/builder.js.map +1 -0
- package/dist/release/daemon-omni.d.ts +41 -0
- package/dist/release/daemon-omni.d.ts.map +1 -0
- package/dist/release/daemon-omni.js +59 -0
- package/dist/release/daemon-omni.js.map +1 -0
- package/dist/release/definition-inputs.d.ts +34 -0
- package/dist/release/definition-inputs.d.ts.map +1 -0
- package/dist/release/definition-inputs.js +90 -0
- package/dist/release/definition-inputs.js.map +1 -0
- package/dist/release/delivered.d.ts +53 -0
- package/dist/release/delivered.d.ts.map +1 -0
- package/dist/release/delivered.js +81 -0
- package/dist/release/delivered.js.map +1 -0
- package/dist/release/layout.d.ts +61 -0
- package/dist/release/layout.d.ts.map +1 -0
- package/dist/release/layout.js +114 -0
- package/dist/release/layout.js.map +1 -0
- package/dist/release/load.d.ts +44 -0
- package/dist/release/load.d.ts.map +1 -0
- package/dist/release/load.js +84 -0
- package/dist/release/load.js.map +1 -0
- package/dist/release/manifest.d.ts +299 -0
- package/dist/release/manifest.d.ts.map +1 -0
- package/dist/release/manifest.js +176 -0
- package/dist/release/manifest.js.map +1 -0
- package/dist/release/migrations.d.ts +32 -0
- package/dist/release/migrations.d.ts.map +1 -0
- package/dist/release/migrations.js +51 -0
- package/dist/release/migrations.js.map +1 -0
- package/dist/release/publish.d.ts +71 -0
- package/dist/release/publish.d.ts.map +1 -0
- package/dist/release/publish.js +102 -0
- package/dist/release/publish.js.map +1 -0
- package/dist/release/store.d.ts +174 -0
- package/dist/release/store.d.ts.map +1 -0
- package/dist/release/store.js +257 -0
- package/dist/release/store.js.map +1 -0
- package/dist/services/alert.rpc-service.d.ts +11 -15
- package/dist/services/alert.rpc-service.d.ts.map +1 -1
- package/dist/services/alert.rpc-service.js +29 -14
- package/dist/services/alert.rpc-service.js.map +1 -1
- package/dist/services/alert.service.d.ts +96 -59
- package/dist/services/alert.service.d.ts.map +1 -1
- package/dist/services/alert.service.js +455 -70
- package/dist/services/alert.service.js.map +1 -1
- package/dist/services/anonymous-input.d.ts +26 -0
- package/dist/services/anonymous-input.d.ts.map +1 -0
- package/dist/services/anonymous-input.js +44 -0
- package/dist/services/anonymous-input.js.map +1 -0
- package/dist/services/audit.rpc-service.d.ts +27 -0
- package/dist/services/audit.rpc-service.d.ts.map +1 -0
- package/dist/services/audit.rpc-service.js +57 -0
- package/dist/services/audit.rpc-service.js.map +1 -0
- package/dist/services/audit.service.d.ts +165 -0
- package/dist/services/audit.service.d.ts.map +1 -0
- package/dist/services/audit.service.js +242 -0
- package/dist/services/audit.service.js.map +1 -0
- package/dist/services/auth-context.d.ts +21 -1
- package/dist/services/auth-context.d.ts.map +1 -1
- package/dist/services/auth-context.js +51 -5
- package/dist/services/auth-context.js.map +1 -1
- package/dist/services/auth.rpc-service.d.ts +0 -1
- package/dist/services/auth.rpc-service.d.ts.map +1 -1
- package/dist/services/auth.rpc-service.js +23 -9
- package/dist/services/auth.rpc-service.js.map +1 -1
- package/dist/services/auth.service.d.ts +47 -33
- package/dist/services/auth.service.d.ts.map +1 -1
- package/dist/services/auth.service.js +163 -24
- package/dist/services/auth.service.js.map +1 -1
- package/dist/services/backup-pipeline.d.ts +97 -0
- package/dist/services/backup-pipeline.d.ts.map +1 -0
- package/dist/services/backup-pipeline.js +286 -0
- package/dist/services/backup-pipeline.js.map +1 -0
- package/dist/services/backup-schedule.d.ts +69 -0
- package/dist/services/backup-schedule.d.ts.map +1 -0
- package/dist/services/backup-schedule.js +132 -0
- package/dist/services/backup-schedule.js.map +1 -0
- package/dist/services/backup.rpc-service.d.ts +31 -2
- package/dist/services/backup.rpc-service.d.ts.map +1 -1
- package/dist/services/backup.rpc-service.js +53 -0
- package/dist/services/backup.rpc-service.js.map +1 -1
- package/dist/services/backup.service.d.ts +458 -14
- package/dist/services/backup.service.d.ts.map +1 -1
- package/dist/services/backup.service.js +1368 -170
- package/dist/services/backup.service.js.map +1 -1
- package/dist/services/bundle-build-worker.d.ts +19 -0
- package/dist/services/bundle-build-worker.d.ts.map +1 -0
- package/dist/services/bundle-build-worker.js +40 -0
- package/dist/services/bundle-build-worker.js.map +1 -0
- package/dist/services/bundle-builder.d.ts +487 -0
- package/dist/services/bundle-builder.d.ts.map +1 -0
- package/dist/services/bundle-builder.js +1083 -0
- package/dist/services/bundle-builder.js.map +1 -0
- package/dist/services/bundle-worker-protocol.d.ts +19 -0
- package/dist/services/bundle-worker-protocol.d.ts.map +1 -0
- package/dist/services/bundle-worker-protocol.js +32 -0
- package/dist/services/bundle-worker-protocol.js.map +1 -0
- package/dist/services/deploy.rpc-service.d.ts +4 -1
- package/dist/services/deploy.rpc-service.d.ts.map +1 -1
- package/dist/services/deploy.rpc-service.js +10 -0
- package/dist/services/deploy.rpc-service.js.map +1 -1
- package/dist/services/deploy.service.d.ts +14 -24
- package/dist/services/deploy.service.d.ts.map +1 -1
- package/dist/services/deploy.service.js +46 -5
- package/dist/services/deploy.service.js.map +1 -1
- package/dist/services/discovery.rpc-service.d.ts +2 -1
- package/dist/services/discovery.rpc-service.d.ts.map +1 -1
- package/dist/services/discovery.rpc-service.js.map +1 -1
- package/dist/services/discovery.service.d.ts +32 -28
- package/dist/services/discovery.service.d.ts.map +1 -1
- package/dist/services/discovery.service.js +108 -133
- package/dist/services/discovery.service.js.map +1 -1
- package/dist/services/event-broadcaster.rpc-service.d.ts +36 -23
- package/dist/services/event-broadcaster.rpc-service.d.ts.map +1 -1
- package/dist/services/event-broadcaster.rpc-service.js +70 -32
- package/dist/services/event-broadcaster.rpc-service.js.map +1 -1
- package/dist/services/event-broadcaster.service.d.ts.map +1 -1
- package/dist/services/event-broadcaster.service.js.map +1 -1
- package/dist/services/fleet.rpc-service.d.ts +2 -1
- package/dist/services/fleet.rpc-service.d.ts.map +1 -1
- package/dist/services/fleet.rpc-service.js +3 -1
- package/dist/services/fleet.rpc-service.js.map +1 -1
- package/dist/services/fleet.service.d.ts +32 -34
- package/dist/services/fleet.service.d.ts.map +1 -1
- package/dist/services/fleet.service.js +66 -10
- package/dist/services/fleet.service.js.map +1 -1
- package/dist/services/health-check.rpc-service.d.ts +2 -1
- package/dist/services/health-check.rpc-service.d.ts.map +1 -1
- package/dist/services/health-check.rpc-service.js.map +1 -1
- package/dist/services/health-check.service.d.ts +50 -36
- package/dist/services/health-check.service.d.ts.map +1 -1
- package/dist/services/health-check.service.js +179 -182
- package/dist/services/health-check.service.js.map +1 -1
- package/dist/services/infrastructure.rpc-service.d.ts +297 -2
- package/dist/services/infrastructure.rpc-service.d.ts.map +1 -1
- package/dist/services/infrastructure.rpc-service.js +523 -11
- package/dist/services/infrastructure.rpc-service.js.map +1 -1
- package/dist/services/kubernetes.rpc-service.d.ts +2 -1
- package/dist/services/kubernetes.rpc-service.d.ts.map +1 -1
- package/dist/services/kubernetes.rpc-service.js.map +1 -1
- package/dist/services/kubernetes.service.d.ts +5 -39
- package/dist/services/kubernetes.service.d.ts.map +1 -1
- package/dist/services/kubernetes.service.js +29 -5
- package/dist/services/kubernetes.service.js.map +1 -1
- package/dist/services/local-bundle.d.ts +151 -0
- package/dist/services/local-bundle.d.ts.map +1 -0
- package/dist/services/local-bundle.js +233 -0
- package/dist/services/local-bundle.js.map +1 -0
- package/dist/services/lockout.d.ts +28 -0
- package/dist/services/lockout.d.ts.map +1 -0
- package/dist/services/lockout.js +41 -0
- package/dist/services/lockout.js.map +1 -0
- package/dist/services/log-collector.rpc-service.d.ts +16 -0
- package/dist/services/log-collector.rpc-service.d.ts.map +1 -1
- package/dist/services/log-collector.rpc-service.js +16 -0
- package/dist/services/log-collector.rpc-service.js.map +1 -1
- package/dist/services/log-collector.service.d.ts +172 -54
- package/dist/services/log-collector.service.d.ts.map +1 -1
- package/dist/services/log-collector.service.js +492 -102
- package/dist/services/log-collector.service.js.map +1 -1
- package/dist/services/log-retention.d.ts +50 -0
- package/dist/services/log-retention.d.ts.map +1 -0
- package/dist/services/log-retention.js +52 -0
- package/dist/services/log-retention.js.map +1 -0
- package/dist/services/maintenance-lock.d.ts +162 -0
- package/dist/services/maintenance-lock.d.ts.map +1 -0
- package/dist/services/maintenance-lock.js +316 -0
- package/dist/services/maintenance-lock.js.map +1 -0
- package/dist/services/master-address.d.ts +107 -0
- package/dist/services/master-address.d.ts.map +1 -0
- package/dist/services/master-address.js +154 -0
- package/dist/services/master-address.js.map +1 -0
- package/dist/services/migration-output.d.ts +16 -0
- package/dist/services/migration-output.d.ts.map +1 -0
- package/dist/services/migration-output.js +22 -0
- package/dist/services/migration-output.js.map +1 -0
- package/dist/services/netron-telemetry-transport.d.ts.map +1 -1
- package/dist/services/netron-telemetry-transport.js +0 -4
- package/dist/services/netron-telemetry-transport.js.map +1 -1
- package/dist/services/node-deploy-lease.d.ts +154 -0
- package/dist/services/node-deploy-lease.d.ts.map +1 -0
- package/dist/services/node-deploy-lease.js +386 -0
- package/dist/services/node-deploy-lease.js.map +1 -0
- package/dist/services/node-health.repository.d.ts +63 -4
- package/dist/services/node-health.repository.d.ts.map +1 -1
- package/dist/services/node-health.repository.js +108 -67
- package/dist/services/node-health.repository.js.map +1 -1
- package/dist/services/node-manager.rpc-service.d.ts +355 -10
- package/dist/services/node-manager.rpc-service.d.ts.map +1 -1
- package/dist/services/node-manager.rpc-service.js +730 -34
- package/dist/services/node-manager.rpc-service.js.map +1 -1
- package/dist/services/node-manager.service.d.ts +267 -95
- package/dist/services/node-manager.service.d.ts.map +1 -1
- package/dist/services/node-manager.service.js +806 -104
- package/dist/services/node-manager.service.js.map +1 -1
- package/dist/services/node-upgrade.d.ts +129 -0
- package/dist/services/node-upgrade.d.ts.map +1 -0
- package/dist/services/node-upgrade.js +161 -0
- package/dist/services/node-upgrade.js.map +1 -0
- package/dist/services/node-upgrade.service.d.ts +232 -0
- package/dist/services/node-upgrade.service.d.ts.map +1 -0
- package/dist/services/node-upgrade.service.js +468 -0
- package/dist/services/node-upgrade.service.js.map +1 -0
- package/dist/services/pipeline.rpc-service.d.ts +2 -1
- package/dist/services/pipeline.rpc-service.d.ts.map +1 -1
- package/dist/services/pipeline.rpc-service.js.map +1 -1
- package/dist/services/pipeline.service.d.ts +30 -46
- package/dist/services/pipeline.service.d.ts.map +1 -1
- package/dist/services/pipeline.service.js +125 -11
- package/dist/services/pipeline.service.js.map +1 -1
- package/dist/services/project.rpc-service.d.ts +137 -5
- package/dist/services/project.rpc-service.d.ts.map +1 -1
- package/dist/services/project.rpc-service.js +258 -22
- package/dist/services/project.rpc-service.js.map +1 -1
- package/dist/services/project.service.d.ts +680 -9
- package/dist/services/project.service.d.ts.map +1 -1
- package/dist/services/project.service.js +2914 -293
- package/dist/services/project.service.js.map +1 -1
- package/dist/services/redeploy-decision.d.ts +176 -0
- package/dist/services/redeploy-decision.d.ts.map +1 -0
- package/dist/services/redeploy-decision.js +142 -0
- package/dist/services/redeploy-decision.js.map +1 -0
- package/dist/services/release.rpc-service.d.ts +159 -0
- package/dist/services/release.rpc-service.d.ts.map +1 -0
- package/dist/services/release.rpc-service.js +395 -0
- package/dist/services/release.rpc-service.js.map +1 -0
- package/dist/services/release.service.d.ts +114 -0
- package/dist/services/release.service.d.ts.map +1 -0
- package/dist/services/release.service.js +208 -0
- package/dist/services/release.service.js.map +1 -0
- package/dist/services/remote-deployer.service.d.ts +569 -8
- package/dist/services/remote-deployer.service.d.ts.map +1 -1
- package/dist/services/remote-deployer.service.js +1618 -169
- package/dist/services/remote-deployer.service.js.map +1 -1
- package/dist/services/remote-ops.service.d.ts +51 -9
- package/dist/services/remote-ops.service.d.ts.map +1 -1
- package/dist/services/remote-ops.service.js +235 -39
- package/dist/services/remote-ops.service.js.map +1 -1
- package/dist/services/remote-provisioner.d.ts +122 -0
- package/dist/services/remote-provisioner.d.ts.map +1 -0
- package/dist/services/remote-provisioner.js +258 -0
- package/dist/services/remote-provisioner.js.map +1 -0
- package/dist/services/secrets.rpc-service.d.ts +51 -2
- package/dist/services/secrets.rpc-service.d.ts.map +1 -1
- package/dist/services/secrets.rpc-service.js +81 -2
- package/dist/services/secrets.rpc-service.js.map +1 -1
- package/dist/services/secrets.service.d.ts +108 -9
- package/dist/services/secrets.service.d.ts.map +1 -1
- package/dist/services/secrets.service.js +313 -48
- package/dist/services/secrets.service.js.map +1 -1
- package/dist/services/slave-storage.service.d.ts +107 -0
- package/dist/services/slave-storage.service.d.ts.map +1 -1
- package/dist/services/slave-storage.service.js +171 -2
- package/dist/services/slave-storage.service.js.map +1 -1
- package/dist/services/static-carry-forward.d.ts +85 -0
- package/dist/services/static-carry-forward.d.ts.map +1 -0
- package/dist/services/static-carry-forward.js +159 -0
- package/dist/services/static-carry-forward.js.map +1 -0
- package/dist/services/sync-policy.d.ts +167 -0
- package/dist/services/sync-policy.d.ts.map +1 -0
- package/dist/services/sync-policy.js +161 -0
- package/dist/services/sync-policy.js.map +1 -0
- package/dist/services/sync-summary.d.ts +39 -0
- package/dist/services/sync-summary.d.ts.map +1 -0
- package/dist/services/sync-summary.js +62 -0
- package/dist/services/sync-summary.js.map +1 -0
- package/dist/services/sync.rpc-service.d.ts +19 -7
- package/dist/services/sync.rpc-service.d.ts.map +1 -1
- package/dist/services/sync.rpc-service.js +36 -5
- package/dist/services/sync.rpc-service.js.map +1 -1
- package/dist/services/sync.service.d.ts +350 -18
- package/dist/services/sync.service.d.ts.map +1 -1
- package/dist/services/sync.service.js +864 -116
- package/dist/services/sync.service.js.map +1 -1
- package/dist/services/system-info.rpc-service.d.ts +2 -1
- package/dist/services/system-info.rpc-service.d.ts.map +1 -1
- package/dist/services/system-info.rpc-service.js.map +1 -1
- package/dist/services/system-info.service.d.ts +56 -68
- package/dist/services/system-info.service.d.ts.map +1 -1
- package/dist/services/system-info.service.js +240 -127
- package/dist/services/system-info.service.js.map +1 -1
- package/dist/services/telemetry.rpc-service.d.ts.map +1 -1
- package/dist/services/telemetry.rpc-service.js +9 -3
- package/dist/services/telemetry.rpc-service.js.map +1 -1
- package/dist/services/telemetry.service.d.ts +4 -2
- package/dist/services/telemetry.service.d.ts.map +1 -1
- package/dist/services/telemetry.service.js +4 -2
- package/dist/services/telemetry.service.js.map +1 -1
- package/dist/services/trace-collector.rpc-service.d.ts +2 -1
- package/dist/services/trace-collector.rpc-service.d.ts.map +1 -1
- package/dist/services/trace-collector.rpc-service.js.map +1 -1
- package/dist/services/trace-collector.service.d.ts +2 -41
- package/dist/services/trace-collector.service.d.ts.map +1 -1
- package/dist/services/trace-collector.service.js +27 -7
- package/dist/services/trace-collector.service.js.map +1 -1
- package/dist/services/wait-for-postgres.d.ts +28 -0
- package/dist/services/wait-for-postgres.d.ts.map +1 -0
- package/dist/services/wait-for-postgres.js +77 -0
- package/dist/services/wait-for-postgres.js.map +1 -0
- package/dist/shared/alert-expression.d.ts +60 -0
- package/dist/shared/alert-expression.d.ts.map +1 -0
- package/dist/shared/alert-expression.js +124 -0
- package/dist/shared/alert-expression.js.map +1 -0
- package/dist/shared/audit-outcome.d.ts +39 -0
- package/dist/shared/audit-outcome.d.ts.map +1 -0
- package/dist/shared/audit-outcome.js +45 -0
- package/dist/shared/audit-outcome.js.map +1 -0
- package/dist/shared/build-freshness.d.ts +59 -0
- package/dist/shared/build-freshness.d.ts.map +1 -0
- package/dist/shared/build-freshness.js +113 -0
- package/dist/shared/build-freshness.js.map +1 -0
- package/dist/shared/build-stamp.d.ts +26 -0
- package/dist/shared/build-stamp.d.ts.map +1 -0
- package/dist/shared/build-stamp.js +36 -0
- package/dist/shared/build-stamp.js.map +1 -0
- package/dist/shared/describe-error.d.ts +22 -0
- package/dist/shared/describe-error.d.ts.map +1 -0
- package/dist/shared/describe-error.js +39 -0
- package/dist/shared/describe-error.js.map +1 -0
- package/dist/shared/dto/alerts.d.ts +101 -0
- package/dist/shared/dto/alerts.d.ts.map +1 -0
- package/dist/shared/dto/alerts.js +9 -0
- package/dist/shared/dto/alerts.js.map +1 -0
- package/dist/shared/dto/auth.d.ts +49 -0
- package/dist/shared/dto/auth.d.ts.map +1 -0
- package/dist/shared/dto/auth.js +14 -0
- package/dist/shared/dto/auth.js.map +1 -0
- package/dist/shared/dto/backups.d.ts +16 -0
- package/dist/shared/dto/backups.d.ts.map +1 -0
- package/dist/shared/dto/backups.js +9 -0
- package/dist/shared/dto/backups.js.map +1 -0
- package/dist/shared/dto/deploy.d.ts +28 -0
- package/dist/shared/dto/deploy.d.ts.map +1 -0
- package/dist/shared/dto/deploy.js +8 -0
- package/dist/shared/dto/deploy.js.map +1 -0
- package/dist/shared/dto/discovery.d.ts +24 -0
- package/dist/shared/dto/discovery.d.ts.map +1 -0
- package/dist/shared/dto/discovery.js +9 -0
- package/dist/shared/dto/discovery.js.map +1 -0
- package/dist/shared/dto/fleet.d.ts +35 -0
- package/dist/shared/dto/fleet.d.ts.map +1 -0
- package/dist/shared/dto/fleet.js +8 -0
- package/dist/shared/dto/fleet.js.map +1 -0
- package/dist/shared/dto/health.d.ts +26 -0
- package/dist/shared/dto/health.d.ts.map +1 -0
- package/dist/shared/dto/health.js +9 -0
- package/dist/shared/dto/health.js.map +1 -0
- package/dist/shared/dto/kubernetes.d.ts +44 -0
- package/dist/shared/dto/kubernetes.d.ts.map +1 -0
- package/dist/shared/dto/kubernetes.js +9 -0
- package/dist/shared/dto/kubernetes.js.map +1 -0
- package/dist/shared/dto/logs.d.ts +77 -0
- package/dist/shared/dto/logs.d.ts.map +1 -0
- package/dist/shared/dto/logs.js +10 -0
- package/dist/shared/dto/logs.js.map +1 -0
- package/dist/shared/dto/metrics.d.ts +65 -0
- package/dist/shared/dto/metrics.d.ts.map +1 -0
- package/dist/shared/dto/metrics.js +17 -0
- package/dist/shared/dto/metrics.js.map +1 -0
- package/dist/shared/dto/nodes.d.ts +347 -0
- package/dist/shared/dto/nodes.d.ts.map +1 -0
- package/dist/shared/dto/nodes.js +8 -0
- package/dist/shared/dto/nodes.js.map +1 -0
- package/dist/shared/dto/pipelines.d.ts +53 -0
- package/dist/shared/dto/pipelines.d.ts.map +1 -0
- package/dist/shared/dto/pipelines.js +9 -0
- package/dist/shared/dto/pipelines.js.map +1 -0
- package/dist/shared/dto/project.d.ts +143 -9
- package/dist/shared/dto/project.d.ts.map +1 -1
- package/dist/shared/dto/services.d.ts +855 -7
- package/dist/shared/dto/services.d.ts.map +1 -1
- package/dist/shared/dto/system-info.d.ts +93 -0
- package/dist/shared/dto/system-info.d.ts.map +1 -0
- package/dist/shared/dto/system-info.js +9 -0
- package/dist/shared/dto/system-info.js.map +1 -0
- package/dist/shared/dto/traces.d.ts +49 -0
- package/dist/shared/dto/traces.d.ts.map +1 -0
- package/dist/shared/dto/traces.js +9 -0
- package/dist/shared/dto/traces.js.map +1 -0
- package/dist/shared/env-config.d.ts +74 -0
- package/dist/shared/env-config.d.ts.map +1 -0
- package/dist/shared/env-config.js +99 -0
- package/dist/shared/env-config.js.map +1 -0
- package/dist/shared/events.d.ts +45 -39
- package/dist/shared/events.d.ts.map +1 -1
- package/dist/shared/events.js +6 -0
- package/dist/shared/events.js.map +1 -1
- package/dist/shared/format.d.ts +1 -1
- package/dist/shared/format.d.ts.map +1 -1
- package/dist/shared/format.js +4 -0
- package/dist/shared/format.js.map +1 -1
- package/dist/shared/local-infra-host.d.ts +28 -0
- package/dist/shared/local-infra-host.d.ts.map +1 -0
- package/dist/shared/local-infra-host.js +28 -0
- package/dist/shared/local-infra-host.js.map +1 -0
- package/dist/shared/node-check.d.ts +37 -0
- package/dist/shared/node-check.d.ts.map +1 -0
- package/dist/shared/node-check.js +41 -0
- package/dist/shared/node-check.js.map +1 -0
- package/dist/shared/paths.d.ts +29 -0
- package/dist/shared/paths.d.ts.map +1 -0
- package/dist/shared/paths.js +54 -0
- package/dist/shared/paths.js.map +1 -0
- package/dist/shared/pnpm.d.ts +29 -0
- package/dist/shared/pnpm.d.ts.map +1 -0
- package/dist/shared/pnpm.js +57 -0
- package/dist/shared/pnpm.js.map +1 -0
- package/dist/shared/private-files.d.ts +29 -0
- package/dist/shared/private-files.d.ts.map +1 -0
- package/dist/shared/private-files.js +109 -0
- package/dist/shared/private-files.js.map +1 -0
- package/dist/shared/redact-env.d.ts +24 -0
- package/dist/shared/redact-env.d.ts.map +1 -0
- package/dist/shared/redact-env.js +36 -0
- package/dist/shared/redact-env.js.map +1 -0
- package/dist/shared/release-reading.d.ts +85 -0
- package/dist/shared/release-reading.d.ts.map +1 -0
- package/dist/shared/release-reading.js +81 -0
- package/dist/shared/release-reading.js.map +1 -0
- package/dist/shared/roles.d.ts +58 -3
- package/dist/shared/roles.d.ts.map +1 -1
- package/dist/shared/roles.js +55 -0
- package/dist/shared/roles.js.map +1 -1
- package/dist/shared/rpcauth.d.ts +13 -0
- package/dist/shared/rpcauth.d.ts.map +1 -0
- package/dist/shared/rpcauth.js +23 -0
- package/dist/shared/rpcauth.js.map +1 -0
- package/dist/shared/shell-escape.d.ts +9 -0
- package/dist/shared/shell-escape.d.ts.map +1 -0
- package/dist/shared/shell-escape.js +11 -0
- package/dist/shared/shell-escape.js.map +1 -0
- package/dist/shared/sync-reading.d.ts +45 -0
- package/dist/shared/sync-reading.d.ts.map +1 -0
- package/dist/shared/sync-reading.js +42 -0
- package/dist/shared/sync-reading.js.map +1 -0
- package/dist/shared/tokens.d.ts +19 -0
- package/dist/shared/tokens.d.ts.map +1 -1
- package/dist/shared/tokens.js +28 -0
- package/dist/shared/tokens.js.map +1 -1
- package/dist/webapp/webapp.service.d.ts +33 -2
- package/dist/webapp/webapp.service.d.ts.map +1 -1
- package/dist/webapp/webapp.service.js +143 -31
- package/dist/webapp/webapp.service.js.map +1 -1
- package/dist/workers/health-monitor-process.d.ts.map +1 -1
- package/dist/workers/health-monitor-process.js +10 -0
- package/dist/workers/health-monitor-process.js.map +1 -1
- package/dist/workers/health-monitor.service.d.ts +45 -3
- package/dist/workers/health-monitor.service.d.ts.map +1 -1
- package/dist/workers/health-monitor.service.js +156 -26
- package/dist/workers/health-monitor.service.js.map +1 -1
- package/dist/workers/system-worker-manager.d.ts +43 -0
- package/dist/workers/system-worker-manager.d.ts.map +1 -1
- package/dist/workers/system-worker-manager.js +150 -3
- package/dist/workers/system-worker-manager.js.map +1 -1
- package/dist/workers/types.d.ts +11 -2
- package/dist/workers/types.d.ts.map +1 -1
- package/package.json +64 -29
- package/webapp/dist/BUILD.json +5 -0
- package/webapp/dist/assets/FormControlLabel-BrH2diyl.js +1 -0
- package/webapp/dist/assets/Grid-BZ7xhoHQ.js +1 -0
- package/webapp/dist/assets/Switch-DGSAxQPM.js +1 -0
- package/webapp/dist/assets/alerts-sXqHJbsa.js +1 -0
- package/webapp/dist/assets/apps-B_Ah8Flh.js +1 -0
- package/webapp/dist/assets/audit-YXihwyui.js +1 -0
- package/webapp/dist/assets/client-CGbBvy9k.js +1 -0
- package/webapp/dist/assets/constants-veKGFkC6.js +1 -0
- package/webapp/dist/assets/containers-Di4NOTLf.js +2 -0
- package/webapp/dist/assets/daemon-memory-CCPZ4cV_.js +1 -0
- package/webapp/dist/assets/dashboard-Cfuf95N3.js +1 -0
- package/webapp/dist/assets/dashboard-builder-CowaYb-M.js +1 -0
- package/webapp/dist/assets/deploy-release-dialog-D6C-XcKY.js +1 -0
- package/webapp/dist/assets/deployments-ChTq5kCw.js +1 -0
- package/webapp/dist/assets/detail-BsGWv7Dj.js +1 -0
- package/webapp/dist/assets/detail-CyVf_cw5.js +2 -0
- package/webapp/dist/assets/detail-D027j6aO.js +8 -0
- package/webapp/dist/assets/formatters-BqjL73N3.js +1 -0
- package/webapp/dist/assets/index-B8AHIPc-.css +1 -0
- package/webapp/dist/assets/index-DiZbrwxT.js +414 -0
- package/webapp/dist/assets/logs-L2b8Oqmo.js +12 -0
- package/webapp/dist/assets/maplibre-gl-MlmG2kbn.js +801 -0
- package/webapp/dist/assets/metrics-CQgjz2Lz.js +2 -0
- package/webapp/dist/assets/nodes-Nwia2Ty6.js +7 -0
- package/webapp/dist/assets/pipelines-DCu_rXOr.js +4 -0
- package/webapp/dist/assets/projects-Bb4aV76h.js +1 -0
- package/webapp/dist/assets/react-CwJFpaho.js +1 -0
- package/webapp/dist/assets/react-apexcharts.esm-Dy0Fez42.js +1956 -0
- package/webapp/dist/assets/release-bits-Dj4EOhhb.js +1 -0
- package/webapp/dist/assets/release-build-panel-D6OCVlhT.js +1 -0
- package/webapp/dist/assets/releases-CnAt3yO6.js +2 -0
- package/webapp/dist/assets/rolldown-runtime-hePW80VL.js +1 -0
- package/webapp/dist/assets/settings-DU-mjiwb.js +1 -0
- package/webapp/dist/assets/settled-pair-C7SjZuns.js +1 -0
- package/webapp/dist/assets/sign-in-Bcvjrr4C.js +1 -0
- package/webapp/dist/assets/stacks-B-e8p50I.js +1 -0
- package/webapp/dist/assets/sync-reading-C3PnJ7AX.js +1 -0
- package/webapp/dist/assets/system-info-BtE3dgfw.js +1 -0
- package/webapp/dist/assets/topology-Cwch-uF1.js +12 -0
- package/webapp/dist/assets/topology-DLioOiRN.css +1 -0
- package/webapp/dist/assets/traces-BbShR-4P.js +1 -0
- package/webapp/dist/assets/use-stack-context-CZUgPq88.js +1 -0
- package/webapp/dist/index.html +6 -7
- package/dist/commands/dev.d.ts +0 -23
- package/dist/commands/dev.d.ts.map +0 -1
- package/dist/commands/dev.js +0 -76
- package/dist/commands/dev.js.map +0 -1
- package/dist/config/define-app.d.ts +0 -6
- package/dist/config/define-app.d.ts.map +0 -1
- package/dist/config/define-app.js +0 -38
- package/dist/config/define-app.js.map +0 -1
- package/dist/infrastructure/gateway-generator.d.ts +0 -63
- package/dist/infrastructure/gateway-generator.d.ts.map +0 -1
- package/dist/infrastructure/gateway-generator.js +0 -253
- package/dist/infrastructure/gateway-generator.js.map +0 -1
- package/dist/monitoring/health.service.d.ts +0 -14
- package/dist/monitoring/health.service.d.ts.map +0 -1
- package/dist/monitoring/health.service.js +0 -50
- package/dist/monitoring/health.service.js.map +0 -1
- package/dist/monitoring/metrics.service.d.ts +0 -14
- package/dist/monitoring/metrics.service.d.ts.map +0 -1
- package/dist/monitoring/metrics.service.js +0 -40
- package/dist/monitoring/metrics.service.js.map +0 -1
- package/dist/orchestrator/bootstrap-runner.d.ts +0 -15
- package/dist/orchestrator/bootstrap-runner.d.ts.map +0 -1
- package/dist/orchestrator/bootstrap-runner.js +0 -152
- package/dist/orchestrator/bootstrap-runner.js.map +0 -1
- package/dist/orchestrator/topology-proxy.d.ts +0 -65
- package/dist/orchestrator/topology-proxy.d.ts.map +0 -1
- package/dist/orchestrator/topology-proxy.js +0 -209
- package/dist/orchestrator/topology-proxy.js.map +0 -1
- package/dist/orchestrator/topology-relay.d.ts +0 -111
- package/dist/orchestrator/topology-relay.d.ts.map +0 -1
- package/dist/orchestrator/topology-relay.js +0 -255
- package/dist/orchestrator/topology-relay.js.map +0 -1
- package/dist/services/auth-middleware.d.ts +0 -47
- package/dist/services/auth-middleware.d.ts.map +0 -1
- package/dist/services/auth-middleware.js +0 -158
- package/dist/services/auth-middleware.js.map +0 -1
- package/dist/services/environment.rpc-service.d.ts +0 -22
- package/dist/services/environment.rpc-service.d.ts.map +0 -1
- package/dist/services/environment.rpc-service.js +0 -65
- package/dist/services/environment.rpc-service.js.map +0 -1
- package/dist/services/environment.service.d.ts +0 -87
- package/dist/services/environment.service.d.ts.map +0 -1
- package/dist/services/environment.service.js +0 -220
- package/dist/services/environment.service.js.map +0 -1
- package/dist/services/metrics-collector.service.d.ts +0 -103
- package/dist/services/metrics-collector.service.d.ts.map +0 -1
- package/dist/services/metrics-collector.service.js +0 -235
- package/dist/services/metrics-collector.service.js.map +0 -1
- package/dist/services/metrics.rpc-service.d.ts +0 -41
- package/dist/services/metrics.rpc-service.d.ts.map +0 -1
- package/dist/services/metrics.rpc-service.js +0 -90
- package/dist/services/metrics.rpc-service.js.map +0 -1
- package/dist/services/metrics.service.d.ts +0 -72
- package/dist/services/metrics.service.d.ts.map +0 -1
- package/dist/services/metrics.service.js +0 -185
- package/dist/services/metrics.service.js.map +0 -1
- package/dist/supervisor/omnitron-supervisor.d.ts +0 -21
- package/dist/supervisor/omnitron-supervisor.d.ts.map +0 -1
- package/dist/supervisor/omnitron-supervisor.js +0 -43
- package/dist/supervisor/omnitron-supervisor.js.map +0 -1
- package/dist/webapp/static-server.d.ts +0 -24
- package/dist/webapp/static-server.d.ts.map +0 -1
- package/dist/webapp/static-server.js +0 -152
- package/dist/webapp/static-server.js.map +0 -1
- package/webapp/dist/assets/Chip-BySxTjyN.js +0 -1
- package/webapp/dist/assets/DialogContentText-YQWiAVrU.js +0 -1
- package/webapp/dist/assets/DialogTitle-CDkgUIwp.js +0 -1
- package/webapp/dist/assets/FormControlLabel-Bu3CCsBM.js +0 -1
- package/webapp/dist/assets/Grid-BFY8hAzL.js +0 -1
- package/webapp/dist/assets/InputAdornment-_Dayk-3y.js +0 -3
- package/webapp/dist/assets/Skeleton-BGj6Sw8m.js +0 -37
- package/webapp/dist/assets/Switch-CA6682vv.js +0 -1
- package/webapp/dist/assets/TableContainer-4epq10KJ.js +0 -1
- package/webapp/dist/assets/TextField-C73KHMVN.js +0 -1
- package/webapp/dist/assets/alerts-JEQALS7Y.js +0 -1
- package/webapp/dist/assets/apps-DMPysEKB.js +0 -1
- package/webapp/dist/assets/client-DlGPter5.js +0 -110
- package/webapp/dist/assets/constants-DqFQJ27U.js +0 -1
- package/webapp/dist/assets/containers-CvkC7do8.js +0 -1
- package/webapp/dist/assets/dashboard-C0Fre9pC.js +0 -1
- package/webapp/dist/assets/dashboard-builder-DAghqdpE.js +0 -1
- package/webapp/dist/assets/deployments-Cg1lI1uT.js +0 -1
- package/webapp/dist/assets/detail-CyFtEson.js +0 -1
- package/webapp/dist/assets/detail-DqoSGKT1.js +0 -8
- package/webapp/dist/assets/formatters-J3Xqn6BQ.js +0 -1
- package/webapp/dist/assets/icons-ynIWYfBP.js +0 -3
- package/webapp/dist/assets/index-CHVXOsTS.css +0 -1
- package/webapp/dist/assets/index-vKrDmEA_.js +0 -259
- package/webapp/dist/assets/isMuiElement-BRL3YuCT.js +0 -1
- package/webapp/dist/assets/logs-IovIij1C.js +0 -11
- package/webapp/dist/assets/metrics-Dufwtwjq.js +0 -1
- package/webapp/dist/assets/nodes-Brvu8W3j.js +0 -7
- package/webapp/dist/assets/pipelines-t40X4MuC.js +0 -4
- package/webapp/dist/assets/prop-types-CvVtHxp4.js +0 -1
- package/webapp/dist/assets/react-apexcharts.esm-CLO6gala.js +0 -954
- package/webapp/dist/assets/settings-DTAIST3v.js +0 -1
- package/webapp/dist/assets/sign-in-DQEAkJum.js +0 -1
- package/webapp/dist/assets/stacks-BjM1f45N.js +0 -1
- package/webapp/dist/assets/system-info-BQSbSEK-.js +0 -1
- package/webapp/dist/assets/topology-6N8-Az_Z.js +0 -12
- package/webapp/dist/assets/topology-CHpVij2M.css +0 -1
- package/webapp/dist/assets/traces-DVXyb-4m.js +0 -1
- package/webapp/dist/assets/use-stack-context-CbE8fxBA.js +0 -1
- /package/webapp/dist/assets/{fs-uP8eiZAR.js → fs-Ccl3sAcp.js} +0 -0
|
@@ -11,7 +11,13 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Classic mode:
|
|
13
13
|
* - child_process.fork() with manual exit handling (legacy compat)
|
|
14
|
-
*
|
|
14
|
+
*
|
|
15
|
+
* Metrics for both modes come from one `ps -o pid=,rss=,%cpu=` call covering
|
|
16
|
+
* every managed pid — supervisor children and pool workers alike. This line
|
|
17
|
+
* used to say "real CPU/memory metrics via pidusage"; `pidusage` is not a
|
|
18
|
+
* dependency of this package and appears nowhere else in the source. The
|
|
19
|
+
* sentence outlived the code it described, in the one area where a stale
|
|
20
|
+
* claim is hardest to catch: metrics that are wrong still look like metrics.
|
|
15
21
|
*
|
|
16
22
|
* Architecture:
|
|
17
23
|
* pm.supervisor({ children: [{ process: bootstrap-process.js, ... }] })
|
|
@@ -21,17 +27,59 @@
|
|
|
21
27
|
* → Two Netron instances: PM management plane + app HTTP data plane
|
|
22
28
|
*/
|
|
23
29
|
import fs from 'node:fs';
|
|
30
|
+
import { readDeclaredConfig, withDeclaredConfig } from '../project/declared-config.js';
|
|
24
31
|
import path from 'node:path';
|
|
25
32
|
import { fileURLToPath } from 'node:url';
|
|
26
|
-
import { PoolStrategy, SupervisionStrategy,
|
|
33
|
+
import { PoolStrategy, SupervisionStrategy, ProcessLifecycleQueue, LifecyclePreempted, computeBackoff, isAlive, discoverManagedProcesses, } from '@omnitron-dev/titan-pm';
|
|
27
34
|
import { EventEmitter } from '@omnitron-dev/eventemitter';
|
|
28
35
|
import { AppHandle } from './app-handle.js';
|
|
36
|
+
import { parsePsLine, parsePsBatch } from './ps-metrics.js';
|
|
29
37
|
import { launchClassic } from './classic-launcher.js';
|
|
30
38
|
import { resolveStartupOrder, resolveShutdownOrder } from './dependency-resolver.js';
|
|
31
39
|
import { buildRestartPolicy } from '../supervisor/restart-policy.js';
|
|
40
|
+
/**
|
|
41
|
+
* How long to wait for a pool worker to finish exposing its `@Service`
|
|
42
|
+
* classes before giving up on `topology.expose`. A worker reports ready
|
|
43
|
+
* before its Application starts, and exposure happens during start, so the
|
|
44
|
+
* gap is a real one — measured at ~7s for a worker that opens database
|
|
45
|
+
* connections first. The consumer that needs the service starts right after
|
|
46
|
+
* this returns, so the wait is paid once at boot and buys a feature that
|
|
47
|
+
* otherwise stays off until the next restart.
|
|
48
|
+
*/
|
|
49
|
+
/**
|
|
50
|
+
* `childName` is sometimes bare ('http') and sometimes already carries the app
|
|
51
|
+
* ('acme/dev/pricing/http'), depending on which supervisor path raised the
|
|
52
|
+
* event. Prefixing unconditionally produced
|
|
53
|
+
* `acme/dev/pricing/acme/dev/pricing/http` in the log file.
|
|
54
|
+
*/
|
|
55
|
+
function qualifyChildName(appName, childName) {
|
|
56
|
+
return childName.startsWith(`${appName}/`) || childName === appName
|
|
57
|
+
? childName
|
|
58
|
+
: `${appName}/${childName}`;
|
|
59
|
+
}
|
|
60
|
+
const TOPOLOGY_EXPOSE_TIMEOUT_MS = 30_000;
|
|
61
|
+
const TOPOLOGY_EXPOSE_POLL_MS = 250;
|
|
32
62
|
import { ServiceRouter } from './service-router.js';
|
|
33
63
|
import { loadBootstrapConfig, clearCacheFor } from './bootstrap-loader.js';
|
|
64
|
+
/** Levels a pino logger accepts. A child is spawned with nothing else. */
|
|
65
|
+
const PINO_LEVELS = new Set(['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent']);
|
|
66
|
+
/**
|
|
67
|
+
* Process-manager states in which a record no longer describes a live process.
|
|
68
|
+
*
|
|
69
|
+
* Read by `managerOwnedPids`, where the distinction decides whether a pid is
|
|
70
|
+
* protected from the janitor. Written as strings rather than imported from
|
|
71
|
+
* `ProcessStatus` because that export is a const object whose values are
|
|
72
|
+
* these same literals; the set has to hold values, not keys, and a typo here
|
|
73
|
+
* would silently protect a dead pid — so it is pinned by a test.
|
|
74
|
+
*/
|
|
75
|
+
const TERMINAL_PROCESS_STATUSES = new Set(['stopped', 'failed', 'crashed']);
|
|
34
76
|
import { BuildService } from './build-service.js';
|
|
77
|
+
import { execFile } from 'node:child_process';
|
|
78
|
+
import { ProcessJanitor } from './process-janitor.js';
|
|
79
|
+
import { addressInUse, holderCommand, parseHolders, explainConflict } from './port-conflict.js';
|
|
80
|
+
import { collectOwnedPids } from './owned-pids.js';
|
|
81
|
+
import { combineProcessHealth, notRunningHealth, poolHealth } from './app-health.js';
|
|
82
|
+
import { combineProcessTraffic } from './app-traffic.js';
|
|
35
83
|
import { CLI_VERSION } from '../config/defaults.js';
|
|
36
84
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
37
85
|
// Resolve bootstrap-process path — .ts in dev (tsx), .js when compiled
|
|
@@ -55,11 +103,124 @@ function mapStrategy(strategy) {
|
|
|
55
103
|
return SupervisionStrategy.ONE_FOR_ONE;
|
|
56
104
|
}
|
|
57
105
|
}
|
|
106
|
+
/**
|
|
107
|
+
* Strip the `${project}/${stack}/` prefix from a handle key (or
|
|
108
|
+
* leave a bare name untouched). Used to identify "the same physical
|
|
109
|
+
* app under any registration form" so duplicate registrations from
|
|
110
|
+
* stack mode + bare-name RPC paths can be deduplicated.
|
|
111
|
+
*
|
|
112
|
+
* Examples:
|
|
113
|
+
* effectiveAppName('omni/dev/payments') === 'payments'
|
|
114
|
+
* effectiveAppName('payments') === 'payments'
|
|
115
|
+
* effectiveAppName('a/b/c/payments') === 'payments' (last segment)
|
|
116
|
+
*/
|
|
117
|
+
export function effectiveAppName(key) {
|
|
118
|
+
const idx = key.lastIndexOf('/');
|
|
119
|
+
return idx >= 0 ? key.slice(idx + 1) : key;
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Auto-promote a bare `entry.name` to canonical `${project}/${stack}/${name}`
|
|
123
|
+
* form when the env carries OMNITRON_PROJECT + OMNITRON_STACK. The stack-mode
|
|
124
|
+
* boot path (ProjectService.startStack) already prefixes the name explicitly;
|
|
125
|
+
* this helper closes every other entry point that could otherwise register a
|
|
126
|
+
* bare-name handle which then becomes invisible to the stack-prefix-based
|
|
127
|
+
* UI lookup (ProjectService.toStackInfo filters on `${project}/${stack}/`).
|
|
128
|
+
*
|
|
129
|
+
* Idempotent: if the entry already looks namespaced (or env lacks the keys),
|
|
130
|
+
* the input is returned unchanged. An entry qualified under a DIFFERENT
|
|
131
|
+
* project/stack is preserved too — that's operator intent (cross-project
|
|
132
|
+
* administrative run), not a bug.
|
|
133
|
+
*/
|
|
134
|
+
export function ensureNamespacedEntry(entry) {
|
|
135
|
+
const project = entry.env?.['OMNITRON_PROJECT'];
|
|
136
|
+
const stack = entry.env?.['OMNITRON_STACK'];
|
|
137
|
+
if (!project || !stack)
|
|
138
|
+
return entry;
|
|
139
|
+
const canonicalPrefix = `${project}/${stack}/`;
|
|
140
|
+
if (entry.name.startsWith(canonicalPrefix))
|
|
141
|
+
return entry;
|
|
142
|
+
if (entry.name.includes('/'))
|
|
143
|
+
return entry;
|
|
144
|
+
return { ...entry, name: `${canonicalPrefix}${entry.name}` };
|
|
145
|
+
}
|
|
146
|
+
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
147
|
+
/**
|
|
148
|
+
* Startup budget for a child process that declares none of its own.
|
|
149
|
+
*
|
|
150
|
+
* There were TWO of these and they disagreed — `?? 60_000` on the pooled path
|
|
151
|
+
* and `?? 30_000` on the single-process path — and, worse, the pooled one did
|
|
152
|
+
* not consult `entry.startupTimeout` at all. So an operator who set
|
|
153
|
+
* `startupTimeout: 120_000` on an app had it honoured for the app and
|
|
154
|
+
* SILENTLY IGNORED for its child processes, which kept 60 seconds and could
|
|
155
|
+
* not be given more by any setting.
|
|
156
|
+
*
|
|
157
|
+
* Measured on this host: an app whose own successful boot takes 120.6 seconds
|
|
158
|
+
* carried a 120 s budget, and its `deposit-worker` child — a deposit scanner
|
|
159
|
+
* — stayed `stopped` while the app itself came up. Two copies of one decision,
|
|
160
|
+
* one of which discarded the only knob an operator has.
|
|
161
|
+
*
|
|
162
|
+
* 60 s rather than 30 s, for the reason the project's own config already
|
|
163
|
+
* writes down: an over-long ceiling costs a slower failure report, a short one
|
|
164
|
+
* costs the whole stack.
|
|
165
|
+
*/
|
|
166
|
+
const DEFAULT_CHILD_STARTUP_TIMEOUT = 60_000;
|
|
167
|
+
/**
|
|
168
|
+
* Deadline for an RPC the daemon makes to a child it supervises.
|
|
169
|
+
*
|
|
170
|
+
* A call across this connection is WORK — aggregate five minutes of candles,
|
|
171
|
+
* answer a health probe that touches a pool, hand back metrics — not a wire
|
|
172
|
+
* request, and netron's default (5000 ms) is sized for the latter. The pool
|
|
173
|
+
* path stated this number and the single-child path stated nothing, so a
|
|
174
|
+
* client built for a single child fell back to that default: measured, 94
|
|
175
|
+
* timeouts «after 5000ms» in the daemon log, every one on
|
|
176
|
+
* `callExposedService`, which lives on exactly those single children.
|
|
177
|
+
*
|
|
178
|
+
* It belongs here rather than in titan-pm because the orchestrator is what
|
|
179
|
+
* knows the nature of the work; the library's own fallback is 30 s
|
|
180
|
+
* (`process-pool.ts:212`) and applies when a caller states nothing — which,
|
|
181
|
+
* after this, no caller here does.
|
|
182
|
+
*/
|
|
183
|
+
const CHILD_REQUEST_TIMEOUT = 120_000;
|
|
58
184
|
export class OrchestratorService extends EventEmitter {
|
|
59
185
|
logger;
|
|
60
186
|
pm;
|
|
61
187
|
stateStore;
|
|
62
188
|
handles = new Map();
|
|
189
|
+
/**
|
|
190
|
+
* Per-name in-flight start lock (T#57). When two `startApp()` calls
|
|
191
|
+
* arrive concurrently for the same app, the first one's promise is
|
|
192
|
+
* stored here; the second one awaits and returns the same handle
|
|
193
|
+
* rather than racing through to a parallel `launchBootstrapMode` /
|
|
194
|
+
* `launchClassicMode` that would spawn a second worker pool and
|
|
195
|
+
* leak the original. The map is cleared in a `finally` once the
|
|
196
|
+
* launch settles (success or failure) so the next legitimate call
|
|
197
|
+
* for the same name isn't permanently locked out.
|
|
198
|
+
*/
|
|
199
|
+
startingApps = new Map();
|
|
200
|
+
/**
|
|
201
|
+
* Per-app async serial-execution queue covering start / stop / restart
|
|
202
|
+
* / crash-restart-timer (P0-E + P0-I + P2-B in the audit). Pre-this-
|
|
203
|
+
* queue each path had its own coalescer:
|
|
204
|
+
* - startingApps: same-name start coalescing only — `stopApp` did
|
|
205
|
+
* NOT consult it, so stop racing in-flight start left an orphan
|
|
206
|
+
* supervisor when stop arrived before start wrote childProcess.
|
|
207
|
+
* - restartCoalescer: same-name restart coalescing; independent of
|
|
208
|
+
* stop+start.
|
|
209
|
+
* - handleClassicCrash setTimeout: a raw timer that could fire
|
|
210
|
+
* during the `await stopChild(...)` window and spawn a zombie
|
|
211
|
+
* against an app the operator just stopped.
|
|
212
|
+
* The queue collapses all four into one primitive ordered by
|
|
213
|
+
* priority (stop > restart > start > crash-restart). The same-key
|
|
214
|
+
* mutex means stop ALWAYS observes a completed start, and an
|
|
215
|
+
* explicit stop ALWAYS preempts a pending crash-restart timer.
|
|
216
|
+
*/
|
|
217
|
+
lifecycleQueue = new ProcessLifecycleQueue();
|
|
218
|
+
/**
|
|
219
|
+
* Pending post-escalation cool-down timers, keyed by app name.
|
|
220
|
+
* Held here rather than inside a lifecycle-queue slot so that a
|
|
221
|
+
* crashed app still answers operator commands while it waits.
|
|
222
|
+
*/
|
|
223
|
+
coolDownTimers = new Map();
|
|
63
224
|
config = null;
|
|
64
225
|
cwd;
|
|
65
226
|
metricsTimer = null;
|
|
@@ -70,16 +231,84 @@ export class OrchestratorService extends EventEmitter {
|
|
|
70
231
|
buildService = null;
|
|
71
232
|
/** Cached build results per app name */
|
|
72
233
|
buildResults = new Map();
|
|
234
|
+
/**
|
|
235
|
+
* Per-app restart coalescer state.
|
|
236
|
+
*
|
|
237
|
+
* `inFlight` — the currently-running restart promise (so a concurrent
|
|
238
|
+
* trigger doesn't kick off a parallel restart and tear down the half-
|
|
239
|
+
* started state from the previous one).
|
|
240
|
+
* `triggerSerial` — monotonically incremented by external trigger
|
|
241
|
+
* entry points (BuildService onRebuild, RPC restartApp, manual
|
|
242
|
+
* `omnitron restart`, deploy.service). Internal coalescer arms
|
|
243
|
+
* that queue themselves DO NOT bump it.
|
|
244
|
+
* `firedSerial` — value of `triggerSerial` that the most recently
|
|
245
|
+
* completed restart consumed. A trailing coalesced restart only
|
|
246
|
+
* fires when `triggerSerial > firedSerial`, i.e. a genuinely new
|
|
247
|
+
* external trigger arrived during the in-flight window. Two
|
|
248
|
+
* coalescer arms colliding can no longer cause a self-perpetuating
|
|
249
|
+
* loop because neither bumps `triggerSerial`.
|
|
250
|
+
* `cooldownUntil` — absolute epoch ms timestamp. Any trigger received
|
|
251
|
+
* before this clears is queued instead of fired immediately.
|
|
252
|
+
* Prevents the storm where esbuild's onEnd fires for a freshly-
|
|
253
|
+
* spawned context milliseconds after the app reports online.
|
|
254
|
+
*/
|
|
255
|
+
restartCoalescer = new Map();
|
|
256
|
+
/**
|
|
257
|
+
* Wall-clock window in ms during which restart triggers from
|
|
258
|
+
* BuildService.watchApp are deferred after `app:online`. Sized so a
|
|
259
|
+
* normal bundle build + child spawn + DI bootstrap fits comfortably
|
|
260
|
+
* inside; anything legitimately changing more often than this is a
|
|
261
|
+
* developer typing very fast, which the regular debounce handles.
|
|
262
|
+
*/
|
|
263
|
+
static POST_ONLINE_COOLDOWN_MS = 3_000;
|
|
264
|
+
/**
|
|
265
|
+
* Orphan-process reaper. Lazily created on first reconcile so unit
|
|
266
|
+
* tests that hit `startApp` directly don't get a periodic timer
|
|
267
|
+
* until the first real reconcile runs (which they bypass by
|
|
268
|
+
* stubbing `bootReconciled = true`).
|
|
269
|
+
*/
|
|
270
|
+
janitor = null;
|
|
271
|
+
/**
|
|
272
|
+
* One-shot guard for cold-start reconciliation. T#63 — every code
|
|
273
|
+
* path that lands an app on this orchestrator goes through
|
|
274
|
+
* `ensureBootReconciled()`, regardless of whether the entry-point
|
|
275
|
+
* is `startAll` (daemon-config) or `startApp` (project/stack
|
|
276
|
+
* mode). Pre-T#63 the cold-start sweep only fired inside
|
|
277
|
+
* `startAll`, so project-mode boots silently inherited orphans
|
|
278
|
+
* from the previous daemon generation.
|
|
279
|
+
*/
|
|
280
|
+
bootReconciled = false;
|
|
281
|
+
bootReconcilePromise = null;
|
|
73
282
|
/** Daemon's Netron instance for native service routing */
|
|
74
283
|
daemonNetron = null;
|
|
75
284
|
/** Daemon Unix socket path for child-to-daemon Netron connections */
|
|
76
285
|
daemonSocketPath = null;
|
|
286
|
+
/**
|
|
287
|
+
* Promise that resolves once the daemon's Netron is wired up. Apps that
|
|
288
|
+
* declare topology cannot launch until this is ready — instead of failing
|
|
289
|
+
* with "daemon Netron not available" the launcher awaits this promise
|
|
290
|
+
* (with a generous timeout) so transient startup-order interleavings
|
|
291
|
+
* resolve themselves instead of becoming user-visible errors.
|
|
292
|
+
*/
|
|
293
|
+
daemonNetronReady;
|
|
294
|
+
resolveDaemonNetronReady = () => undefined;
|
|
295
|
+
/**
|
|
296
|
+
* Optional metrics sink. Set by the daemon at boot via
|
|
297
|
+
* `setMetricsBridge` so the orchestrator can forward janitor sweeps,
|
|
298
|
+
* esbuild rebuilds, and lifecycle events without coupling itself to
|
|
299
|
+
* the metrics module. Null means metrics are disabled (tests, slim
|
|
300
|
+
* bootstraps).
|
|
301
|
+
*/
|
|
302
|
+
metricsBridge = null;
|
|
77
303
|
constructor(logger, pm, stateStore, cwd) {
|
|
78
304
|
super();
|
|
79
305
|
this.logger = logger;
|
|
80
306
|
this.pm = pm;
|
|
81
307
|
this.stateStore = stateStore;
|
|
82
308
|
this.cwd = cwd ?? process.cwd();
|
|
309
|
+
this.daemonNetronReady = new Promise((resolve) => {
|
|
310
|
+
this.resolveDaemonNetronReady = resolve;
|
|
311
|
+
});
|
|
83
312
|
}
|
|
84
313
|
/**
|
|
85
314
|
* Set the daemon Netron instance and socket path for native topology routing.
|
|
@@ -88,28 +317,356 @@ export class OrchestratorService extends EventEmitter {
|
|
|
88
317
|
setDaemonNetron(netron, socketPath) {
|
|
89
318
|
this.daemonNetron = netron;
|
|
90
319
|
this.daemonSocketPath = socketPath;
|
|
320
|
+
this.resolveDaemonNetronReady();
|
|
91
321
|
this.logger.info({ socketPath }, 'Daemon Netron set for native topology routing');
|
|
92
322
|
}
|
|
323
|
+
/**
|
|
324
|
+
* Wire the metrics bridge. Called once during daemon bootstrap.
|
|
325
|
+
* After this, janitor sweeps, esbuild rebuilds, and orchestrator
|
|
326
|
+
* lifecycle events are forwarded to Prometheus.
|
|
327
|
+
*/
|
|
328
|
+
setMetricsBridge(bridge) {
|
|
329
|
+
this.metricsBridge = bridge;
|
|
330
|
+
bridge.attachOrchestrator(this);
|
|
331
|
+
this.wireOrchestratorTelemetry();
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Subscribe to our own EventEmitter to forward state-change events to
|
|
335
|
+
* the metrics bridge. Idempotent so repeated `setMetricsBridge` calls
|
|
336
|
+
* (mostly tests) don't multiply listeners.
|
|
337
|
+
*/
|
|
338
|
+
telemetryWired = false;
|
|
339
|
+
wireOrchestratorTelemetry() {
|
|
340
|
+
if (this.telemetryWired)
|
|
341
|
+
return;
|
|
342
|
+
this.telemetryWired = true;
|
|
343
|
+
this.on('app:restart', (app, attempt) => {
|
|
344
|
+
this.metricsBridge?.recordAppRestart(app, attempt, 'crash');
|
|
345
|
+
});
|
|
346
|
+
this.on('app:crash', (app) => {
|
|
347
|
+
this.metricsBridge?.recordAppCrash(app);
|
|
348
|
+
});
|
|
349
|
+
this.on('app:escalated', (app) => {
|
|
350
|
+
this.metricsBridge?.recordAppEscalated(app);
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* Wait until the daemon Netron is available, with a hard ceiling so we
|
|
355
|
+
* never hang forever if `setDaemonNetron()` was never wired in.
|
|
356
|
+
*/
|
|
357
|
+
async ensureDaemonNetronReady(timeoutMs = 30_000) {
|
|
358
|
+
if (this.daemonNetron && this.daemonSocketPath)
|
|
359
|
+
return;
|
|
360
|
+
const timeout = new Promise((_resolve, reject) => {
|
|
361
|
+
const timer = setTimeout(() => reject(new Error(`Daemon Netron not initialized within ${timeoutMs}ms — wiring bug?`)), timeoutMs);
|
|
362
|
+
timer.unref?.();
|
|
363
|
+
});
|
|
364
|
+
await Promise.race([this.daemonNetronReady, timeout]);
|
|
365
|
+
}
|
|
93
366
|
// ============================================================================
|
|
94
367
|
// Lifecycle
|
|
95
368
|
// ============================================================================
|
|
369
|
+
/**
|
|
370
|
+
* Refresh the orchestrator's view of the ecosystem config. Called
|
|
371
|
+
* from `daemon.reloadConfig` so a subsequent `restartApp` reads the
|
|
372
|
+
* new entry definition (mode, env, dependsOn, restart policy)
|
|
373
|
+
* instead of the stale copy captured at first `startAll`. Pre-fix
|
|
374
|
+
* the two refs diverged silently after every reload — operators
|
|
375
|
+
* editing ecosystem.config.ts and running `omnitron reload`
|
|
376
|
+
* believed the daemon picked up the changes; in fact only daemon's
|
|
377
|
+
* own RPC service did, and individual app restarts kept running
|
|
378
|
+
* against the pre-reload definition until the daemon itself
|
|
379
|
+
* restarted. P0-G in the audit.
|
|
380
|
+
*/
|
|
381
|
+
setConfig(config) {
|
|
382
|
+
this.config = config;
|
|
383
|
+
// Metrics polling is armed here rather than in `startAll` alone.
|
|
384
|
+
//
|
|
385
|
+
// It used to run only from `startAll`, which is the `omnitron start`
|
|
386
|
+
// path. Apps brought up any other way — `stack start`, a single
|
|
387
|
+
// `omnitron start <app>`, or the daemon resuming a stack at boot — were
|
|
388
|
+
// never sampled, so `omnitron list`, `status` and the console's app
|
|
389
|
+
// cards reported cpu 0 / memory 0 for processes that were serving
|
|
390
|
+
// traffic. Observed live: six apps online, every one showing 0 while
|
|
391
|
+
// `ps` reported ~105 MB RSS apiece.
|
|
392
|
+
//
|
|
393
|
+
// `setConfig` is the one point every path passes through, and the call
|
|
394
|
+
// is idempotent — an existing timer is cleared first.
|
|
395
|
+
this.ensureMetricsPolling();
|
|
396
|
+
}
|
|
397
|
+
/**
|
|
398
|
+
* Start metrics sampling if it is not already running.
|
|
399
|
+
*
|
|
400
|
+
* Called from every path that can produce a managed process, so no app can
|
|
401
|
+
* end up unsampled because of how it was started.
|
|
402
|
+
*/
|
|
403
|
+
ensureMetricsPolling() {
|
|
404
|
+
if (this.metricsTimer)
|
|
405
|
+
return;
|
|
406
|
+
this.startMetricsPolling(this.config?.monitoring?.metrics?.interval ?? 5_000);
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* One-shot cold-start reconciliation: discover pre-existing managed
|
|
410
|
+
* processes, reap fork-workers left over from a previous daemon
|
|
411
|
+
* generation, and arm the periodic janitor sweep. Idempotent —
|
|
412
|
+
* concurrent callers join the in-flight promise; subsequent calls
|
|
413
|
+
* after success are no-ops.
|
|
414
|
+
*
|
|
415
|
+
* Called from BOTH `startAll` (top-level daemon-config boot) and
|
|
416
|
+
* `startApp` (project/stack mode boot), so every code path that
|
|
417
|
+
* lands an app on this orchestrator runs the cleanup. Pre-T#63 the
|
|
418
|
+
* cold-start sweep was gated on `startAll` and never ran in
|
|
419
|
+
* project mode, so an orphan from a previous daemon survived
|
|
420
|
+
* silently and held the listen port hostage from its successor.
|
|
421
|
+
*/
|
|
422
|
+
async ensureBootReconciled(config) {
|
|
423
|
+
if (this.bootReconciled)
|
|
424
|
+
return;
|
|
425
|
+
if (this.bootReconcilePromise) {
|
|
426
|
+
await this.bootReconcilePromise;
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
this.bootReconcilePromise = (async () => {
|
|
430
|
+
// Discovery: surface OMNITRON_MANAGED-tagged processes that match
|
|
431
|
+
// a configured app so the operator can see what's about to be
|
|
432
|
+
// reaped (or — once the ChildContract handshake lands — adopted).
|
|
433
|
+
// P2-A in the audit.
|
|
434
|
+
try {
|
|
435
|
+
const discovered = discoverManagedProcesses();
|
|
436
|
+
const adopted = [];
|
|
437
|
+
const knownApps = new Set(config?.apps.map((a) => a.name) ?? []);
|
|
438
|
+
for (const proc of discovered) {
|
|
439
|
+
if (proc.ppid === process.pid)
|
|
440
|
+
continue; // our own freshly-spawned child
|
|
441
|
+
const fqn = proc.fullyQualifiedName;
|
|
442
|
+
const bare = proc.appName;
|
|
443
|
+
if (knownApps.has(fqn) || knownApps.has(bare)) {
|
|
444
|
+
adopted.push(`${fqn}#${proc.pid}`);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
if (adopted.length > 0) {
|
|
448
|
+
this.logger.warn({ discovered: adopted.length, apps: adopted }, 'Found pre-existing managed processes — they will be reaped by the janitor in 60s unless adopted');
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
catch (err) {
|
|
452
|
+
this.logger.debug({ err: err.message }, 'Process discovery failed — proceeding without reconcile');
|
|
453
|
+
}
|
|
454
|
+
// Reap any fork-workers left over from a previous daemon BEFORE
|
|
455
|
+
// launching apps. Without this, a hard-killed daemon's children
|
|
456
|
+
// are still alive (now reparented to init); they hold DB
|
|
457
|
+
// connections and listen ports that the new apps will fight
|
|
458
|
+
// for. The reap is bounded by `gracefulMs` (default 3s).
|
|
459
|
+
await this.ensureJanitor().coldStartSweep();
|
|
460
|
+
// Arm the periodic sweep so runtime leaks from partially-failed
|
|
461
|
+
// restart cycles (esbuild-rebuild → spawn-fail → orphan) get
|
|
462
|
+
// collected on every interval.
|
|
463
|
+
this.ensureJanitor().start();
|
|
464
|
+
})();
|
|
465
|
+
try {
|
|
466
|
+
await this.bootReconcilePromise;
|
|
467
|
+
this.bootReconciled = true;
|
|
468
|
+
}
|
|
469
|
+
finally {
|
|
470
|
+
this.bootReconcilePromise = null;
|
|
471
|
+
}
|
|
472
|
+
}
|
|
96
473
|
async startAll(config) {
|
|
97
474
|
this.config = config;
|
|
475
|
+
// Cold-start orphan reap + periodic janitor arming. Idempotent —
|
|
476
|
+
// safe to call on every boot path. Project-mode flows that call
|
|
477
|
+
// `startApp` directly trigger the same one-shot reconcile, so
|
|
478
|
+
// every boot path lands in the same post-condition. (T#63)
|
|
479
|
+
await this.ensureBootReconciled(config);
|
|
480
|
+
// Dependency-aware skip: if app A fails to start, every app that
|
|
481
|
+
// declares A in `dependsOn` (transitively) is skipped instead of
|
|
482
|
+
// attempted. Without this, a downstream app would launch, fail to
|
|
483
|
+
// reach its missing dependency, crash, and chew through its restart
|
|
484
|
+
// budget — masking the real root cause.
|
|
485
|
+
const failed = new Set();
|
|
486
|
+
const blocked = new Set();
|
|
98
487
|
for (const batch of resolveStartupOrder(config.apps)) {
|
|
99
488
|
const enabled = batch.filter((e) => e.enabled !== false);
|
|
100
489
|
await Promise.all(enabled.map(async (e) => {
|
|
490
|
+
const blockingDep = (e.dependsOn ?? []).find((dep) => failed.has(dep) || blocked.has(dep));
|
|
491
|
+
if (blockingDep) {
|
|
492
|
+
blocked.add(e.name);
|
|
493
|
+
this.logger.warn({ app: e.name, blockedBy: blockingDep }, 'Skipping app startup — depends on a failed/blocked app');
|
|
494
|
+
return;
|
|
495
|
+
}
|
|
101
496
|
try {
|
|
102
497
|
await this.startApp(e, config);
|
|
103
498
|
}
|
|
104
499
|
catch (err) {
|
|
500
|
+
failed.add(e.name);
|
|
105
501
|
this.logger.error({ app: e.name, error: err.message }, 'Failed to start app — continuing with remaining apps');
|
|
106
502
|
}
|
|
107
503
|
}));
|
|
108
504
|
}
|
|
109
|
-
|
|
505
|
+
if (failed.size > 0 || blocked.size > 0) {
|
|
506
|
+
this.logger.warn({ failed: [...failed], blocked: [...blocked] }, 'Startup completed with failures — some apps did not launch');
|
|
507
|
+
}
|
|
508
|
+
// Periodic sweep was armed by `ensureBootReconciled` above, and metrics
|
|
509
|
+
// polling by `setConfig` — no second call needed for either.
|
|
110
510
|
this.persistState();
|
|
111
511
|
}
|
|
512
|
+
/**
|
|
513
|
+
* Lazily create the orphan-process janitor with the live "owned
|
|
514
|
+
* pids" snapshot wired to the orchestrator's handle map.
|
|
515
|
+
*/
|
|
516
|
+
ensureJanitor() {
|
|
517
|
+
if (this.janitor)
|
|
518
|
+
return this.janitor;
|
|
519
|
+
this.janitor = new ProcessJanitor({
|
|
520
|
+
logger: this.logger,
|
|
521
|
+
getOwnedPids: () => this.collectOwnedPids(),
|
|
522
|
+
onMetrics: (m) => this.metricsBridge?.recordJanitorSweep(m),
|
|
523
|
+
});
|
|
524
|
+
return this.janitor;
|
|
525
|
+
}
|
|
526
|
+
/**
|
|
527
|
+
* Every OS pid the orchestrator currently considers ours. Cheap — called
|
|
528
|
+
* on every janitor sweep, and the janitor kills whatever this set omits.
|
|
529
|
+
*
|
|
530
|
+
* The rule itself lives in `owned-pids.ts`, where it can be tested: the
|
|
531
|
+
* janitor's own tests supply this set as a fixture, so they could never
|
|
532
|
+
* catch it being incomplete — and it was, by a whole class of process.
|
|
533
|
+
*/
|
|
534
|
+
/**
|
|
535
|
+
* Who is holding the port an app could not bind, in one sentence.
|
|
536
|
+
*
|
|
537
|
+
* Called where a start or restart failed, because that is the moment the
|
|
538
|
+
* answer is worth having and the moment nobody has it: the error names the
|
|
539
|
+
* address and never the holder. Measured on this daemon — four
|
|
540
|
+
* `EADDRINUSE` failures on 3005 in thirty-one seconds, `omnitron list`
|
|
541
|
+
* reporting the app crashed with no pid, and the port held the whole time
|
|
542
|
+
* by a child of this very daemon that nothing in its records claimed.
|
|
543
|
+
*
|
|
544
|
+
* Returns null when the failure was not a port clash, and says so out loud
|
|
545
|
+
* when it could not read the machine — a reading that failed and a port
|
|
546
|
+
* nobody holds are different findings.
|
|
547
|
+
*/
|
|
548
|
+
async explainPortConflict(message) {
|
|
549
|
+
const address = addressInUse(message);
|
|
550
|
+
if (!address)
|
|
551
|
+
return null;
|
|
552
|
+
const { file, args } = holderCommand(address.port, process.platform);
|
|
553
|
+
let output;
|
|
554
|
+
try {
|
|
555
|
+
output = await new Promise((resolve, reject) => {
|
|
556
|
+
execFile(file, args, { encoding: 'utf8', timeout: 5_000 }, (err, stdout) => {
|
|
557
|
+
// `ss` and `lsof` both exit non-zero when nothing matches, which is
|
|
558
|
+
// an answer rather than a failure; the empty stdout carries it.
|
|
559
|
+
if (err && !stdout)
|
|
560
|
+
reject(err);
|
|
561
|
+
else
|
|
562
|
+
resolve(stdout);
|
|
563
|
+
});
|
|
564
|
+
});
|
|
565
|
+
}
|
|
566
|
+
catch (err) {
|
|
567
|
+
return `could not read who holds port ${address.port}: ${file} ${err.message}`;
|
|
568
|
+
}
|
|
569
|
+
const holders = parseHolders(output, process.platform);
|
|
570
|
+
// One `ps` for the parents, before deciding: the decision is pure and
|
|
571
|
+
// takes the answers, so that it can be tested without a machine.
|
|
572
|
+
const parents = new Map();
|
|
573
|
+
for (const holder of holders) {
|
|
574
|
+
const ppid = await this.parentOf(holder.pid);
|
|
575
|
+
if (ppid !== undefined)
|
|
576
|
+
parents.set(holder.pid, ppid);
|
|
577
|
+
}
|
|
578
|
+
const verdict = explainConflict({
|
|
579
|
+
port: address.port,
|
|
580
|
+
holders,
|
|
581
|
+
parentOf: (pid) => parents.get(pid),
|
|
582
|
+
daemonPid: process.pid,
|
|
583
|
+
owned: this.collectOwnedPids(),
|
|
584
|
+
});
|
|
585
|
+
return verdict.because;
|
|
586
|
+
}
|
|
587
|
+
/** The parent of one process, or undefined when it cannot be read. */
|
|
588
|
+
async parentOf(pid) {
|
|
589
|
+
try {
|
|
590
|
+
const out = await new Promise((resolve, reject) => {
|
|
591
|
+
execFile('ps', ['-o', 'ppid=', '-p', String(pid)], { encoding: 'utf8', timeout: 5_000 }, (err, stdout) => {
|
|
592
|
+
if (err && !stdout)
|
|
593
|
+
reject(err);
|
|
594
|
+
else
|
|
595
|
+
resolve(stdout);
|
|
596
|
+
});
|
|
597
|
+
});
|
|
598
|
+
const parsed = Number(out.trim());
|
|
599
|
+
return Number.isInteger(parsed) ? parsed : undefined;
|
|
600
|
+
}
|
|
601
|
+
catch {
|
|
602
|
+
// A process that exited between the two readings has no parent to
|
|
603
|
+
// report, and that is not an error worth a line of its own — the
|
|
604
|
+
// verdict already says the holder was gone.
|
|
605
|
+
return undefined;
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
collectOwnedPids() {
|
|
609
|
+
return collectOwnedPids(this.handles.values(), (workerId) => this.pm.getWorkerHandle(workerId)?.pid, this.managerOwnedPids());
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* Every pid the process manager currently vouches for.
|
|
613
|
+
*
|
|
614
|
+
* Two windows, and a process is only safe if both are asked. `getForkedPids`
|
|
615
|
+
* holds a claim from `fork()` until `spawn()` resolves and releases it —
|
|
616
|
+
* after that its own docstring says "the WorkerHandle registry is the
|
|
617
|
+
* authority". `listProcesses()` IS that registry, and it is the only source
|
|
618
|
+
* that answers for a process the orchestrator did not start: the daemon's
|
|
619
|
+
* system workers go through this same manager but belong to no app handle,
|
|
620
|
+
* so walking `this.handles` can never see them.
|
|
621
|
+
*
|
|
622
|
+
* Terminal records are excluded deliberately. They keep the pid they had,
|
|
623
|
+
* the OS reuses pids, and a stale one here would vouch for exactly the
|
|
624
|
+
* leaked worker the janitor exists to reap.
|
|
625
|
+
*/
|
|
626
|
+
managerOwnedPids() {
|
|
627
|
+
const pids = [...(this.pm.getForkedPids?.() ?? [])];
|
|
628
|
+
for (const info of this.pm.listProcesses?.() ?? []) {
|
|
629
|
+
if (typeof info.pid !== 'number')
|
|
630
|
+
continue;
|
|
631
|
+
if (TERMINAL_PROCESS_STATUSES.has(info.status))
|
|
632
|
+
continue;
|
|
633
|
+
pids.push(info.pid);
|
|
634
|
+
}
|
|
635
|
+
return pids;
|
|
636
|
+
}
|
|
112
637
|
async startApp(entry, config) {
|
|
638
|
+
// T#63: run the one-shot cold-start reconcile (orphan reap +
|
|
639
|
+
// janitor arming) before ANY app spawns. Project-mode boots flow
|
|
640
|
+
// through `projectService.startStack → orchestrator.startApp`
|
|
641
|
+
// and historically bypassed the reap entirely — a previous
|
|
642
|
+
// daemon's children (now reparented to init) kept their HTTP
|
|
643
|
+
// listen ports and the fresh child failed to bind silently.
|
|
644
|
+
// No-op after first call thanks to `bootReconciled`.
|
|
645
|
+
await this.ensureBootReconciled(config);
|
|
646
|
+
// Defensive auto-namespacing — promotes a bare entry.name to the
|
|
647
|
+
// canonical `${project}/${stack}/${name}` form when the env carries
|
|
648
|
+
// OMNITRON_PROJECT + OMNITRON_STACK. Catches every code path that
|
|
649
|
+
// bypasses ProjectService.startStack:
|
|
650
|
+
// - direct CLI `omnitron start <name>` (no stack context)
|
|
651
|
+
// - legacy ecosystem configs without per-app namespace
|
|
652
|
+
// - daemon startAll() flows over a not-yet-namespaced config
|
|
653
|
+
entry = ensureNamespacedEntry(entry);
|
|
654
|
+
const inflight = this.startingApps.get(entry.name);
|
|
655
|
+
if (inflight) {
|
|
656
|
+
this.logger.debug({ app: entry.name }, 'Coalescing concurrent startApp into in-flight launch (T#57)');
|
|
657
|
+
return inflight;
|
|
658
|
+
}
|
|
659
|
+
// Serialise against stop/restart for the same app via the
|
|
660
|
+
// lifecycle queue. A stopApp arriving while this start is queued
|
|
661
|
+
// (or in-flight) now waits its turn instead of racing through
|
|
662
|
+
// and leaving an orphan supervisor.
|
|
663
|
+
const promise = this.lifecycleQueue.enqueue(entry.name, 'start', () => this.startAppInternal(entry, config)).finally(() => {
|
|
664
|
+
this.startingApps.delete(entry.name);
|
|
665
|
+
});
|
|
666
|
+
this.startingApps.set(entry.name, promise);
|
|
667
|
+
return promise;
|
|
668
|
+
}
|
|
669
|
+
async startAppInternal(entry, config) {
|
|
113
670
|
// Persist ecosystem config for restartApp — stack mode passes config per-app,
|
|
114
671
|
// but restartApp calls startApp without it.
|
|
115
672
|
if (config && !this.config) {
|
|
@@ -118,14 +675,106 @@ export class OrchestratorService extends EventEmitter {
|
|
|
118
675
|
const cfg = config ?? this.config;
|
|
119
676
|
if (!cfg)
|
|
120
677
|
throw new Error('Ecosystem config not loaded');
|
|
678
|
+
// Deduplicate by effective app identity. Project-mode registers
|
|
679
|
+
// apps as `${project}/${stack}/${app}`; non-project paths use the
|
|
680
|
+
// bare `${app}` short name. They refer to the SAME physical app
|
|
681
|
+
// and must never coexist as separate handles — the duplicate
|
|
682
|
+
// would compete for the port and the orphan-reaper's owned-pid
|
|
683
|
+
// accounting would split between them.
|
|
684
|
+
//
|
|
685
|
+
// Policy:
|
|
686
|
+
// - exact key already exists, online → return it (idempotent)
|
|
687
|
+
// - exact key already exists, not online → fall through and re-launch
|
|
688
|
+
// under the same key
|
|
689
|
+
// - DIFFERENT key with the same effective short name:
|
|
690
|
+
// - new is MORE specific (namespaced) and existing is bare → stop
|
|
691
|
+
// the bare one and re-launch under the namespaced key. Without
|
|
692
|
+
// this, a boot-race where bare-name auto-start beats the
|
|
693
|
+
// project/stack flow strands the app under the wrong key and
|
|
694
|
+
// webapp/CLI lookups by namespaced key see it as stopped.
|
|
695
|
+
// - new is LESS-or-equally specific → refuse: operator should
|
|
696
|
+
// restart by canonical name, not register an alias.
|
|
697
|
+
const effective = effectiveAppName(entry.name);
|
|
698
|
+
const newIsNamespaced = entry.name.includes('/');
|
|
699
|
+
for (const [key, h] of this.handles) {
|
|
700
|
+
if (key === entry.name)
|
|
701
|
+
continue;
|
|
702
|
+
if (effectiveAppName(key) !== effective)
|
|
703
|
+
continue;
|
|
704
|
+
const existingIsNamespaced = key.includes('/');
|
|
705
|
+
// Same effective app under a different key.
|
|
706
|
+
if (h.status === 'online') {
|
|
707
|
+
if (newIsNamespaced && !existingIsNamespaced) {
|
|
708
|
+
this.logger.info({ stale: key, replacing_with: entry.name, effective }, 'Stopping bare-name handle to re-register under namespaced key');
|
|
709
|
+
try {
|
|
710
|
+
await this.stopApp(key);
|
|
711
|
+
}
|
|
712
|
+
catch (err) {
|
|
713
|
+
this.logger.warn({ stale: key, error: err.message }, 'stopApp during dedup failed — clearing handle anyway');
|
|
714
|
+
}
|
|
715
|
+
this.handles.delete(key);
|
|
716
|
+
if (key !== entry.name)
|
|
717
|
+
this.metricsBridge?.evictApp(key);
|
|
718
|
+
continue;
|
|
719
|
+
}
|
|
720
|
+
this.logger.warn({ existing: key, attempted: entry.name, effective }, 'App already running under a canonical name — refusing duplicate registration');
|
|
721
|
+
return h;
|
|
722
|
+
}
|
|
723
|
+
// Existing handle is errored/stopped — replace silently so a
|
|
724
|
+
// failed previous launch doesn't permanently block the canonical
|
|
725
|
+
// form.
|
|
726
|
+
this.logger.info({ stale: key, replacing_with: entry.name, effective }, 'Replacing stale duplicate handle');
|
|
727
|
+
// The handle is errored or stopped, so `stopApp` never ran for it and
|
|
728
|
+
// its services are still registered on the daemon's Netron. Dropping the
|
|
729
|
+
// handle here is the last moment anything knows they exist — or that its
|
|
730
|
+
// processes still run: an errored app's other children and its pools.
|
|
731
|
+
await this.releaseExposedServices(h);
|
|
732
|
+
await this.tearDownProcesses(h);
|
|
733
|
+
this.handles.delete(key);
|
|
734
|
+
// Drop any metrics that were collected under the stale alias so it
|
|
735
|
+
// doesn't haunt future snapshots as a ghost offline app.
|
|
736
|
+
if (key !== entry.name)
|
|
737
|
+
this.metricsBridge?.evictApp(key);
|
|
738
|
+
}
|
|
121
739
|
const existing = this.handles.get(entry.name);
|
|
122
740
|
if (existing && existing.status === 'online') {
|
|
123
741
|
this.logger.warn({ app: entry.name }, 'App already running');
|
|
124
742
|
return existing;
|
|
125
743
|
}
|
|
744
|
+
// A handle that is not online may still run processes. A crash restart and
|
|
745
|
+
// a cool-down recovery only flip its status before coming here; one child
|
|
746
|
+
// crashing leaves its siblings — the server among them — up; and its pools
|
|
747
|
+
// were never any supervisor's to stop. Stopped before the new handle
|
|
748
|
+
// starts, or the new one finds its own predecessor on its port: on
|
|
749
|
+
// daos/test (2026-09-25) paysys's crash restarts failed five times on
|
|
750
|
+
// `EADDRINUSE :3004`, «held by … a child this daemon still accounts for».
|
|
751
|
+
if (existing) {
|
|
752
|
+
await this.releaseExposedServices(existing);
|
|
753
|
+
await this.tearDownProcesses(existing);
|
|
754
|
+
}
|
|
126
755
|
const mode = entry.bootstrap ? 'bootstrap' : 'classic';
|
|
127
756
|
const handle = new AppHandle(entry, mode);
|
|
757
|
+
// The restart budget belongs to the APP, not to this handle instance.
|
|
758
|
+
// Every start replaces the handle, so a counter that starts at zero here
|
|
759
|
+
// can never reach `maxRestarts` — which made the budget, the escalation
|
|
760
|
+
// and the cool-down unreachable even once the restart itself worked. The
|
|
761
|
+
// window is enforced by the policy, not by forgetting.
|
|
762
|
+
if (existing)
|
|
763
|
+
handle.restarts = existing.restarts;
|
|
128
764
|
this.handles.set(entry.name, handle);
|
|
765
|
+
// Arm metrics sampling here — the one place an app enters the registry.
|
|
766
|
+
//
|
|
767
|
+
// It used to be armed only from `startAll` (the `omnitron start` path).
|
|
768
|
+
// Anything else — `stack start`, a single `omnitron start <app>`, the
|
|
769
|
+
// daemon resuming a stack at boot — was never sampled, so `list`,
|
|
770
|
+
// `status` and the console's app cards reported cpu 0 / memory 0 for
|
|
771
|
+
// processes serving traffic. Observed: six apps online, every one at
|
|
772
|
+
// zero, while `ps` showed ~105 MB RSS apiece.
|
|
773
|
+
//
|
|
774
|
+
// `setConfig` looked like the right seam but is only called on config
|
|
775
|
+
// RELOAD, so it never fired on a normal start. This is the seam that
|
|
776
|
+
// actually covers every path. Idempotent: re-arming replaces the timer.
|
|
777
|
+
this.ensureMetricsPolling();
|
|
129
778
|
this.logger.info({ app: entry.name, mode }, 'Starting app');
|
|
130
779
|
try {
|
|
131
780
|
if (mode === 'classic') {
|
|
@@ -134,42 +783,113 @@ export class OrchestratorService extends EventEmitter {
|
|
|
134
783
|
else {
|
|
135
784
|
await this.launchBootstrapMode(entry, handle, cfg);
|
|
136
785
|
}
|
|
137
|
-
|
|
786
|
+
// «App started» only for an app that is. It was logged in the same
|
|
787
|
+
// millisecond as a child's «Process failed to start», for an app whose
|
|
788
|
+
// processes never all came up.
|
|
789
|
+
if (handle.status === 'online') {
|
|
790
|
+
this.logger.info({ app: entry.name, pid: handle.pid, mode }, 'App started');
|
|
791
|
+
}
|
|
792
|
+
else {
|
|
793
|
+
this.logger.error({ app: entry.name, status: handle.status, mode }, 'App did not come up — not every declared process started');
|
|
794
|
+
}
|
|
138
795
|
this.persistState();
|
|
139
796
|
return handle;
|
|
140
797
|
}
|
|
141
798
|
catch (err) {
|
|
799
|
+
// `errored` rather than `crashed` is the right REPORT — the app never
|
|
800
|
+
// ran. It was also, until now, the end of the story: `'errored'` is read
|
|
801
|
+
// by no code in this repository outside formatters, `status` and a
|
|
802
|
+
// metrics label, so a start that failed was never attempted again. One
|
|
803
|
+
// try, then errored until a human noticed.
|
|
804
|
+
//
|
|
805
|
+
// That asymmetry is backwards. A failure to start is MORE likely to be
|
|
806
|
+
// transient than a crash after a successful boot — a dependency still
|
|
807
|
+
// coming up, a disk still cold, a machine briefly loaded. Six bootstrap
|
|
808
|
+
// apps here timed out at `config:loading` under a load average of 149
|
|
809
|
+
// and stayed down after the load passed, because nothing asked again.
|
|
810
|
+
//
|
|
811
|
+
// The counter is bumped by hand because `markErrored()` deliberately
|
|
812
|
+
// does not (an app that never ran has not "restarted"), and without it
|
|
813
|
+
// the budget below can never trip and the retry would be unbounded.
|
|
142
814
|
handle.markErrored();
|
|
815
|
+
handle.restarts++;
|
|
143
816
|
this.persistState();
|
|
817
|
+
this.scheduleRecovery(entry, handle, cfg);
|
|
144
818
|
throw err;
|
|
145
819
|
}
|
|
146
820
|
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
821
|
+
/**
|
|
822
|
+
* Everything a handle still runs, stopped: its supervisor's children (or its
|
|
823
|
+
* classic child process) AND its topology pools.
|
|
824
|
+
*
|
|
825
|
+
* The pools were missed. A pool's workers are not the supervisor's children
|
|
826
|
+
* — they are the daemon's, siblings of the server, recorded only in
|
|
827
|
+
* `topologyPools` — so `supervisor.stop()` never reached them, and nothing
|
|
828
|
+
* in the orchestrator destroyed a pool: only the daemon's own shutdown did.
|
|
829
|
+
* Every restart of an app with `instances > 1` left the previous workers
|
|
830
|
+
* running. On daos/test (2026-09-25): priceverse's `ohlcv-aggregator` and
|
|
831
|
+
* storage's `transform` in three generations at once.
|
|
832
|
+
*
|
|
833
|
+
* The pools go even when the supervisor's stop throws: a stop that failed
|
|
834
|
+
* half-way must not also leave the half nobody asked about.
|
|
835
|
+
*/
|
|
836
|
+
async tearDownProcesses(handle, force = false, timeout = 10_000) {
|
|
837
|
+
try {
|
|
838
|
+
if (handle.mode === 'bootstrap' && handle.supervisor) {
|
|
839
|
+
await handle.supervisor.stop();
|
|
158
840
|
}
|
|
159
|
-
|
|
160
|
-
|
|
841
|
+
else if (handle.childProcess) {
|
|
842
|
+
await this.stopChildProcess(handle, force, timeout);
|
|
161
843
|
}
|
|
162
|
-
handle.serviceRouter = null;
|
|
163
844
|
}
|
|
164
|
-
|
|
165
|
-
|
|
845
|
+
finally {
|
|
846
|
+
for (const [name, pool] of handle.topologyPools) {
|
|
847
|
+
try {
|
|
848
|
+
await pool.destroy();
|
|
849
|
+
}
|
|
850
|
+
catch (err) {
|
|
851
|
+
this.logger.warn({ app: handle.name, pool: name, error: err.message }, 'Destroying a topology pool failed — its workers may still run');
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
handle.topologyPools.clear();
|
|
166
855
|
}
|
|
167
|
-
|
|
168
|
-
|
|
856
|
+
}
|
|
857
|
+
/**
|
|
858
|
+
* Give back every service an app exposed on the daemon's Netron.
|
|
859
|
+
*
|
|
860
|
+
* A registration outlives the process behind it: the daemon keeps answering
|
|
861
|
+
* `queryInterface` with the full method list and every call through the name
|
|
862
|
+
* fails at the socket. `takeOverExisting` rescues a name the next launch
|
|
863
|
+
* re-exposes; one it does not stays advertised for the life of the daemon.
|
|
864
|
+
*/
|
|
865
|
+
async releaseExposedServices(handle) {
|
|
866
|
+
if (!handle.serviceRouter)
|
|
867
|
+
return;
|
|
868
|
+
try {
|
|
869
|
+
await handle.serviceRouter.releaseAll();
|
|
169
870
|
}
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
871
|
+
catch (err) {
|
|
872
|
+
this.logger.warn({ app: handle.name, error: err.message }, 'Releasing exposed services failed — the daemon may still advertise them');
|
|
873
|
+
}
|
|
874
|
+
handle.serviceRouter = null;
|
|
875
|
+
}
|
|
876
|
+
async stopApp(name, force = false, timeout = 10_000) {
|
|
877
|
+
const canonical = this.resolveAppName(name) ?? name;
|
|
878
|
+
// Outside the queue on purpose: the operator's intent lands the
|
|
879
|
+
// moment the command arrives, not whenever the queue drains.
|
|
880
|
+
this.clearCoolDown(canonical);
|
|
881
|
+
return this.lifecycleQueue.enqueue(canonical, 'stop', async () => {
|
|
882
|
+
const handle = this.handles.get(canonical);
|
|
883
|
+
if (!handle || handle.status === 'stopped')
|
|
884
|
+
return;
|
|
885
|
+
handle.markStopping();
|
|
886
|
+
// Unexpose topology services from daemon Netron before stopping processes
|
|
887
|
+
await this.releaseExposedServices(handle);
|
|
888
|
+
await this.tearDownProcesses(handle, force, timeout);
|
|
889
|
+
handle.markStopped();
|
|
890
|
+
this.persistState();
|
|
891
|
+
this.logger.info({ app: name }, 'App stopped');
|
|
892
|
+
});
|
|
173
893
|
}
|
|
174
894
|
async stopAll(force = false) {
|
|
175
895
|
if (!this.config)
|
|
@@ -178,6 +898,11 @@ export class OrchestratorService extends EventEmitter {
|
|
|
178
898
|
clearInterval(this.metricsTimer);
|
|
179
899
|
this.metricsTimer = null;
|
|
180
900
|
}
|
|
901
|
+
// Stop the janitor first so it doesn't race the controlled
|
|
902
|
+
// shutdown by trying to "reap" children that are mid-stop.
|
|
903
|
+
if (this.janitor) {
|
|
904
|
+
this.janitor.stop();
|
|
905
|
+
}
|
|
181
906
|
let count = 0;
|
|
182
907
|
// When force=true (daemon shutdown), use short timeout per app (3s SIGTERM, then SIGKILL)
|
|
183
908
|
const perAppTimeout = force ? 3_000 : 10_000;
|
|
@@ -209,36 +934,270 @@ export class OrchestratorService extends EventEmitter {
|
|
|
209
934
|
this.buildResults.clear();
|
|
210
935
|
return count;
|
|
211
936
|
}
|
|
937
|
+
/**
|
|
938
|
+
* Give a supervised app the definition its next start will use — its
|
|
939
|
+
* environment, its command. The running process is not touched; the next
|
|
940
|
+
* `restartApp` starts from this. `false` when nothing by that name runs
|
|
941
|
+
* here. See `AppHandle.redefine`.
|
|
942
|
+
*/
|
|
943
|
+
redefineApp(name, entry) {
|
|
944
|
+
const handle = this.handles.get(this.resolveAppName(name) ?? name);
|
|
945
|
+
if (!handle)
|
|
946
|
+
return false;
|
|
947
|
+
handle.redefine(entry);
|
|
948
|
+
return true;
|
|
949
|
+
}
|
|
950
|
+
/**
|
|
951
|
+
* Restart an app, coalescing concurrent triggers.
|
|
952
|
+
*
|
|
953
|
+
* The dev-mode hot-reload path can deliver many restart triggers per
|
|
954
|
+
* second (file save → esbuild rebuild → onEnd → debounced → restart),
|
|
955
|
+
* and naive parallelism here ends in cross-stomping state mutations
|
|
956
|
+
* (one restart's stopApp tearing down the other's freshly-spawned
|
|
957
|
+
* child). This wrapper serialises restarts per app:
|
|
958
|
+
*
|
|
959
|
+
* - If a restart is already in flight, we set the `pending` bit and
|
|
960
|
+
* await the current restart. When it completes we run exactly one
|
|
961
|
+
* follow-up — extra triggers received in the meantime collapse onto
|
|
962
|
+
* the same bit, so a burst of N triggers produces at most 2 actual
|
|
963
|
+
* restarts (the current one + one trailing).
|
|
964
|
+
* - If we're inside the post-online cooldown window, the trigger is
|
|
965
|
+
* queued (via `pending`) and fires after the cooldown clears.
|
|
966
|
+
*
|
|
967
|
+
* The hot path is `restartAppNow()` which contains the actual work and
|
|
968
|
+
* is what tests / non-coalescing callers can hit directly.
|
|
969
|
+
*/
|
|
212
970
|
async restartApp(name) {
|
|
971
|
+
// External entry — record one new trigger and dispatch.
|
|
972
|
+
return this.restartAppCoalesced(name, /* fromExternal */ true);
|
|
973
|
+
}
|
|
974
|
+
/**
|
|
975
|
+
* Internal coalescer dispatcher.
|
|
976
|
+
*
|
|
977
|
+
* `fromExternal=true` bumps `triggerSerial` (a user / file-watcher /
|
|
978
|
+
* RPC asked us to restart). `fromExternal=false` is used by the
|
|
979
|
+
* recursive arms (cooldown-wait, await-inflight) which are NOT new
|
|
980
|
+
* triggers, just continuations — they must not bump or the trailing
|
|
981
|
+
* coalesced restart would re-fire forever.
|
|
982
|
+
*/
|
|
983
|
+
async restartAppCoalesced(name, fromExternal) {
|
|
984
|
+
const canonical = this.resolveAppName(name) ?? name;
|
|
985
|
+
const handle = this.handles.get(canonical);
|
|
986
|
+
if (!handle) {
|
|
987
|
+
// A PROJECT-QUALIFIED name with no live handle is not this daemon's to
|
|
988
|
+
// start, and matching it against the daemon's own config starts the
|
|
989
|
+
// WRONG APP.
|
|
990
|
+
//
|
|
991
|
+
// The names collide in practice. omnitron's own `omnitron.config.ts`
|
|
992
|
+
// declares `main`, `payments`, `messaging`, `storage`, `pricing` — the
|
|
993
|
+
// same five a downstream stack uses. On 2026-09-10, after payments died,
|
|
994
|
+
// `omnitron restart acme/dev/payments` matched omnitron's sample entry by
|
|
995
|
+
// its last path segment and launched `apps/payments/src/main.ts` relative
|
|
996
|
+
// to the DAEMON's cwd. It died with ERR_MODULE_NOT_FOUND on a path that
|
|
997
|
+
// exists in neither repository, and the backend stayed down while the
|
|
998
|
+
// command reported that it had restarted it.
|
|
999
|
+
//
|
|
1000
|
+
// Starting the wrong app is worse than refusing: the operator is told
|
|
1001
|
+
// their service is coming back, and the log that says otherwise is the
|
|
1002
|
+
// one they are not reading yet. `startApp` on the daemon already refuses
|
|
1003
|
+
// this exact shape and names the command that works; this is the same
|
|
1004
|
+
// refusal on the same reasoning, at the point where "no live handle" is
|
|
1005
|
+
// already known.
|
|
1006
|
+
const qualified = name.includes('/');
|
|
1007
|
+
const ownProject = this.config?.project;
|
|
1008
|
+
if (qualified && !ownProject) {
|
|
1009
|
+
const [project, stack] = name.split('/');
|
|
1010
|
+
throw new Error(`'${name}' belongs to a registered project, which this daemon restarts through its ` +
|
|
1011
|
+
`stack. Use \`omnitron stack start ${project} ${stack}\`, or restart the app by its ` +
|
|
1012
|
+
`bare name if you mean the daemon's own config.`);
|
|
1013
|
+
}
|
|
1014
|
+
// P1-F — fall through to startApp when there's no live handle but
|
|
1015
|
+
// the ecosystem config declares the app. Operators expect
|
|
1016
|
+
// `omnitron restart foo` to work even when `foo` was never started
|
|
1017
|
+
// (or was stopped, or crashed and was reaped). Previously this
|
|
1018
|
+
// threw "Unknown app", forcing the operator to first run start
|
|
1019
|
+
// explicitly.
|
|
1020
|
+
const effective = effectiveAppName(name);
|
|
1021
|
+
const entry = this.config?.apps.find((a) => a.name === name || a.name === effective || effectiveAppName(a.name) === effective);
|
|
1022
|
+
if (entry) {
|
|
1023
|
+
this.logger.info({ app: name }, 'restartApp: no live handle — falling through to startApp');
|
|
1024
|
+
// Restart under the name the OPERATOR asked for. The match above
|
|
1025
|
+
// accepts a bare entry for a qualified request, and handing it to
|
|
1026
|
+
// `startApp` unchanged re-registers the app bare —
|
|
1027
|
+
// `ensureNamespacedEntry` leaves an entry with no OMNITRON_PROJECT /
|
|
1028
|
+
// OMNITRON_STACK env alone by design. The app then silently leaves its
|
|
1029
|
+
// stack, because `ProjectService.toStackInfo` filters handles on the
|
|
1030
|
+
// `${project}/${stack}/` prefix.
|
|
1031
|
+
const requested = name.includes('/') && !entry.name.includes('/')
|
|
1032
|
+
? { ...entry, name }
|
|
1033
|
+
: entry;
|
|
1034
|
+
return this.startApp(requested);
|
|
1035
|
+
}
|
|
1036
|
+
throw new Error(`Unknown app: ${name}`);
|
|
1037
|
+
}
|
|
1038
|
+
name = canonical;
|
|
1039
|
+
const state = this.getCoalescerState(name);
|
|
1040
|
+
if (fromExternal)
|
|
1041
|
+
state.triggerSerial += 1;
|
|
1042
|
+
const observedSerial = state.triggerSerial;
|
|
1043
|
+
// Cooldown: the app reported online recently. Wait it out, then
|
|
1044
|
+
// proceed as a non-external continuation (we already counted this
|
|
1045
|
+
// trigger when we first arrived).
|
|
1046
|
+
const now = Date.now();
|
|
1047
|
+
if (now < state.cooldownUntil) {
|
|
1048
|
+
const waitMs = state.cooldownUntil - now;
|
|
1049
|
+
this.logger.debug({ app: name, cooldownRemainingMs: waitMs }, 'Restart trigger received during post-online cooldown — queued');
|
|
1050
|
+
await sleep(waitMs);
|
|
1051
|
+
return this.restartAppCoalesced(name, /* fromExternal */ false);
|
|
1052
|
+
}
|
|
1053
|
+
// Already restarting — wait for it, then re-enter as a continuation.
|
|
1054
|
+
// The trailing-fire gate at the end of the in-flight finally block
|
|
1055
|
+
// is what re-fires us if our serial outranks what got consumed.
|
|
1056
|
+
if (state.inFlight) {
|
|
1057
|
+
await state.inFlight.catch(() => { });
|
|
1058
|
+
// If the in-flight restart already consumed our serial (or a
|
|
1059
|
+
// later one), we have nothing new to do — return its result by
|
|
1060
|
+
// re-reading the current handle. Otherwise fall through and
|
|
1061
|
+
// claim the slot ourselves.
|
|
1062
|
+
if (state.firedSerial >= observedSerial) {
|
|
1063
|
+
const refreshed = this.handles.get(name);
|
|
1064
|
+
if (refreshed)
|
|
1065
|
+
return refreshed;
|
|
1066
|
+
}
|
|
1067
|
+
return this.restartAppCoalesced(name, /* fromExternal */ false);
|
|
1068
|
+
}
|
|
1069
|
+
// Take the slot. Capture which serial this run is consuming —
|
|
1070
|
+
// anything bumped after this is the next run's problem.
|
|
1071
|
+
const consumedSerial = state.triggerSerial;
|
|
1072
|
+
const promise = this.restartAppNow(name);
|
|
1073
|
+
state.inFlight = promise;
|
|
1074
|
+
try {
|
|
1075
|
+
// Cooldown window is started from the `app:online` transition
|
|
1076
|
+
// inside startApp(), not here, so the timer is tied to the moment
|
|
1077
|
+
// the new child reports ready, not the moment our promise resolves.
|
|
1078
|
+
return await promise;
|
|
1079
|
+
}
|
|
1080
|
+
finally {
|
|
1081
|
+
state.inFlight = null;
|
|
1082
|
+
state.firedSerial = consumedSerial;
|
|
1083
|
+
// Trailing coalesced fire — only if a GENUINELY NEW external
|
|
1084
|
+
// trigger arrived between when we claimed the slot and now.
|
|
1085
|
+
// Two coalescer arms colliding can no longer cause this branch
|
|
1086
|
+
// to fire because they don't bump triggerSerial.
|
|
1087
|
+
if (state.triggerSerial > consumedSerial) {
|
|
1088
|
+
this.restartAppCoalesced(name, /* fromExternal */ false).catch((err) => {
|
|
1089
|
+
this.logger.error({ app: name, error: err.message }, 'Trailing coalesced restart failed');
|
|
1090
|
+
});
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
/**
|
|
1095
|
+
* Internal restart hot path — assumes coalescer has cleared us to proceed.
|
|
1096
|
+
*
|
|
1097
|
+
* Architecture note (Fix #247): we deliberately do NOT tear down the
|
|
1098
|
+
* esbuild watch or invalidate buildResults here. The watch is what
|
|
1099
|
+
* triggered us in the first place — its bundle is already fresh, and
|
|
1100
|
+
* disposing the context creates a window where the next save can be
|
|
1101
|
+
* missed (esbuild's metafile doesn't carry over between contexts).
|
|
1102
|
+
*
|
|
1103
|
+
* `launchBootstrapMode` sees the live buildResult + watch and skips
|
|
1104
|
+
* the cold rebuild, so a restart cycle is just: kill old child →
|
|
1105
|
+
* spawn new child with the bundle that was already on disk. The
|
|
1106
|
+
* bootstrap-config cache is cleared because changes to `bootstrap.ts`
|
|
1107
|
+
* (topology definition) need to take effect on the next start.
|
|
1108
|
+
*/
|
|
1109
|
+
/**
|
|
1110
|
+
* Start an app this daemon already supervises, using the entry its handle
|
|
1111
|
+
* carries.
|
|
1112
|
+
*
|
|
1113
|
+
* `startApp(entry)` needs an ecosystem entry, which the daemon's own config
|
|
1114
|
+
* is only one source of — a project stack registers handles this daemon
|
|
1115
|
+
* never declared. `restartAppNow` has always taken the entry off the handle
|
|
1116
|
+
* for exactly that reason; this is the same move for a start, so the name
|
|
1117
|
+
* `list` prints is a name `start` accepts.
|
|
1118
|
+
*
|
|
1119
|
+
* Returns `undefined` when the app is unknown, leaving the caller to decide
|
|
1120
|
+
* what to say about a name that names nothing.
|
|
1121
|
+
*/
|
|
1122
|
+
async startKnownApp(name) {
|
|
1123
|
+
const canonical = this.resolveAppName(name) ?? name;
|
|
1124
|
+
const handle = this.handles.get(canonical);
|
|
1125
|
+
if (!handle)
|
|
1126
|
+
return undefined;
|
|
1127
|
+
return this.startApp(handle.entry);
|
|
1128
|
+
}
|
|
1129
|
+
async restartAppNow(name) {
|
|
213
1130
|
const handle = this.handles.get(name);
|
|
214
1131
|
if (!handle)
|
|
215
1132
|
throw new Error(`Unknown app: ${name}`);
|
|
216
1133
|
const entry = handle.entry;
|
|
217
|
-
// In dev mode, clear bootstrap config cache so topology changes are picked up
|
|
218
1134
|
if (this.devMode && entry.bootstrap) {
|
|
219
|
-
clearCacheFor(path.resolve(this.cwd, entry.bootstrap));
|
|
220
|
-
// Stop esbuild watch and clear stale build result — launchBootstrapMode will rebuild + re-watch
|
|
221
|
-
if (this.buildService) {
|
|
222
|
-
await this.buildService.unwatchApp(name);
|
|
223
|
-
}
|
|
224
|
-
this.buildResults.delete(name);
|
|
1135
|
+
clearCacheFor(path.resolve(entry.cwd ?? this.cwd, entry.bootstrap));
|
|
225
1136
|
}
|
|
226
1137
|
await this.stopApp(name);
|
|
227
1138
|
return this.startApp(entry);
|
|
228
1139
|
}
|
|
1140
|
+
getCoalescerState(name) {
|
|
1141
|
+
let state = this.restartCoalescer.get(name);
|
|
1142
|
+
if (!state) {
|
|
1143
|
+
state = { inFlight: null, triggerSerial: 0, firedSerial: 0, cooldownUntil: 0 };
|
|
1144
|
+
this.restartCoalescer.set(name, state);
|
|
1145
|
+
}
|
|
1146
|
+
return state;
|
|
1147
|
+
}
|
|
1148
|
+
/**
|
|
1149
|
+
* Start the post-online cooldown for an app. Called from the `app:online`
|
|
1150
|
+
* transition (both initial startApp and subsequent restartApp paths
|
|
1151
|
+
* funnel through this), so a freshly-online app cannot be torn down by
|
|
1152
|
+
* a rebuild signal that was already in flight when it finished booting.
|
|
1153
|
+
*/
|
|
1154
|
+
startPostOnlineCooldown(name) {
|
|
1155
|
+
const state = this.getCoalescerState(name);
|
|
1156
|
+
state.cooldownUntil = Date.now() + OrchestratorService.POST_ONLINE_COOLDOWN_MS;
|
|
1157
|
+
}
|
|
229
1158
|
/**
|
|
230
|
-
* Resolve
|
|
231
|
-
*
|
|
232
|
-
*
|
|
1159
|
+
* Resolve a user-supplied name to a canonical handle key.
|
|
1160
|
+
*
|
|
1161
|
+
* Inputs accepted:
|
|
1162
|
+
* - exact canonical key (`omni/dev/payments`) → returned as-is
|
|
1163
|
+
* - bare short name (`payments`) → returns the canonical key whose
|
|
1164
|
+
* last `/`-segment matches, IF AND ONLY IF the match is unique.
|
|
1165
|
+
*
|
|
1166
|
+
* A qualified name that matches no handle exactly resolves to nothing: it
|
|
1167
|
+
* names a deployment this daemon does not have, and the nearest app with
|
|
1168
|
+
* the same last segment is a different one.
|
|
1169
|
+
*
|
|
1170
|
+
* Returns `undefined` for no match. Throws on ambiguity (multiple
|
|
1171
|
+
* stacks expose the same short name) so the caller can surface a
|
|
1172
|
+
* clear error to the operator instead of operating on the wrong
|
|
1173
|
+
* app.
|
|
233
1174
|
*/
|
|
234
1175
|
resolveAppName(rawName) {
|
|
235
1176
|
if (this.handles.has(rawName))
|
|
236
1177
|
return rawName;
|
|
1178
|
+
// A qualified name is an ADDRESS, and the only handle that satisfies it
|
|
1179
|
+
// is the one with that address. Falling through to the short-name match
|
|
1180
|
+
// answered a question about one deployment with another's data:
|
|
1181
|
+
//
|
|
1182
|
+
// omnitron logs daos/deployed/main → daos/dev/main
|
|
1183
|
+
//
|
|
1184
|
+
// — the node's app asked for, this laptop's app printed, with nothing
|
|
1185
|
+
// said. `startApp` already refuses this for the same reason ("a
|
|
1186
|
+
// PROJECT-QUALIFIED name with no live handle is not this daemon's to
|
|
1187
|
+
// start"); the resolver every other caller goes through did not.
|
|
1188
|
+
if (rawName.includes('/'))
|
|
1189
|
+
return undefined;
|
|
1190
|
+
const effective = effectiveAppName(rawName);
|
|
1191
|
+
const matches = [];
|
|
237
1192
|
for (const key of this.handles.keys()) {
|
|
238
|
-
if (key
|
|
239
|
-
|
|
1193
|
+
if (effectiveAppName(key) === effective)
|
|
1194
|
+
matches.push(key);
|
|
240
1195
|
}
|
|
241
|
-
|
|
1196
|
+
if (matches.length === 0)
|
|
1197
|
+
return undefined;
|
|
1198
|
+
if (matches.length === 1)
|
|
1199
|
+
return matches[0];
|
|
1200
|
+
throw new Error(`App name '${rawName}' is ambiguous — multiple matches: ${matches.join(', ')}. Use the full canonical name.`);
|
|
242
1201
|
}
|
|
243
1202
|
/**
|
|
244
1203
|
* Reload app with zero-downtime strategy:
|
|
@@ -251,9 +1210,11 @@ export class OrchestratorService extends EventEmitter {
|
|
|
251
1210
|
* zero-downtime is not possible (single-process, no supervisor).
|
|
252
1211
|
*/
|
|
253
1212
|
async reloadApp(name) {
|
|
254
|
-
const
|
|
1213
|
+
const canonical = this.resolveAppName(name) ?? name;
|
|
1214
|
+
const handle = this.handles.get(canonical);
|
|
255
1215
|
if (!handle)
|
|
256
1216
|
throw new Error(`Unknown app: ${name}`);
|
|
1217
|
+
name = canonical;
|
|
257
1218
|
// Classic mode or no supervisor — cannot do zero-downtime, fall back to restart
|
|
258
1219
|
if (handle.mode !== 'bootstrap' || !handle.supervisor) {
|
|
259
1220
|
return this.restartApp(name);
|
|
@@ -279,8 +1240,11 @@ export class OrchestratorService extends EventEmitter {
|
|
|
279
1240
|
this.logger.info({ app: name, child: childName }, 'Child reloaded successfully');
|
|
280
1241
|
}
|
|
281
1242
|
catch (err) {
|
|
282
|
-
|
|
283
|
-
|
|
1243
|
+
// `error: err`, not `error: err.message`: the logger aliases `error`
|
|
1244
|
+
// onto pino's error serializer, so the object arrives with its stack
|
|
1245
|
+
// and a string arrives as a string with nothing behind it.
|
|
1246
|
+
this.logger.error({ app: name, child: childName, error: err }, 'Failed to reload child — aborting zero-downtime reload');
|
|
1247
|
+
throw new Error(`Zero-downtime reload failed for '${name}' at child '${childName}': ${err.message}`, { cause: err });
|
|
284
1248
|
}
|
|
285
1249
|
}
|
|
286
1250
|
this.logger.info({ app: name }, 'Zero-downtime reload complete');
|
|
@@ -313,9 +1277,11 @@ export class OrchestratorService extends EventEmitter {
|
|
|
313
1277
|
// Scaling — delegates to PM supervisor
|
|
314
1278
|
// ============================================================================
|
|
315
1279
|
async scaleApp(name, instances) {
|
|
316
|
-
const
|
|
1280
|
+
const canonical = this.resolveAppName(name) ?? name;
|
|
1281
|
+
const handle = this.handles.get(canonical);
|
|
317
1282
|
if (!handle)
|
|
318
1283
|
throw new Error(`Unknown app: ${name}`);
|
|
1284
|
+
name = canonical;
|
|
319
1285
|
if (!this.config)
|
|
320
1286
|
throw new Error('Ecosystem config not loaded');
|
|
321
1287
|
if (handle.mode !== 'bootstrap') {
|
|
@@ -345,11 +1311,18 @@ export class OrchestratorService extends EventEmitter {
|
|
|
345
1311
|
return handles.map((h) => this.toProcessInfo(h));
|
|
346
1312
|
}
|
|
347
1313
|
getApp(name) {
|
|
348
|
-
const
|
|
1314
|
+
const canonical = this.resolveAppName(name) ?? name;
|
|
1315
|
+
const h = this.handles.get(canonical);
|
|
349
1316
|
return h ? this.toProcessInfo(h) : null;
|
|
350
1317
|
}
|
|
351
1318
|
getHandle(name) {
|
|
352
|
-
|
|
1319
|
+
// Match getAppStatus / getLogs name-resolution semantics — accept either
|
|
1320
|
+
// a fully namespaced handle key ('project/stack/app') or the bare app
|
|
1321
|
+
// name and resolve to the right handle. Without this, CLI commands like
|
|
1322
|
+
// `omnitron env main` fail with "Unknown app: main" even though the
|
|
1323
|
+
// app exists as `omni/dev/main`.
|
|
1324
|
+
const resolved = this.resolveAppName(name);
|
|
1325
|
+
return this.handles.get(resolved ?? name);
|
|
353
1326
|
}
|
|
354
1327
|
/**
|
|
355
1328
|
* List all handle names matching a prefix.
|
|
@@ -364,41 +1337,63 @@ export class OrchestratorService extends EventEmitter {
|
|
|
364
1337
|
getWorkerHandle(processId) {
|
|
365
1338
|
return this.pm.getWorkerHandle(processId);
|
|
366
1339
|
}
|
|
1340
|
+
/**
|
|
1341
|
+
* Surface the underlying IProcessInfo for a titan-pm processId
|
|
1342
|
+
* so callers (e.g. the daemon's `inspect` RPC) can read uptime
|
|
1343
|
+
* + status without reaching into the process-manager directly.
|
|
1344
|
+
*/
|
|
1345
|
+
getChildProcessInfo(processId) {
|
|
1346
|
+
return this.pm.getProcess(processId);
|
|
1347
|
+
}
|
|
367
1348
|
// ============================================================================
|
|
368
1349
|
// Monitoring — async, delegates to PM supervisor for bootstrap mode
|
|
369
1350
|
// ============================================================================
|
|
1351
|
+
/**
|
|
1352
|
+
* Pull the live DI dependency graph from a running app's first child
|
|
1353
|
+
* worker. Used by the daemon RPC backing `omnitron inspect <app>
|
|
1354
|
+
* --graph`. Returns null if the app isn't running, has no children,
|
|
1355
|
+
* or the worker doesn't expose `getDependencyGraph` (older bootstraps).
|
|
1356
|
+
*/
|
|
1357
|
+
async getDependencyGraph(name) {
|
|
1358
|
+
const handle = this.getHandle(name);
|
|
1359
|
+
if (!handle || handle.mode !== 'bootstrap' || !handle.supervisor)
|
|
1360
|
+
return null;
|
|
1361
|
+
const childNames = handle.supervisor.getChildNames();
|
|
1362
|
+
for (const childName of childNames) {
|
|
1363
|
+
const processId = handle.supervisor.getChildProcessId(childName);
|
|
1364
|
+
if (!processId)
|
|
1365
|
+
continue;
|
|
1366
|
+
const workerHandle = this.pm.getWorkerHandle(processId);
|
|
1367
|
+
const proxy = workerHandle?.proxy;
|
|
1368
|
+
if (!proxy?.getDependencyGraph)
|
|
1369
|
+
continue;
|
|
1370
|
+
try {
|
|
1371
|
+
const graph = await proxy.getDependencyGraph();
|
|
1372
|
+
return graph;
|
|
1373
|
+
}
|
|
1374
|
+
catch (err) {
|
|
1375
|
+
this.logger.warn({ app: name, child: childName, error: err.message }, 'getDependencyGraph: child threw — trying next worker');
|
|
1376
|
+
}
|
|
1377
|
+
}
|
|
1378
|
+
return null;
|
|
1379
|
+
}
|
|
1380
|
+
/**
|
|
1381
|
+
* `name` is resolved like every other per-app question (`main` finds
|
|
1382
|
+
* `daos/dev/main`); it was compared verbatim, so `omnitron metrics main`
|
|
1383
|
+
* printed «(no data)». An unknown name answers `{}`, and the caller says so.
|
|
1384
|
+
*/
|
|
370
1385
|
async getMetrics(name) {
|
|
1386
|
+
const wanted = name === undefined ? undefined : this.resolveAppName(name);
|
|
1387
|
+
if (name !== undefined && wanted === undefined)
|
|
1388
|
+
return {};
|
|
371
1389
|
const result = {};
|
|
372
1390
|
for (const [appName, handle] of this.handles) {
|
|
373
|
-
if (
|
|
1391
|
+
if (wanted && appName !== wanted)
|
|
374
1392
|
continue;
|
|
375
1393
|
if (handle.mode === 'bootstrap' && handle.supervisor) {
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
// Try PM-level RPC metrics first
|
|
380
|
-
const m = await handle.supervisor.getChildMetrics(childName);
|
|
381
|
-
if (m && (m.cpu > 0 || m.memory > 0)) {
|
|
382
|
-
cpu += m.cpu;
|
|
383
|
-
memory += m.memory;
|
|
384
|
-
requests += m.requests ?? 0;
|
|
385
|
-
errors += m.errors ?? 0;
|
|
386
|
-
}
|
|
387
|
-
else {
|
|
388
|
-
// Fallback: OS-level metrics via ps for the child PID
|
|
389
|
-
const processId = handle.supervisor.getChildProcessId(childName);
|
|
390
|
-
if (processId) {
|
|
391
|
-
const workerHandle = this.pm.getWorkerHandle(processId);
|
|
392
|
-
const childPid = workerHandle?.pid;
|
|
393
|
-
if (childPid) {
|
|
394
|
-
const osMetrics = await this.sampleProcessMetrics(childPid);
|
|
395
|
-
cpu += osMetrics.cpu;
|
|
396
|
-
memory += osMetrics.memory;
|
|
397
|
-
}
|
|
398
|
-
}
|
|
399
|
-
}
|
|
400
|
-
}
|
|
401
|
-
const aggregated = { cpu, memory, requests, errors };
|
|
1394
|
+
// Same sampler the poller uses, so `omnitron metrics` and `omnitron
|
|
1395
|
+
// list` cannot report different numbers for the same app.
|
|
1396
|
+
const aggregated = await this.sampleAppMetrics(handle);
|
|
402
1397
|
handle.lastMetrics = aggregated;
|
|
403
1398
|
result[appName] = aggregated;
|
|
404
1399
|
}
|
|
@@ -409,21 +1404,36 @@ export class OrchestratorService extends EventEmitter {
|
|
|
409
1404
|
}
|
|
410
1405
|
return result;
|
|
411
1406
|
}
|
|
1407
|
+
/**
|
|
1408
|
+
* Each app's health, asked of every process it runs — its supervisor
|
|
1409
|
+
* children and its worker pools — and combined (`app-health.ts`).
|
|
1410
|
+
*
|
|
1411
|
+
* `name` is resolved like every other per-app question: `main` finds
|
|
1412
|
+
* `daos/dev/main`. It was compared verbatim, so `omnitron health main`
|
|
1413
|
+
* matched nothing and the answer was empty. An unknown name answers `{}`,
|
|
1414
|
+
* and the caller says so.
|
|
1415
|
+
*/
|
|
412
1416
|
async getHealth(name) {
|
|
1417
|
+
const wanted = name === undefined ? undefined : this.resolveAppName(name);
|
|
1418
|
+
if (name !== undefined && wanted === undefined)
|
|
1419
|
+
return {};
|
|
413
1420
|
const result = {};
|
|
414
1421
|
for (const [appName, handle] of this.handles) {
|
|
415
|
-
if (
|
|
1422
|
+
if (wanted && appName !== wanted)
|
|
1423
|
+
continue;
|
|
1424
|
+
if (handle.status !== 'online') {
|
|
1425
|
+
result[appName] = notRunningHealth(handle.status);
|
|
416
1426
|
continue;
|
|
1427
|
+
}
|
|
417
1428
|
if (handle.mode === 'bootstrap' && handle.supervisor) {
|
|
418
|
-
const
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
result[appName] = health;
|
|
423
|
-
}
|
|
424
|
-
else {
|
|
425
|
-
result[appName] = handle.lastHealth;
|
|
1429
|
+
const supervisor = handle.supervisor;
|
|
1430
|
+
const answers = await Promise.all(supervisor.getChildNames().map(async (child) => [child, await supervisor.getChildHealth(child)]));
|
|
1431
|
+
for (const [pool, instance] of handle.topologyPools) {
|
|
1432
|
+
answers.push([pool, poolHealth(instance.metrics)]);
|
|
426
1433
|
}
|
|
1434
|
+
const health = answers.length > 0 ? combineProcessHealth(answers) : handle.lastHealth;
|
|
1435
|
+
handle.lastHealth = health;
|
|
1436
|
+
result[appName] = health;
|
|
427
1437
|
}
|
|
428
1438
|
else {
|
|
429
1439
|
result[appName] = handle.lastHealth;
|
|
@@ -488,8 +1498,13 @@ export class OrchestratorService extends EventEmitter {
|
|
|
488
1498
|
}
|
|
489
1499
|
getLogs(name, lines = 100) {
|
|
490
1500
|
const result = [];
|
|
1501
|
+
// Stack-mode handles are keyed as `<project>/<stack>/<app>` while CLI
|
|
1502
|
+
// users typically pass just the bare app name. Resolve via the same
|
|
1503
|
+
// matcher used by getAppStatus() so `omnitron logs main` works in
|
|
1504
|
+
// both single-app and stack modes.
|
|
1505
|
+
const resolved = name ? this.resolveAppName(name) ?? name : undefined;
|
|
491
1506
|
for (const [appName, handle] of this.handles) {
|
|
492
|
-
if (
|
|
1507
|
+
if (resolved && appName !== resolved)
|
|
493
1508
|
continue;
|
|
494
1509
|
result.push({ app: appName, lines: handle.getLogs(lines) });
|
|
495
1510
|
}
|
|
@@ -521,7 +1536,7 @@ export class OrchestratorService extends EventEmitter {
|
|
|
521
1536
|
else {
|
|
522
1537
|
this.logger.error({ app: entry.name, code, signal }, 'App exited unexpectedly');
|
|
523
1538
|
handle.markCrashed();
|
|
524
|
-
this.
|
|
1539
|
+
this.scheduleRecovery(entry, handle, config);
|
|
525
1540
|
}
|
|
526
1541
|
this.persistState();
|
|
527
1542
|
});
|
|
@@ -535,30 +1550,57 @@ export class OrchestratorService extends EventEmitter {
|
|
|
535
1550
|
*/
|
|
536
1551
|
async launchBootstrapMode(entry, handle, config) {
|
|
537
1552
|
handle.markStarting();
|
|
538
|
-
const bootstrapAbsPath = path.resolve(this.cwd, entry.bootstrap);
|
|
539
|
-
// Load bootstrap definition to check for process topology and requirements
|
|
1553
|
+
const bootstrapAbsPath = path.resolve(entry.cwd ?? this.cwd, entry.bootstrap);
|
|
1554
|
+
// Load bootstrap definition to check for process topology and requirements.
|
|
1555
|
+
//
|
|
1556
|
+
// In dev, in a process of its own (`freshModuleGraph`): imported here, a
|
|
1557
|
+
// package the definition uses would be the one this daemon first saw, for
|
|
1558
|
+
// as long as it runs (bootstrap-loader.ts has the measurement). What comes
|
|
1559
|
+
// back is data, which is all the daemon reads of it — here, in the build
|
|
1560
|
+
// service and in the stack resolver; the app's own processes load the
|
|
1561
|
+
// definition themselves, functions and all.
|
|
540
1562
|
let topology;
|
|
541
1563
|
let definition;
|
|
542
1564
|
try {
|
|
543
|
-
definition = await loadBootstrapConfig(bootstrapAbsPath, { devMode: this.devMode });
|
|
1565
|
+
definition = await loadBootstrapConfig(bootstrapAbsPath, { devMode: this.devMode, freshModuleGraph: true });
|
|
544
1566
|
topology = definition.processes;
|
|
545
1567
|
}
|
|
546
1568
|
catch (err) {
|
|
547
|
-
|
|
1569
|
+
// Refused, with the reason. This fell back to «single-process mode» at
|
|
1570
|
+
// warn — but every definition declares its topology (IAppDefinition:
|
|
1571
|
+
// `processes` is required), so the fallback started the app in a shape
|
|
1572
|
+
// it was never written for. Measured 2026-09-23 on the master: paysys
|
|
1573
|
+
// and messaging, whose definitions failed to load («Parameter
|
|
1574
|
+
// decorators only work when experimental decorators are enabled»), were
|
|
1575
|
+
// started that way and died on «Dependency 'AuthModule:JWTService' not
|
|
1576
|
+
// found» and a module path that single mode could not resolve — the
|
|
1577
|
+
// operator saw those, and the cause sat one line up at warn.
|
|
1578
|
+
throw new Error(`Could not load ${entry.name}'s definition from ${bootstrapAbsPath}: ${err.message}`, { cause: err });
|
|
548
1579
|
}
|
|
549
|
-
//
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
1580
|
+
// The app's `config/default.json`, read on every start and put on a COPY
|
|
1581
|
+
// of the definition. This wrote the section onto the loader's cached
|
|
1582
|
+
// definition «if not set», so the first start's reading was every later
|
|
1583
|
+
// start's — and the logger level beside it, read in the same block, was
|
|
1584
|
+
// never read again at all (`project/declared-config.ts`).
|
|
1585
|
+
if (definition) {
|
|
1586
|
+
const declared = readDeclaredConfig(bootstrapAbsPath);
|
|
1587
|
+
if (declared.malformed) {
|
|
1588
|
+
this.logger.error({ app: entry.name, configPath: declared.configPath, error: declared.malformed }, 'config/default.json does not parse — its `omnitron` section is being ignored');
|
|
1589
|
+
}
|
|
1590
|
+
definition = withDeclaredConfig(definition, declared);
|
|
1591
|
+
// The only place the parent can learn the level the app chose; children
|
|
1592
|
+
// are spawned with it below. Validated rather than trusted: this is a
|
|
1593
|
+
// hand-edited JSON file, and a level pino does not know ("verbose", a
|
|
1594
|
+
// typo) would be handed straight to a child's logger constructor.
|
|
1595
|
+
const level = declared.loggerLevel;
|
|
1596
|
+
if (typeof level === 'string') {
|
|
1597
|
+
if (PINO_LEVELS.has(level)) {
|
|
1598
|
+
handle.appLogLevel = level;
|
|
1599
|
+
}
|
|
1600
|
+
else {
|
|
1601
|
+
this.logger.warn({ app: entry.name, configPath: declared.configPath, level }, 'config/default.json declares a logger level this runtime does not know — children keep the default');
|
|
559
1602
|
}
|
|
560
1603
|
}
|
|
561
|
-
catch { /* config file missing — skip */ }
|
|
562
1604
|
}
|
|
563
1605
|
// Resolve infrastructure config and inject as env vars.
|
|
564
1606
|
// Skip if already injected by project.service.ts (stack flow sets OMNITRON_STACK).
|
|
@@ -588,25 +1630,95 @@ export class OrchestratorService extends EventEmitter {
|
|
|
588
1630
|
}
|
|
589
1631
|
}
|
|
590
1632
|
// Build all entry points via esbuild (dev mode bundles from source .ts)
|
|
1633
|
+
//
|
|
1634
|
+
// On a *restart* (as opposed to a first start), buildResults already
|
|
1635
|
+
// has a fresh entry — esbuild's watcher just produced the bundle
|
|
1636
|
+
// that triggered us. Reusing it skips a redundant cold rebuild
|
|
1637
|
+
// (which was costing ~500ms per restart and racing against the
|
|
1638
|
+
// watcher's own incremental output). The watch context is also
|
|
1639
|
+
// preserved (we never unwatched), so we don't re-arm it either.
|
|
591
1640
|
if (definition && this.devMode) {
|
|
592
1641
|
try {
|
|
593
1642
|
if (!this.buildService) {
|
|
594
|
-
this.buildService = new BuildService(/* isDev */ true
|
|
1643
|
+
this.buildService = new BuildService(/* isDev */ true, {
|
|
1644
|
+
onEsbuildReset: (info) => this.metricsBridge?.recordResilientHandleReset(info),
|
|
1645
|
+
onBuildResult: ({ app, kind, durationMs, ok }) => this.metricsBridge?.recordEsbuildBuild(app, kind, durationMs, ok),
|
|
1646
|
+
});
|
|
1647
|
+
}
|
|
1648
|
+
const existing = this.buildResults.get(entry.name);
|
|
1649
|
+
const existingWatch = this.buildService.isWatching?.(entry.name);
|
|
1650
|
+
// Bundle on-disk presence check. A restart can race against
|
|
1651
|
+
// the user (or a script) removing `.omnitron-build/` between
|
|
1652
|
+
// builds — without this stat the orchestrator hands the
|
|
1653
|
+
// worker a path that no longer exists and the child dies
|
|
1654
|
+
// with `ERR_MODULE_NOT_FOUND` before the supervisor can
|
|
1655
|
+
// recover. Cheap stat, big DX win.
|
|
1656
|
+
const bundlePresent = existing
|
|
1657
|
+
? fs.existsSync(existing.bootstrapPath)
|
|
1658
|
+
: false;
|
|
1659
|
+
if (existing && existingWatch && bundlePresent) {
|
|
1660
|
+
// Restart path — keep the already-built bundle and live watch.
|
|
1661
|
+
this.logger.debug({ app: entry.name, modules: existing.modulePaths.size }, 'Reusing existing esbuild bundle + watch (restart path)');
|
|
595
1662
|
}
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
1663
|
+
else {
|
|
1664
|
+
if (existing && existingWatch && !bundlePresent) {
|
|
1665
|
+
// Bundle was deleted out from under us. Clear the watch
|
|
1666
|
+
// and the cached BuildResult so the next call rebuilds
|
|
1667
|
+
// from cold and re-arms the watcher cleanly.
|
|
1668
|
+
this.logger.warn({ app: entry.name, bootstrapPath: existing.bootstrapPath }, 'Cached bundle missing on disk — forcing rebuild');
|
|
1669
|
+
try {
|
|
1670
|
+
await this.buildService.unwatchApp?.(entry.name);
|
|
1671
|
+
}
|
|
1672
|
+
catch (err) {
|
|
1673
|
+
this.logger.debug({ err, app: entry.name }, 'unwatchApp failed (non-fatal)');
|
|
1674
|
+
}
|
|
1675
|
+
this.buildResults.delete(entry.name);
|
|
1676
|
+
}
|
|
1677
|
+
// First-start path — build everything from cold and arm the watcher.
|
|
1678
|
+
const buildResult = await this.buildService.buildApp(entry.name, bootstrapAbsPath, definition);
|
|
1679
|
+
this.buildResults.set(entry.name, buildResult);
|
|
1680
|
+
this.logger.info({ app: entry.name, modules: buildResult.modulePaths.size }, 'Built entry points via esbuild');
|
|
1681
|
+
// Start esbuild watch for incremental rebuilds on source changes.
|
|
1682
|
+
// On rebuild → fire a restart trigger; the coalescer takes it
|
|
1683
|
+
// from there.
|
|
1684
|
+
await this.buildService.watchApp(entry.name, bootstrapAbsPath, definition, () => {
|
|
1685
|
+
// Gate: skip if the app is in a non-running state — repeated
|
|
1686
|
+
// restarts of an escalated/errored process just churn forever
|
|
1687
|
+
// without forward progress (Fix #248).
|
|
1688
|
+
// Skip restart when the app is in a terminal non-running
|
|
1689
|
+
// state. Includes 'crashed' (supervisor escalated after
|
|
1690
|
+
// exceeding its restart budget) and 'errored' (start
|
|
1691
|
+
// failed). 'stopped' means operator explicitly stopped it.
|
|
1692
|
+
// In all three cases, restarting just churns a known-broken
|
|
1693
|
+
// process; the operator must fix the underlying issue and
|
|
1694
|
+
// call `omnitron start <app>` to clear the state.
|
|
1695
|
+
const status = this.getAppStatus(entry.name);
|
|
1696
|
+
if (status === 'errored' || status === 'crashed' || status === 'stopped') {
|
|
1697
|
+
this.logger.warn({ app: entry.name, status }, 'esbuild rebuild detected but app is not running — skipping restart');
|
|
1698
|
+
return;
|
|
1699
|
+
}
|
|
1700
|
+
this.logger.info({ app: entry.name }, 'esbuild rebuild detected — restarting');
|
|
1701
|
+
this.metricsBridge?.recordEsbuildBuild(entry.name, 'rebuild', 0, true);
|
|
1702
|
+
this.restartApp(entry.name).catch((err) => {
|
|
1703
|
+
this.logger.error({ app: entry.name, error: err.message }, 'Restart after esbuild rebuild failed');
|
|
1704
|
+
});
|
|
605
1705
|
});
|
|
606
|
-
}
|
|
1706
|
+
}
|
|
607
1707
|
}
|
|
608
1708
|
catch (err) {
|
|
609
|
-
|
|
1709
|
+
// The "fall back to tsx" wording was misleading — the orchestrator
|
|
1710
|
+
// does not inject a tsx loader into spawned children, so without
|
|
1711
|
+
// a build the bootstrap-process will try to `import` the source
|
|
1712
|
+
// .ts file (referenced as `./app.module.js` in the topology) and
|
|
1713
|
+
// fail with a cryptic "Cannot find module" downstream. Surface
|
|
1714
|
+
// the build error directly so the operator sees the actual
|
|
1715
|
+
// problem on `omnitron start/restart`.
|
|
1716
|
+
// esbuild's error carries an `errors[]` with file, line and column.
|
|
1717
|
+
// `.message` is the summary; the locations are what an operator needs
|
|
1718
|
+
// to fix the build, and they exist only on the original.
|
|
1719
|
+
this.logger.error({ app: entry.name, error: err }, 'esbuild build failed — app cannot start without bundled output');
|
|
1720
|
+
throw new Error(`Cannot launch '${entry.name}' in dev mode: esbuild build failed (${err.message}). ` +
|
|
1721
|
+
`Fix the build error and retry, or run with bundled output present in apps/${entry.name}/.omnitron-build/.`, { cause: err });
|
|
610
1722
|
}
|
|
611
1723
|
}
|
|
612
1724
|
const buildResult = this.buildResults.get(entry.name);
|
|
@@ -634,10 +1746,11 @@ export class OrchestratorService extends EventEmitter {
|
|
|
634
1746
|
*/
|
|
635
1747
|
async launchTopology(entry, handle, config, bootstrapAbsPath, topology, buildResult) {
|
|
636
1748
|
const policy = buildRestartPolicy(entry, config);
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
1749
|
+
// Wait for daemon Netron to be wired up. Cheap no-op if already ready;
|
|
1750
|
+
// otherwise blocks (with timeout) instead of immediately failing — the
|
|
1751
|
+
// race between daemon startup and stack auto-resume is benign and just
|
|
1752
|
+
// needs a single rendezvous.
|
|
1753
|
+
await this.ensureDaemonNetronReady();
|
|
641
1754
|
const serviceRouter = new ServiceRouter(this.daemonNetron, this.logger);
|
|
642
1755
|
const daemonSocketUrl = `unix://${this.daemonSocketPath}`;
|
|
643
1756
|
// Topology startup order: providers (topology.expose) before consumers (topology.access).
|
|
@@ -658,14 +1771,20 @@ export class OrchestratorService extends EventEmitter {
|
|
|
658
1771
|
size: instances,
|
|
659
1772
|
strategy: PoolStrategy.POWER_OF_TWO,
|
|
660
1773
|
metrics: true,
|
|
661
|
-
replaceUnhealthy:
|
|
662
|
-
|
|
1774
|
+
// `replaceUnhealthy` is deliberately not set: titan-pm declares it
|
|
1775
|
+
// and reads it nowhere. An unhealthy worker is replaced by the
|
|
1776
|
+
// health path regardless, so `true` was a no-op that happened to
|
|
1777
|
+
// match the behaviour — and `false` would have been a no-op that did
|
|
1778
|
+
// not, leaving an operator believing they had turned replacement off.
|
|
1779
|
+
// What actually governs it is `healthCheck.unhealthyThreshold`.
|
|
1780
|
+
requestTimeout: CHILD_REQUEST_TIMEOUT,
|
|
663
1781
|
maxQueueSize: 200,
|
|
664
1782
|
spawnOptions: {
|
|
665
1783
|
name: `${entry.name}/${procEntry.name}`,
|
|
666
1784
|
version: '1.0.0',
|
|
667
1785
|
allMethodsPublic: true,
|
|
668
|
-
|
|
1786
|
+
...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
|
|
1787
|
+
startupTimeout: procEntry.startupTimeout ?? entry.startupTimeout ?? DEFAULT_CHILD_STARTUP_TIMEOUT,
|
|
669
1788
|
...(entry.env && { env: entry.env }),
|
|
670
1789
|
...(entry.cwd && { cwd: entry.cwd }),
|
|
671
1790
|
dependencies: {
|
|
@@ -688,48 +1807,86 @@ export class OrchestratorService extends EventEmitter {
|
|
|
688
1807
|
enabled: procEntry.health?.enabled !== false,
|
|
689
1808
|
interval: procEntry.health?.interval ?? config.monitoring.healthCheck.interval,
|
|
690
1809
|
},
|
|
1810
|
+
// `scaling.strategy` is what opts a pool into autoscaling; anything
|
|
1811
|
+
// else — including the default of not declaring it — means the pool
|
|
1812
|
+
// stays at the declared `instances`, which is what 'fixed' says on
|
|
1813
|
+
// the tin.
|
|
1814
|
+
//
|
|
1815
|
+
// Both arms of this ternary used to set `enabled: true`, differing
|
|
1816
|
+
// only in where the numbers came from. The setting existed, the
|
|
1817
|
+
// mechanism existed, and nothing connected them: every pool in the
|
|
1818
|
+
// platform was autoscaled whether or not it asked to be. Live effect
|
|
1819
|
+
// on two pools declared `instances: 2` — grown to their cap of eight
|
|
1820
|
+
// and held there, sixteen processes where four were configured.
|
|
1821
|
+
//
|
|
1822
|
+
// `min` follows the declared instance count rather than 1: a pool
|
|
1823
|
+
// asked for two workers should not be scaled below two.
|
|
691
1824
|
autoScale: procEntry.scaling?.strategy === 'auto'
|
|
692
1825
|
? {
|
|
693
1826
|
enabled: true,
|
|
694
|
-
min:
|
|
1827
|
+
min: instances,
|
|
695
1828
|
max: procEntry.scaling?.maxInstances ?? instances * 4,
|
|
696
1829
|
targetCPU: procEntry.scaling?.targetCPU ?? 70,
|
|
697
1830
|
targetMemory: procEntry.scaling?.targetMemory ?? 80,
|
|
698
1831
|
queueThreshold: procEntry.scaling?.queueThreshold ?? 10,
|
|
699
1832
|
cooldownPeriod: procEntry.scaling?.cooldownPeriod ?? 30_000,
|
|
700
1833
|
}
|
|
701
|
-
: {
|
|
702
|
-
enabled: true,
|
|
703
|
-
min: 1,
|
|
704
|
-
max: instances * 4,
|
|
705
|
-
targetCPU: 70,
|
|
706
|
-
targetMemory: 80,
|
|
707
|
-
queueThreshold: 10,
|
|
708
|
-
cooldownPeriod: 30_000,
|
|
709
|
-
},
|
|
1834
|
+
: { enabled: false, min: instances, max: instances },
|
|
710
1835
|
};
|
|
711
1836
|
this.logger.info({ app: entry.name, process: procEntry.name, instances }, 'Creating topology process pool');
|
|
712
1837
|
// pm.pool() returns a Proxy wrapping ProcessPool — execute() dispatches to pool workers
|
|
713
1838
|
const pool = await this.pm.pool(BOOTSTRAP_PROCESS_PATH, poolOptions);
|
|
714
1839
|
poolNames.push(procEntry.name);
|
|
1840
|
+
// Track the pool on the AppHandle so `getInfo()` can report the
|
|
1841
|
+
// actual worker count for pool-managed topology entries. Without
|
|
1842
|
+
// this the orchestrator falls back to `supervisor.getChildProcessId()`
|
|
1843
|
+
// which returns null for pool workers (they aren't supervisor
|
|
1844
|
+
// children) and the entry is mis-reported as `status: 'stopped'`.
|
|
1845
|
+
handle.topologyPools.set(procEntry.name, pool);
|
|
1846
|
+
this.attachPoolLogCapture(entry.name, procEntry.name, handle, pool);
|
|
715
1847
|
// Auto-discover @Service metadata from pool workers and expose via ServiceRouter.
|
|
716
1848
|
// No manual `exports` needed — @Service decorator IS the source of truth.
|
|
717
1849
|
// Requires explicit `topology: { expose: true }` — opt-in for security.
|
|
718
1850
|
if (procEntry.topology?.expose === true) {
|
|
719
1851
|
try {
|
|
1852
|
+
// A pool worker reports ready before its Application has started, and
|
|
1853
|
+
// `@Service` classes are exposed to Netron during start — so asking
|
|
1854
|
+
// once, or once more 500ms later, asks a worker that is still booting.
|
|
1855
|
+
// What it answers then is an empty list, or a service whose method
|
|
1856
|
+
// list has not been filled in yet, and both used to end the same way:
|
|
1857
|
+
// nothing registered on the daemon, the consumer process starting
|
|
1858
|
+
// seconds later, querying, and being told the service does not exist.
|
|
1859
|
+
// The consumer resolves its topology proxy exactly once, so that
|
|
1860
|
+
// answer disabled the feature for the life of the process — in the
|
|
1861
|
+
// case that surfaced this, every OHLCV candle aggregation.
|
|
1862
|
+
//
|
|
1863
|
+
// Poll instead, and treat "a service with no methods" as not ready
|
|
1864
|
+
// rather than as a service, because an empty method list produces a
|
|
1865
|
+
// proxy that can do nothing.
|
|
1866
|
+
const usable = (list) => Array.isArray(list) && list.length > 0 && list.some((svc) => svc.methods?.length > 0);
|
|
720
1867
|
let services = await pool.execute('getExposedServices');
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
await new Promise((r) => setTimeout(r, 500));
|
|
1868
|
+
const deadline = Date.now() + TOPOLOGY_EXPOSE_TIMEOUT_MS;
|
|
1869
|
+
while (!usable(services) && Date.now() < deadline) {
|
|
1870
|
+
await new Promise((r) => setTimeout(r, TOPOLOGY_EXPOSE_POLL_MS));
|
|
725
1871
|
services = await pool.execute('getExposedServices');
|
|
726
1872
|
}
|
|
727
|
-
if (!services
|
|
728
|
-
this.logger.warn({
|
|
1873
|
+
if (!usable(services)) {
|
|
1874
|
+
this.logger.warn({
|
|
1875
|
+
app: entry.name,
|
|
1876
|
+
process: procEntry.name,
|
|
1877
|
+
waitedMs: TOPOLOGY_EXPOSE_TIMEOUT_MS,
|
|
1878
|
+
reported: services?.map((svc) => `${svc.name}(${svc.methods?.length ?? 0})`) ?? [],
|
|
1879
|
+
}, 'No usable @Service found on pool worker — topology.expose has no effect, and any ' +
|
|
1880
|
+
'process declaring topology.access on it will start without a proxy');
|
|
729
1881
|
}
|
|
730
|
-
for (const svc of services) {
|
|
731
|
-
if (svc.methods.length === 0)
|
|
1882
|
+
for (const svc of services ?? []) {
|
|
1883
|
+
if (svc.methods.length === 0) {
|
|
1884
|
+
// Silently skipped before. A consumer then failed to find a
|
|
1885
|
+
// service its own declaration named, with nothing anywhere
|
|
1886
|
+
// connecting the two facts.
|
|
1887
|
+
this.logger.warn({ app: entry.name, process: procEntry.name, service: svc.name }, 'Pool service exposes no methods — not registered on the daemon');
|
|
732
1888
|
continue;
|
|
1889
|
+
}
|
|
733
1890
|
await serviceRouter.exposePoolService(procEntry.name, svc.name, svc.version ?? '1.0.0', pool, svc.methods);
|
|
734
1891
|
this.logger.info({ app: entry.name, process: procEntry.name, service: svc.name, methods: svc.methods.length }, 'Auto-discovered and exposed pool service on daemon Netron');
|
|
735
1892
|
}
|
|
@@ -748,7 +1905,10 @@ export class OrchestratorService extends EventEmitter {
|
|
|
748
1905
|
name: `${entry.name}/${procEntry.name}`,
|
|
749
1906
|
version: '1.0.0',
|
|
750
1907
|
allMethodsPublic: true,
|
|
751
|
-
|
|
1908
|
+
// Without this the child's client takes netron's 5000 ms; the pool
|
|
1909
|
+
// path beside it has always stated one. See CHILD_REQUEST_TIMEOUT.
|
|
1910
|
+
requestTimeout: CHILD_REQUEST_TIMEOUT,
|
|
1911
|
+
startupTimeout: procEntry.startupTimeout ?? entry.startupTimeout ?? DEFAULT_CHILD_STARTUP_TIMEOUT,
|
|
752
1912
|
...(entry.env && { env: entry.env }),
|
|
753
1913
|
...(entry.cwd && { cwd: entry.cwd }),
|
|
754
1914
|
dependencies: {
|
|
@@ -766,6 +1926,7 @@ export class OrchestratorService extends EventEmitter {
|
|
|
766
1926
|
timeout: procEntry.health?.timeout ?? config.monitoring.healthCheck.timeout,
|
|
767
1927
|
},
|
|
768
1928
|
observability: { metrics: true },
|
|
1929
|
+
...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
|
|
769
1930
|
};
|
|
770
1931
|
children.push({
|
|
771
1932
|
name: `${entry.name}/${procEntry.name}`,
|
|
@@ -781,25 +1942,20 @@ export class OrchestratorService extends EventEmitter {
|
|
|
781
1942
|
window: policy.window ?? config.supervision.window,
|
|
782
1943
|
backoff: policy.backoff ?? config.supervision.backoff,
|
|
783
1944
|
children,
|
|
784
|
-
onChildCrash
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
1945
|
+
// No custom onChildCrash — supervisor's built-in budget gate
|
|
1946
|
+
// (process-supervisor.ts:524-531) does what we want:
|
|
1947
|
+
// * within maxRestarts/window → RESTART
|
|
1948
|
+
// * over budget → ESCALATE (if child is critical) or IGNORE
|
|
1949
|
+
// Wiring an unconditional RESTART here masked the budget and
|
|
1950
|
+
// turned every crashing child into an infinite respawn storm.
|
|
789
1951
|
};
|
|
790
1952
|
// Create supervisor WITHOUT starting — wire events first to avoid race condition
|
|
791
1953
|
const supervisor = this.pm.createSupervisor(supervisorConfig);
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
const childPid = workerHandle?.pid ?? info?.pid;
|
|
798
|
-
handle.markOnline(childPid ?? process.pid);
|
|
799
|
-
}
|
|
800
|
-
this.persistState();
|
|
801
|
-
this.emit('app:online', entry.name);
|
|
802
|
-
});
|
|
1954
|
+
// The app's pid is its server's: the single process with a transport, the
|
|
1955
|
+
// one `handle.port` names. A topology with none has no pid to give.
|
|
1956
|
+
const server = singleEntries.find((p) => p.transports?.http) ?? singleEntries.find((p) => p.transports) ?? null;
|
|
1957
|
+
handle.declareChildren(children.map((child) => child.name), server ? `${entry.name}/${server.name}` : null);
|
|
1958
|
+
this.onChildStarted(entry, handle, supervisor);
|
|
803
1959
|
// Set supervisor on handle BEFORE start so attachLogCapture can access it
|
|
804
1960
|
handle.supervisor = supervisor;
|
|
805
1961
|
handle.serviceRouter = serviceRouter;
|
|
@@ -808,6 +1964,73 @@ export class OrchestratorService extends EventEmitter {
|
|
|
808
1964
|
this.wireSupervisorEvents(entry, handle, supervisor);
|
|
809
1965
|
// Now start — events will be captured correctly
|
|
810
1966
|
await supervisor.start();
|
|
1967
|
+
// Step 4: register the single processes that declare `topology.expose`.
|
|
1968
|
+
//
|
|
1969
|
+
// This was a pool-only feature, ignored in silence everywhere else: a
|
|
1970
|
+
// process declaring `expose: true` got no registration and no warning,
|
|
1971
|
+
// and every consumer naming it in `topology.access` started without a
|
|
1972
|
+
// proxy. Measured on priceverse, whose `collector` is a single process
|
|
1973
|
+
// holding the only objects that know whether the exchange WebSockets are
|
|
1974
|
+
// up — the server process could not ask, and answered `ready: down —
|
|
1975
|
+
// exchanges unavailable` while three were connected.
|
|
1976
|
+
//
|
|
1977
|
+
// After `supervisor.start()` because there is no child to ask before it.
|
|
1978
|
+
// A consumer among these children may therefore start before the service
|
|
1979
|
+
// is registered; consumers re-resolve their proxy on use for exactly that
|
|
1980
|
+
// reason (see priceverse's `OhlcvSchedulerService`).
|
|
1981
|
+
await this.exposeChildTopologyServices(entry, supervisor, serviceRouter, singleEntries);
|
|
1982
|
+
}
|
|
1983
|
+
/**
|
|
1984
|
+
* Register each single process's `@Service` classes on the daemon.
|
|
1985
|
+
*
|
|
1986
|
+
* Mirrors the pool path above, including its poll: a child reports ready
|
|
1987
|
+
* before its Application has started, and `@Service` classes are exposed to
|
|
1988
|
+
* Netron during start, so asking once asks a process that is still booting
|
|
1989
|
+
* and gets an empty list back.
|
|
1990
|
+
*/
|
|
1991
|
+
async exposeChildTopologyServices(entry, supervisor, serviceRouter, singleEntries) {
|
|
1992
|
+
for (const procEntry of singleEntries) {
|
|
1993
|
+
if (procEntry.topology?.expose !== true)
|
|
1994
|
+
continue;
|
|
1995
|
+
const childName = `${entry.name}/${procEntry.name}`;
|
|
1996
|
+
const getProxy = () => supervisor.getChildProxy(childName);
|
|
1997
|
+
try {
|
|
1998
|
+
const usable = (list) => Array.isArray(list) && list.length > 0 && list.some((svc) => svc.methods?.length > 0);
|
|
1999
|
+
const ask = async () => {
|
|
2000
|
+
const child = getProxy();
|
|
2001
|
+
if (!child?.getExposedServices)
|
|
2002
|
+
return undefined;
|
|
2003
|
+
return (await child.getExposedServices());
|
|
2004
|
+
};
|
|
2005
|
+
let services = await ask();
|
|
2006
|
+
const deadline = Date.now() + TOPOLOGY_EXPOSE_TIMEOUT_MS;
|
|
2007
|
+
while (!usable(services) && Date.now() < deadline) {
|
|
2008
|
+
await new Promise((r) => setTimeout(r, TOPOLOGY_EXPOSE_POLL_MS));
|
|
2009
|
+
services = await ask();
|
|
2010
|
+
}
|
|
2011
|
+
if (!usable(services)) {
|
|
2012
|
+
this.logger.warn({
|
|
2013
|
+
app: entry.name,
|
|
2014
|
+
process: procEntry.name,
|
|
2015
|
+
waitedMs: TOPOLOGY_EXPOSE_TIMEOUT_MS,
|
|
2016
|
+
reported: services?.map((svc) => `${svc.name}(${svc.methods?.length ?? 0})`) ?? [],
|
|
2017
|
+
}, 'No usable @Service found on this process — topology.expose has no effect, and any ' +
|
|
2018
|
+
'process declaring topology.access on it will start without a proxy');
|
|
2019
|
+
continue;
|
|
2020
|
+
}
|
|
2021
|
+
for (const svc of services ?? []) {
|
|
2022
|
+
if (svc.methods.length === 0) {
|
|
2023
|
+
this.logger.warn({ app: entry.name, process: procEntry.name, service: svc.name }, 'Process service exposes no methods — not registered on the daemon');
|
|
2024
|
+
continue;
|
|
2025
|
+
}
|
|
2026
|
+
await serviceRouter.exposeChildService(procEntry.name, svc.name, svc.version ?? '1.0.0', getProxy, svc.methods);
|
|
2027
|
+
this.logger.info({ app: entry.name, process: procEntry.name, service: svc.name, methods: svc.methods.length }, 'Auto-discovered and exposed process service on daemon Netron');
|
|
2028
|
+
}
|
|
2029
|
+
}
|
|
2030
|
+
catch (err) {
|
|
2031
|
+
this.logger.error({ app: entry.name, process: procEntry.name, error: err.message }, "Failed to auto-discover/expose this process's services");
|
|
2032
|
+
}
|
|
2033
|
+
}
|
|
811
2034
|
}
|
|
812
2035
|
/**
|
|
813
2036
|
* Launch single-process bootstrap app (no topology — original behavior).
|
|
@@ -823,11 +2046,22 @@ export class OrchestratorService extends EventEmitter {
|
|
|
823
2046
|
const spawnOpts = {
|
|
824
2047
|
name: entry.name,
|
|
825
2048
|
version: '1.0.0',
|
|
2049
|
+
// As above: a single child's client otherwise runs on netron's default.
|
|
2050
|
+
requestTimeout: CHILD_REQUEST_TIMEOUT,
|
|
826
2051
|
...(entry.env && { env: entry.env }),
|
|
827
2052
|
dependencies: {
|
|
828
2053
|
bootstrapPath: bootstrapAbsPath,
|
|
829
2054
|
...(bundledModulePath && { bundledModulePath }),
|
|
830
2055
|
},
|
|
2056
|
+
// Omitted when unset, which delegates the budget to titan-pm —
|
|
2057
|
+
// `process-spawner.ts`: `options.startupTimeout ?? resources?.timeout ??
|
|
2058
|
+
// 30_000`. Worth naming, because it is otherwise invisible from here:
|
|
2059
|
+
// an APP that declares nothing gets 30s while its CHILD processes get
|
|
2060
|
+
// `DEFAULT_CHILD_STARTUP_TIMEOUT`, 60s. Backwards if anything, since the
|
|
2061
|
+
// app usually does the heavier work — and the downstream project's own
|
|
2062
|
+
// config carries a note about a 30s default that kept a whole stack from
|
|
2063
|
+
// starting. Left as delegation rather than changed here: raising it is a
|
|
2064
|
+
// decision about titan-pm's contract, not about this call site.
|
|
831
2065
|
...(entry.startupTimeout != null && { startupTimeout: entry.startupTimeout }),
|
|
832
2066
|
health: {
|
|
833
2067
|
enabled: true,
|
|
@@ -835,6 +2069,7 @@ export class OrchestratorService extends EventEmitter {
|
|
|
835
2069
|
timeout: config.monitoring.healthCheck.timeout,
|
|
836
2070
|
},
|
|
837
2071
|
observability: { metrics: true },
|
|
2072
|
+
...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
|
|
838
2073
|
};
|
|
839
2074
|
const childConfig = {
|
|
840
2075
|
name: entry.name,
|
|
@@ -847,7 +2082,8 @@ export class OrchestratorService extends EventEmitter {
|
|
|
847
2082
|
size: instances,
|
|
848
2083
|
strategy: PoolStrategy.ROUND_ROBIN,
|
|
849
2084
|
metrics: true,
|
|
850
|
-
|
|
2085
|
+
// See the note on the other pool: `replaceUnhealthy` is declared by
|
|
2086
|
+
// titan-pm and read by nothing.
|
|
851
2087
|
healthCheck: {
|
|
852
2088
|
enabled: true,
|
|
853
2089
|
interval: config.monitoring.healthCheck.interval,
|
|
@@ -876,27 +2112,19 @@ export class OrchestratorService extends EventEmitter {
|
|
|
876
2112
|
window: policy.window ?? config.supervision.window,
|
|
877
2113
|
backoff: policy.backoff ?? config.supervision.backoff,
|
|
878
2114
|
children: [childConfig],
|
|
879
|
-
onChildCrash
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
2115
|
+
// No custom onChildCrash — supervisor's built-in budget gate
|
|
2116
|
+
// (process-supervisor.ts:524-531) does what we want:
|
|
2117
|
+
// * within maxRestarts/window → RESTART
|
|
2118
|
+
// * over budget → ESCALATE (if child is critical) or IGNORE
|
|
2119
|
+
// Wiring an unconditional RESTART here masked the budget and
|
|
2120
|
+
// turned every crashing child into an infinite respawn storm.
|
|
884
2121
|
};
|
|
885
2122
|
// Create supervisor WITHOUT starting — wire events first to avoid race condition
|
|
886
2123
|
const supervisor = this.pm.createSupervisor(supervisorConfig);
|
|
887
|
-
// Wire supervisor events BEFORE start so we catch the initial child:started
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
// Prefer WorkerHandle.pid (actual child OS PID), fallback to IProcessInfo.pid
|
|
892
|
-
const workerHandle = this.pm.getWorkerHandle(processId);
|
|
893
|
-
const info = this.pm.getProcess(processId);
|
|
894
|
-
const childPid = workerHandle?.pid ?? info?.pid;
|
|
895
|
-
handle.markOnline(childPid ?? process.pid);
|
|
896
|
-
}
|
|
897
|
-
this.persistState();
|
|
898
|
-
this.emit('app:online', entry.name);
|
|
899
|
-
});
|
|
2124
|
+
// Wire supervisor events BEFORE start so we catch the initial child:started.
|
|
2125
|
+
// One child, and it is the whole app — so it is the server too.
|
|
2126
|
+
handle.declareChildren([childConfig.name], childConfig.name);
|
|
2127
|
+
this.onChildStarted(entry, handle, supervisor);
|
|
900
2128
|
// Set supervisor on handle BEFORE start so attachLogCapture can access it
|
|
901
2129
|
handle.supervisor = supervisor;
|
|
902
2130
|
handle.instanceCount = instances;
|
|
@@ -904,14 +2132,50 @@ export class OrchestratorService extends EventEmitter {
|
|
|
904
2132
|
// Now start — events will be captured correctly
|
|
905
2133
|
await supervisor.start();
|
|
906
2134
|
}
|
|
2135
|
+
/**
|
|
2136
|
+
* A child's start, for both launch paths: the app is online when the start
|
|
2137
|
+
* completes it, and not before (`AppHandle.childStarted`).
|
|
2138
|
+
*
|
|
2139
|
+
* Each path had its own copy of this handler, and both marked the app
|
|
2140
|
+
* online — with that child's pid — on every child's start. `app:online`
|
|
2141
|
+
* went out once per child with it: three STARTED events for one boot of a
|
|
2142
|
+
* three-process app, the first two about an app still coming up.
|
|
2143
|
+
*/
|
|
2144
|
+
onChildStarted(entry, handle, supervisor) {
|
|
2145
|
+
supervisor.on('child:started', (childName) => {
|
|
2146
|
+
// A pool child's proxy answers every property with a function —
|
|
2147
|
+
// `__processId` included — so only a string is a process id. The old
|
|
2148
|
+
// handler took that function for one, found no worker behind it, and
|
|
2149
|
+
// gave such an app the DAEMON's pid.
|
|
2150
|
+
const processId = supervisor.getChildProcessId(childName);
|
|
2151
|
+
const pid = typeof processId === 'string'
|
|
2152
|
+
? (this.pm.getWorkerHandle(processId)?.pid ?? this.pm.getProcess(processId)?.pid ?? null)
|
|
2153
|
+
: null;
|
|
2154
|
+
const whole = handle.childStarted(childName, pid);
|
|
2155
|
+
this.persistState();
|
|
2156
|
+
if (!whole)
|
|
2157
|
+
return;
|
|
2158
|
+
this.startPostOnlineCooldown(entry.name);
|
|
2159
|
+
this.emit('app:online', entry.name);
|
|
2160
|
+
});
|
|
2161
|
+
}
|
|
907
2162
|
/**
|
|
908
2163
|
* Wire common supervisor events (crash, restart, escalate) and log capture.
|
|
909
2164
|
*/
|
|
910
2165
|
wireSupervisorEvents(entry, handle, supervisor) {
|
|
911
2166
|
supervisor.on('child:started', (childName) => {
|
|
912
2167
|
this.attachLogCapture(entry.name, childName, handle);
|
|
2168
|
+
this.attachExitCapture(entry.name, childName, handle);
|
|
2169
|
+
// The app re-entered a live state — drop the previous crash
|
|
2170
|
+
// context so a future `omnitron inspect` shows the *next*
|
|
2171
|
+
// crash, not the last one.
|
|
2172
|
+
handle.clearLastExit();
|
|
913
2173
|
});
|
|
2174
|
+
// A child that is down leaves the app partial until it is back — a
|
|
2175
|
+
// sibling's restart must not bring the app `online` around it.
|
|
2176
|
+
supervisor.on('child:stopped', (childName) => handle.childGone(childName));
|
|
914
2177
|
supervisor.on('child:crash', (childName, error) => {
|
|
2178
|
+
handle.childGone(childName);
|
|
915
2179
|
// If app never reached 'online', mark as errored (not crashed — it never ran)
|
|
916
2180
|
if (handle.status === 'starting') {
|
|
917
2181
|
handle.markErrored();
|
|
@@ -919,10 +2183,94 @@ export class OrchestratorService extends EventEmitter {
|
|
|
919
2183
|
else {
|
|
920
2184
|
handle.markCrashed();
|
|
921
2185
|
}
|
|
2186
|
+
// Stash the supervisor-provided error message onto whatever
|
|
2187
|
+
// exit context the workerHandle.onExit hook already wrote. If
|
|
2188
|
+
// the exit event hasn't fired yet (race) we still need to
|
|
2189
|
+
// persist *something* — record a synthetic entry that the
|
|
2190
|
+
// exit hook below will overwrite when it does fire.
|
|
2191
|
+
if (handle.lastExit) {
|
|
2192
|
+
handle.lastExit.message = error.message;
|
|
2193
|
+
}
|
|
2194
|
+
else {
|
|
2195
|
+
handle.recordExit({ code: null, signal: null, expected: false, message: error.message });
|
|
2196
|
+
}
|
|
922
2197
|
this.logger.error({ app: entry.name, child: childName, error: error.message }, 'Process crashed');
|
|
923
2198
|
this.persistState();
|
|
924
2199
|
this.emit('app:crash', entry.name, error);
|
|
925
2200
|
});
|
|
2201
|
+
// T#60 — startup failures (child throws/rejects in waitForReady
|
|
2202
|
+
// before ever reporting ready) come through this event, NOT
|
|
2203
|
+
// child:crash. Without a handler the orchestrator just sees
|
|
2204
|
+
// status='errored' with no clue what went wrong. titan-pm
|
|
2205
|
+
// already stuffs the captured stderr into `error.details.stderr`
|
|
2206
|
+
// so we surface it on `lastExit.stderrTail`.
|
|
2207
|
+
supervisor.on('child:start-failed', (childName, error) => {
|
|
2208
|
+
const details = error.details;
|
|
2209
|
+
const stderrTail = typeof details?.stderr === 'string'
|
|
2210
|
+
? details.stderr.split('\n').filter((l) => l.length > 0).slice(-80)
|
|
2211
|
+
: [];
|
|
2212
|
+
for (const line of stderrTail)
|
|
2213
|
+
handle.appendStderr(line);
|
|
2214
|
+
// A child that never came up because WE stopped the app is not a
|
|
2215
|
+
// failure, and reporting it as one is worse than saying nothing: the
|
|
2216
|
+
// message titan-pm builds for an aborted start says the process "did not
|
|
2217
|
+
// choose to exit, so its own logs will not explain this" — true, and it
|
|
2218
|
+
// sends the reader hunting a fault in an application that was doing
|
|
2219
|
+
// nothing wrong. Every `stack stop` issued while anything was still
|
|
2220
|
+
// booting wrote one of these per child, at error level, into the app's
|
|
2221
|
+
// own log.
|
|
2222
|
+
//
|
|
2223
|
+
// `'stopping'` only — NOT `'stopped'`, which is also an AppHandle's
|
|
2224
|
+
// initial value, so accepting it would swallow the first genuine failure
|
|
2225
|
+
// of any app whose handle had not yet been marked starting.
|
|
2226
|
+
const expected = handle.status === 'stopping' ||
|
|
2227
|
+
error.stoppedDuringStartup === true;
|
|
2228
|
+
if (expected) {
|
|
2229
|
+
handle.recordExit({
|
|
2230
|
+
code: null,
|
|
2231
|
+
signal: null,
|
|
2232
|
+
expected: true,
|
|
2233
|
+
message: error.message,
|
|
2234
|
+
});
|
|
2235
|
+
this.logger.info({ app: entry.name, child: childName, err: error.message }, 'Process stopped before it finished starting');
|
|
2236
|
+
this.persistState();
|
|
2237
|
+
return;
|
|
2238
|
+
}
|
|
2239
|
+
// Put what the child printed into the app's LOG FILE, not only into the
|
|
2240
|
+
// in-memory ring `omnitron inspect` reads.
|
|
2241
|
+
//
|
|
2242
|
+
// A child that dies during startup never produces a WorkerHandle at all
|
|
2243
|
+
// — titan-pm registers one only after `waitForReady` resolves — so there
|
|
2244
|
+
// is no `onLog` to subscribe to here and nothing to replay. What does
|
|
2245
|
+
// exist is the output `waitForReady` captured while waiting, which it
|
|
2246
|
+
// attaches to the error. Feeding that to the log handlers is the only
|
|
2247
|
+
// way a failed boot reaches disk, and a failed boot is the case where
|
|
2248
|
+
// reading it matters most: an application that dies before its logger
|
|
2249
|
+
// exists has said everything it is ever going to say on these streams.
|
|
2250
|
+
// The failure itself first. A child can fail without printing a word —
|
|
2251
|
+
// it reports the error over the IPC channel and exits, which is what a
|
|
2252
|
+
// module that throws on import does — so the streams below are often
|
|
2253
|
+
// empty and the only account of what happened is on the error.
|
|
2254
|
+
this.persistChildFailure(entry.name, childName, error);
|
|
2255
|
+
this.persistChildOutput(entry.name, childName, details?.stdout, 'stdout');
|
|
2256
|
+
this.persistChildOutput(entry.name, childName, details?.stderr, 'stderr');
|
|
2257
|
+
handle.recordExit({
|
|
2258
|
+
code: null,
|
|
2259
|
+
signal: null,
|
|
2260
|
+
expected: false,
|
|
2261
|
+
message: error.message,
|
|
2262
|
+
});
|
|
2263
|
+
// The supervisor gives up on a non-critical child whose start failed (a
|
|
2264
|
+
// critical one throws, and `startApp` says so). That went unseen while
|
|
2265
|
+
// the first child's start made the app `online`; now the app would wait
|
|
2266
|
+
// for a child that is not coming, in `starting` — which `omnitron start`
|
|
2267
|
+
// takes for a start in progress and leaves alone. A declared process
|
|
2268
|
+
// that did not start is an app that did not start.
|
|
2269
|
+
if (handle.status === 'starting')
|
|
2270
|
+
handle.markErrored();
|
|
2271
|
+
this.logger.error({ app: entry.name, child: childName, err: error.message }, 'Process failed to start');
|
|
2272
|
+
this.persistState();
|
|
2273
|
+
});
|
|
926
2274
|
supervisor.on('child:restart', (childName, attempt) => {
|
|
927
2275
|
handle.restarts = attempt;
|
|
928
2276
|
this.logger.info({ app: entry.name, child: childName, attempt }, 'Process restarting');
|
|
@@ -949,9 +2297,45 @@ export class OrchestratorService extends EventEmitter {
|
|
|
949
2297
|
const workerHandle = this.pm.getWorkerHandle(processId);
|
|
950
2298
|
if (!workerHandle?.onLog)
|
|
951
2299
|
return;
|
|
952
|
-
workerHandle.onLog((
|
|
2300
|
+
workerHandle.onLog(this.childLogForwarder(appName, handle));
|
|
2301
|
+
}
|
|
2302
|
+
/**
|
|
2303
|
+
* Attach log capture to the workers of a topology POOL.
|
|
2304
|
+
*
|
|
2305
|
+
* `attachLogCapture` above is driven by `child:started`, which only
|
|
2306
|
+
* supervisor children emit — a pool worker is not one, so nothing ever
|
|
2307
|
+
* subscribed to its output. Everything those workers print went nowhere:
|
|
2308
|
+
* the OHLCV aggregator and the image transformer have been mute since they
|
|
2309
|
+
* were written, and since titan-pm started holding pre-capture output it was
|
|
2310
|
+
* also a megabyte per worker retained for a reader that never arrived.
|
|
2311
|
+
*
|
|
2312
|
+
* Attached once, at pool creation. A worker replaced later by autoscaling or
|
|
2313
|
+
* a health restart is not covered; these pools are fixed-size, and the
|
|
2314
|
+
* alternative is an event this package does not expose yet.
|
|
2315
|
+
*/
|
|
2316
|
+
attachPoolLogCapture(appName, processName, handle, pool) {
|
|
2317
|
+
const forward = this.childLogForwarder(appName, handle);
|
|
2318
|
+
let attached = 0;
|
|
2319
|
+
for (const workerId of pool.getWorkerIds()) {
|
|
2320
|
+
const workerHandle = pool.getWorkerHandle(workerId);
|
|
2321
|
+
if (!workerHandle?.onLog)
|
|
2322
|
+
continue;
|
|
2323
|
+
workerHandle.onLog(forward);
|
|
2324
|
+
attached++;
|
|
2325
|
+
}
|
|
2326
|
+
this.logger.debug({ app: appName, process: processName, workers: attached }, 'Pool log capture attached');
|
|
2327
|
+
}
|
|
2328
|
+
/** The one place a child's line becomes an app's line. */
|
|
2329
|
+
childLogForwarder(appName, handle) {
|
|
2330
|
+
return (line, stream) => {
|
|
953
2331
|
// In-memory ring buffer (for live CLI tailing)
|
|
954
2332
|
handle.appendLog(line);
|
|
2333
|
+
// Separate stderr ring — feeds `lastExit.stderrTail` on death.
|
|
2334
|
+
// Without this, the operator running `omnitron inspect` after
|
|
2335
|
+
// a crash sees no hint at WHY (the in-flight pipe is gone).
|
|
2336
|
+
if (stream === 'stderr') {
|
|
2337
|
+
handle.appendStderr(line);
|
|
2338
|
+
}
|
|
955
2339
|
// Persist to disk via registered log handlers (LogManager)
|
|
956
2340
|
for (const handler of this.appLogHandlers) {
|
|
957
2341
|
try {
|
|
@@ -961,6 +2345,108 @@ export class OrchestratorService extends EventEmitter {
|
|
|
961
2345
|
// Log handler failure must not break the app
|
|
962
2346
|
}
|
|
963
2347
|
}
|
|
2348
|
+
};
|
|
2349
|
+
}
|
|
2350
|
+
/**
|
|
2351
|
+
* Write the failure a child reported before it could start to the app's log
|
|
2352
|
+
* handlers, as one structured line.
|
|
2353
|
+
*
|
|
2354
|
+
* `error.childStack` is the stack from inside the child when it managed to
|
|
2355
|
+
* send one — for a module that throws on import that is the entire
|
|
2356
|
+
* diagnosis, and it exists nowhere else once the process is gone.
|
|
2357
|
+
*/
|
|
2358
|
+
persistChildFailure(appName, childName, error) {
|
|
2359
|
+
const childStack = error.childStack;
|
|
2360
|
+
const entry = JSON.stringify({
|
|
2361
|
+
level: 50,
|
|
2362
|
+
time: new Date().toISOString(),
|
|
2363
|
+
processName: qualifyChildName(appName, childName),
|
|
2364
|
+
event: 'process.start_failed',
|
|
2365
|
+
err: { message: error.message, ...(childStack ? { stack: childStack } : {}) },
|
|
2366
|
+
msg: `Process failed to start: ${error.message}`,
|
|
2367
|
+
});
|
|
2368
|
+
for (const handler of this.appLogHandlers) {
|
|
2369
|
+
try {
|
|
2370
|
+
handler(appName, entry);
|
|
2371
|
+
}
|
|
2372
|
+
catch {
|
|
2373
|
+
// A log handler must not break failure reporting.
|
|
2374
|
+
}
|
|
2375
|
+
}
|
|
2376
|
+
}
|
|
2377
|
+
/**
|
|
2378
|
+
* Write a dead child's captured output to the app's log handlers.
|
|
2379
|
+
*
|
|
2380
|
+
* Emitted as pino-shaped JSON so it lands in the same file, at the same
|
|
2381
|
+
* level, as everything else that app ever logged — a reader tailing
|
|
2382
|
+
* `app.log` after a failed start finds the reason there rather than having
|
|
2383
|
+
* to know that this particular failure hides in a different place.
|
|
2384
|
+
*/
|
|
2385
|
+
persistChildOutput(appName, childName, output, stream) {
|
|
2386
|
+
if (!output)
|
|
2387
|
+
return;
|
|
2388
|
+
const time = new Date().toISOString();
|
|
2389
|
+
for (const line of output.split('\n')) {
|
|
2390
|
+
if (!line.trim())
|
|
2391
|
+
continue;
|
|
2392
|
+
// Already-structured lines pass through untouched; the child may well
|
|
2393
|
+
// have got far enough to produce them.
|
|
2394
|
+
const entry = line.trimStart().startsWith('{')
|
|
2395
|
+
? line
|
|
2396
|
+
: JSON.stringify({
|
|
2397
|
+
// stderr is not error. A boot script's own progress markers go
|
|
2398
|
+
// there, and a failing start emits a run of them — classifying
|
|
2399
|
+
// every one at 50 filled the log with errors that are nothing of
|
|
2400
|
+
// the sort, next to the single line that IS the failure. The
|
|
2401
|
+
// failure is reported at 50 by `persistChildFailure`; this is the
|
|
2402
|
+
// context around it, and `stream` keeps it greppable.
|
|
2403
|
+
level: stream === 'stderr' ? 40 : 30,
|
|
2404
|
+
time,
|
|
2405
|
+
processName: qualifyChildName(appName, childName),
|
|
2406
|
+
stream,
|
|
2407
|
+
msg: line,
|
|
2408
|
+
});
|
|
2409
|
+
for (const handler of this.appLogHandlers) {
|
|
2410
|
+
try {
|
|
2411
|
+
handler(appName, entry);
|
|
2412
|
+
}
|
|
2413
|
+
catch {
|
|
2414
|
+
// A log handler must not break failure reporting.
|
|
2415
|
+
}
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
/**
|
|
2420
|
+
* Subscribe to the WorkerHandle's `exit` event so the orchestrator
|
|
2421
|
+
* keeps the exit code / signal / "expected" flag past the
|
|
2422
|
+
* WorkerHandle's GC. Pre-fix the supervisor's `child:crash` event
|
|
2423
|
+
* fired with an Error but never surfaced the underlying
|
|
2424
|
+
* code/signal — operators had no way to tell SIGKILL from a
|
|
2425
|
+
* thrown TypeError. The captured info lands on
|
|
2426
|
+
* `handle.lastExit` and is surfaced by `omnitron inspect`.
|
|
2427
|
+
*/
|
|
2428
|
+
attachExitCapture(appName, childName, handle) {
|
|
2429
|
+
const supervisor = handle.supervisor;
|
|
2430
|
+
if (!supervisor)
|
|
2431
|
+
return;
|
|
2432
|
+
const processId = supervisor.getChildProcessId(childName);
|
|
2433
|
+
if (!processId)
|
|
2434
|
+
return;
|
|
2435
|
+
const workerHandle = this.pm.getWorkerHandle(processId);
|
|
2436
|
+
if (!workerHandle?.onExit)
|
|
2437
|
+
return;
|
|
2438
|
+
workerHandle.onExit((info) => {
|
|
2439
|
+
// Preserve the supervisor's error message if `child:crash`
|
|
2440
|
+
// already fired before us. recordExit overwrites the synthetic
|
|
2441
|
+
// entry with the authoritative code/signal.
|
|
2442
|
+
const message = handle.lastExit?.message;
|
|
2443
|
+
handle.recordExit({
|
|
2444
|
+
code: info.code,
|
|
2445
|
+
signal: info.signal,
|
|
2446
|
+
expected: info.expected,
|
|
2447
|
+
...(message !== undefined && { message }),
|
|
2448
|
+
});
|
|
2449
|
+
this.logger.debug({ app: appName, child: childName, code: info.code, signal: info.signal, expected: info.expected }, 'Captured worker exit context');
|
|
964
2450
|
});
|
|
965
2451
|
}
|
|
966
2452
|
// ============================================================================
|
|
@@ -970,30 +2456,58 @@ export class OrchestratorService extends EventEmitter {
|
|
|
970
2456
|
const child = handle.childProcess;
|
|
971
2457
|
if (!child || !child.pid)
|
|
972
2458
|
return;
|
|
2459
|
+
const pid = child.pid;
|
|
2460
|
+
// P0-D — Signal the whole process group, not just the direct
|
|
2461
|
+
// child. classic-launcher spawns with `detached: true` so the
|
|
2462
|
+
// child becomes group leader (pgid === pid). `process.kill(-pid)`
|
|
2463
|
+
// delivers the signal to every process in the group, reaping
|
|
2464
|
+
// grandchildren (ffmpeg, docker, worker pools …) that ignore the
|
|
2465
|
+
// direct-child kill. Falls back to `child.kill` when the negative-
|
|
2466
|
+
// pid call fails (test environments, restricted permissions, the
|
|
2467
|
+
// child wasn't started detached — bootstrap-mode currently isn't).
|
|
2468
|
+
const sendSignal = (sig) => {
|
|
2469
|
+
try {
|
|
2470
|
+
process.kill(-pid, sig);
|
|
2471
|
+
return;
|
|
2472
|
+
}
|
|
2473
|
+
catch (err) {
|
|
2474
|
+
const code = err.code;
|
|
2475
|
+
if (code !== 'ESRCH' && code !== 'EPERM') {
|
|
2476
|
+
// ESRCH/EPERM = group already gone or not a group leader.
|
|
2477
|
+
// Anything else (EINVAL etc.) → log via best-effort.
|
|
2478
|
+
this.logger.debug({ app: handle.name, pid, sig, code }, 'process.kill(-pid) fallback');
|
|
2479
|
+
}
|
|
2480
|
+
}
|
|
2481
|
+
try {
|
|
2482
|
+
child.kill(sig);
|
|
2483
|
+
}
|
|
2484
|
+
catch { /* best-effort */ }
|
|
2485
|
+
};
|
|
973
2486
|
await new Promise((resolve) => {
|
|
974
2487
|
const timer = setTimeout(() => {
|
|
975
|
-
|
|
976
|
-
child.kill('SIGKILL');
|
|
977
|
-
}
|
|
978
|
-
catch {
|
|
979
|
-
// SIGKILL send is best-effort
|
|
980
|
-
}
|
|
2488
|
+
sendSignal('SIGKILL');
|
|
981
2489
|
resolve();
|
|
982
2490
|
}, timeout);
|
|
983
2491
|
child.once('exit', () => {
|
|
984
2492
|
clearTimeout(timer);
|
|
985
2493
|
resolve();
|
|
986
2494
|
});
|
|
987
|
-
|
|
988
|
-
child.kill(force ? 'SIGKILL' : 'SIGTERM');
|
|
989
|
-
}
|
|
990
|
-
catch {
|
|
991
|
-
clearTimeout(timer);
|
|
992
|
-
resolve();
|
|
993
|
-
}
|
|
2495
|
+
sendSignal(force ? 'SIGKILL' : 'SIGTERM');
|
|
994
2496
|
});
|
|
995
2497
|
}
|
|
996
|
-
|
|
2498
|
+
/**
|
|
2499
|
+
* Cancel a pending crash cool-down for `name`, if one is armed.
|
|
2500
|
+
* An explicit stop means "this app is going down now, stay down",
|
|
2501
|
+
* which the cool-down must not undo ten minutes later.
|
|
2502
|
+
*/
|
|
2503
|
+
clearCoolDown(name) {
|
|
2504
|
+
const timer = this.coolDownTimers.get(name);
|
|
2505
|
+
if (timer) {
|
|
2506
|
+
clearTimeout(timer);
|
|
2507
|
+
this.coolDownTimers.delete(name);
|
|
2508
|
+
}
|
|
2509
|
+
}
|
|
2510
|
+
async scheduleRecovery(entry, handle, config) {
|
|
997
2511
|
const policy = buildRestartPolicy(entry, config);
|
|
998
2512
|
if (!policy.enabled)
|
|
999
2513
|
return;
|
|
@@ -1001,87 +2515,416 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1001
2515
|
handle.status = 'crashed';
|
|
1002
2516
|
this.logger.error({ app: entry.name }, 'Max restarts exceeded');
|
|
1003
2517
|
this.emit('app:escalated', entry.name);
|
|
2518
|
+
// P1-I — auto-recovery after a cool-down. pm2 has this as
|
|
2519
|
+
// `restart_delay` + a fresh attempt window; pre-fix omnitron
|
|
2520
|
+
// left the app permanently 'crashed' until human intervention.
|
|
2521
|
+
// We reset the counter + schedule one fresh attempt 10 minutes
|
|
2522
|
+
// out. If THAT attempt also crashes through maxRestarts we
|
|
2523
|
+
// land back here and pause another 10 minutes — bounded
|
|
2524
|
+
// self-healing without thrashing.
|
|
2525
|
+
//
|
|
2526
|
+
// The wait happens OUTSIDE the lifecycle queue, and that is the
|
|
2527
|
+
// whole point. It used to be a queued `crash-restart` whose
|
|
2528
|
+
// `run()` slept for the entire cool-down — but `enqueue` will
|
|
2529
|
+
// not preempt a RUNNING head, and with an empty queue the
|
|
2530
|
+
// sleeper always became the head immediately. So the `onPreempt`
|
|
2531
|
+
// hook meant to cancel it could never fire, and an operator's
|
|
2532
|
+
// `stop` or `restart` simply queued behind ten minutes of sleep:
|
|
2533
|
+
// the CLI's RPC timed out after 60s while the op stayed queued,
|
|
2534
|
+
// to fire whenever the cool-down finally elapsed. An app that
|
|
2535
|
+
// exhausted its restart budget was unreachable by every command
|
|
2536
|
+
// the CLI offers — `restart` hung, `stop` hung, `stack start`
|
|
2537
|
+
// skipped it — and the only recovery was restarting the daemon.
|
|
2538
|
+
const COOL_DOWN_MS = 10 * 60_000;
|
|
2539
|
+
this.clearCoolDown(entry.name);
|
|
2540
|
+
const timer = setTimeout(() => {
|
|
2541
|
+
this.coolDownTimers.delete(entry.name);
|
|
2542
|
+
// Re-read the handle: every start replaces it (see
|
|
2543
|
+
// `startAppInternal`), so the one captured above is stale the
|
|
2544
|
+
// moment anything restarts this app. Only a still-crashed app
|
|
2545
|
+
// wants this attempt — an operator who started it meanwhile
|
|
2546
|
+
// has already settled the question.
|
|
2547
|
+
const current = this.handles.get(entry.name);
|
|
2548
|
+
if (!current || current.status !== 'crashed')
|
|
2549
|
+
return;
|
|
2550
|
+
this.logger.info({ app: entry.name }, 'Cool-down elapsed — attempting fresh recovery');
|
|
2551
|
+
current.restarts = 0; // fresh attempt window
|
|
2552
|
+
current.status = 'stopped';
|
|
2553
|
+
void this.lifecycleQueue
|
|
2554
|
+
.enqueue(entry.name, 'crash-restart', () => this.startAppInternal(entry, config))
|
|
2555
|
+
.catch((err) => {
|
|
2556
|
+
if (err instanceof LifecyclePreempted)
|
|
2557
|
+
return;
|
|
2558
|
+
this.logger.error({ app: entry.name, err: err.message }, 'Cool-down recovery failed');
|
|
2559
|
+
});
|
|
2560
|
+
}, COOL_DOWN_MS);
|
|
2561
|
+
timer.unref?.();
|
|
2562
|
+
this.coolDownTimers.set(entry.name, timer);
|
|
1004
2563
|
return;
|
|
1005
2564
|
}
|
|
1006
2565
|
const backoff = policy.backoff ?? config.supervision.backoff;
|
|
1007
2566
|
const delay = this.calculateBackoff(handle.restarts, backoff);
|
|
1008
2567
|
this.logger.info({ app: entry.name, delay, attempt: handle.restarts + 1 }, 'Scheduling restart');
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
2568
|
+
// P0-I — schedule the crash-restart through the lifecycle queue
|
|
2569
|
+
// so an explicit stop preempts the pending restart before it
|
|
2570
|
+
// fires. The setTimeout is still the timing source, but the
|
|
2571
|
+
// queue.enqueue with `onPreempt` guarantees the timer is cleared
|
|
2572
|
+
// (and the promise rejected with LifecyclePreempted) the moment a
|
|
2573
|
+
// higher-priority op for the same app arrives. Pre-fix the timer
|
|
2574
|
+
// could fire during stopApp's `await stopChild(...)` window and
|
|
2575
|
+
// spawn a zombie under the same handle.
|
|
2576
|
+
let timer = null;
|
|
2577
|
+
const restartPromise = this.lifecycleQueue.enqueue(entry.name, 'crash-restart', () => new Promise((resolve, reject) => {
|
|
2578
|
+
timer = setTimeout(async () => {
|
|
2579
|
+
timer = null;
|
|
2580
|
+
// Guard: don't restart if orchestrator was stopped during backoff
|
|
2581
|
+
if (handle.status === 'stopping' || handle.status === 'stopped') {
|
|
2582
|
+
resolve();
|
|
2583
|
+
return;
|
|
2584
|
+
}
|
|
2585
|
+
try {
|
|
2586
|
+
handle.status = 'stopped';
|
|
2587
|
+
// `startAppInternal`, not `startApp`.
|
|
2588
|
+
//
|
|
2589
|
+
// This task holds the lifecycle-queue slot for `entry.name`.
|
|
2590
|
+
// `startApp` enqueues 'start' on that same key, and the queue
|
|
2591
|
+
// cannot run it until this task returns — so the await never
|
|
2592
|
+
// settles and the restart silently never happens. Measured:
|
|
2593
|
+
// `startApp:enter` with no matching exit, and the app left in
|
|
2594
|
+
// whatever state the crash put it in.
|
|
2595
|
+
//
|
|
2596
|
+
// This is why the whole policy below — bounded attempts,
|
|
2597
|
+
// exponential backoff, the ten-minute cool-down described in its
|
|
2598
|
+
// own comment as "bounded self-healing" — had never restarted
|
|
2599
|
+
// anything. The queue was introduced (P0-I) to stop a restart
|
|
2600
|
+
// timer racing a stop; it also made the restart unreachable, and
|
|
2601
|
+
// nothing failed loudly enough to say so.
|
|
2602
|
+
await this.startAppInternal(entry, config);
|
|
2603
|
+
resolve();
|
|
2604
|
+
}
|
|
2605
|
+
catch (err) {
|
|
2606
|
+
const message = err.message;
|
|
2607
|
+
this.logger.error({ app: entry.name, error: message }, 'Restart failed');
|
|
2608
|
+
// Said separately and after, because reading the process table
|
|
2609
|
+
// takes a moment and the failure itself must not wait for it.
|
|
2610
|
+
void this.explainPortConflict(message).then((held) => {
|
|
2611
|
+
if (held)
|
|
2612
|
+
this.logger.error({ app: entry.name }, `Restart failed — ${held}`);
|
|
2613
|
+
});
|
|
2614
|
+
reject(err);
|
|
2615
|
+
}
|
|
2616
|
+
}, delay);
|
|
2617
|
+
}), () => {
|
|
2618
|
+
// onPreempt — explicit stop / restart arrived. Cancel the
|
|
2619
|
+
// pending timer so the restart never fires.
|
|
2620
|
+
if (timer) {
|
|
2621
|
+
clearTimeout(timer);
|
|
2622
|
+
timer = null;
|
|
1019
2623
|
}
|
|
1020
|
-
}
|
|
1021
|
-
//
|
|
2624
|
+
});
|
|
2625
|
+
// Swallow LifecyclePreempted — that's the explicit-stop happy
|
|
2626
|
+
// path, not an error. Anything else gets logged.
|
|
2627
|
+
restartPromise.catch((err) => {
|
|
2628
|
+
if (err instanceof LifecyclePreempted)
|
|
2629
|
+
return;
|
|
2630
|
+
this.logger.warn({ app: entry.name, err: err.message }, 'Crash restart task rejected');
|
|
2631
|
+
});
|
|
2632
|
+
// Legacy field — callers still poke handle.crashRestartTimer to
|
|
2633
|
+
// check "is a restart pending"; keep it in sync with the queued
|
|
2634
|
+
// timer so existing inspect / status code paths don't break.
|
|
1022
2635
|
handle.crashRestartTimer = timer;
|
|
1023
2636
|
}
|
|
1024
2637
|
calculateBackoff(attempt, backoff) {
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
2638
|
+
// Delegate to the shared backoff library in titan-pm. Pre-fix the
|
|
2639
|
+
// inline impl here defaulted `initial=1000` while supervisor's
|
|
2640
|
+
// copy defaulted to 300 — same config produced 3.3× longer
|
|
2641
|
+
// delays in classic-mode than bootstrap-mode. Now both routes
|
|
2642
|
+
// run the same math.
|
|
2643
|
+
return computeBackoff(attempt + 1, backoff);
|
|
2644
|
+
}
|
|
2645
|
+
/**
|
|
2646
|
+
* The live OS processes behind one topology entry.
|
|
2647
|
+
*
|
|
2648
|
+
* Single-instance entries run as supervisor children; entries with
|
|
2649
|
+
* `instances > 1` run in a pool the supervisor knows nothing about. Both
|
|
2650
|
+
* shapes are resolved here, in one place, because metrics sampling and
|
|
2651
|
+
* status reporting each used to resolve them separately — and disagreed.
|
|
2652
|
+
* The pool branch asserted `online` from `pool.size > 0` before looking at
|
|
2653
|
+
* a pid at all, so a worker whose process had exited still read as online
|
|
2654
|
+
* (observed live: two pool workers reporting `online` on pids that `kill
|
|
2655
|
+
* -0` said were gone).
|
|
2656
|
+
*/
|
|
2657
|
+
resolveTopologyPids(handle, topo) {
|
|
2658
|
+
return this.resolveTopologyMembers(handle, topo).members.map((m) => m.pid);
|
|
2659
|
+
}
|
|
2660
|
+
/**
|
|
2661
|
+
* The live processes behind one topology entry, keyed by the PM process id
|
|
2662
|
+
* as well as the OS pid.
|
|
2663
|
+
*
|
|
2664
|
+
* The pid alone answers "is it alive"; the process id is what any question
|
|
2665
|
+
* ABOUT that process has to be asked with — `pm.getProcess(id)` carries the
|
|
2666
|
+
* spawn time, and the supervisor's restart counter is keyed by child name.
|
|
2667
|
+
* `resolveTopologyPids` used to discard both on the way out, which is why
|
|
2668
|
+
* the `omnitron list` DTO had nothing to report but the parent's numbers.
|
|
2669
|
+
*/
|
|
2670
|
+
resolveTopologyMembers(handle, topo) {
|
|
2671
|
+
const isPoolEntry = (topo.instances ?? 1) > 1;
|
|
2672
|
+
if (isPoolEntry) {
|
|
2673
|
+
const pool = handle.topologyPools.get(topo.name);
|
|
2674
|
+
if (!pool || pool.size === 0)
|
|
2675
|
+
return { members: [], childName: null };
|
|
2676
|
+
const members = pool
|
|
2677
|
+
.getWorkerIds()
|
|
2678
|
+
.map((processId) => ({ processId, pid: this.pm.getWorkerHandle(processId)?.pid }))
|
|
2679
|
+
.filter((m) => typeof m.pid === 'number');
|
|
2680
|
+
return { members, childName: null };
|
|
2681
|
+
}
|
|
2682
|
+
const childNames = handle.supervisor?.getChildNames() ?? [];
|
|
2683
|
+
const childName = childNames.find((cn) => cn.includes(topo.name)) ?? topo.name;
|
|
2684
|
+
const processId = handle.supervisor?.getChildProcessId(childName);
|
|
2685
|
+
if (!processId)
|
|
2686
|
+
return { members: [], childName };
|
|
2687
|
+
const pid = this.pm.getWorkerHandle(processId)?.pid;
|
|
2688
|
+
return { members: typeof pid === 'number' ? [{ processId, pid }] : [], childName };
|
|
2689
|
+
}
|
|
2690
|
+
/**
|
|
2691
|
+
* Sample every process an app owns, and record the per-entry readings on
|
|
2692
|
+
* the handle on the way through.
|
|
2693
|
+
*
|
|
2694
|
+
* A bootstrap app is several OS processes, so "the memory this app uses"
|
|
2695
|
+
* is their sum. The periodic poller used to sample the single pid stored
|
|
2696
|
+
* on the handle — which for a multi-process app is whichever child was
|
|
2697
|
+
* registered last — and overwrite `lastMetrics` with it every five
|
|
2698
|
+
* seconds, while `getMetrics()` right beside it aggregated correctly and
|
|
2699
|
+
* wrote to the same field. The poller won on frequency, so the aggregate
|
|
2700
|
+
* was never what anyone saw: `acme/dev/main` reported 122 MB against 437 MB
|
|
2701
|
+
* actually resident across its three processes.
|
|
2702
|
+
*
|
|
2703
|
+
* ## Why `ps` and not the child's own report
|
|
2704
|
+
*
|
|
2705
|
+
* The aggregating path preferred the PM-level metrics a child reports over
|
|
2706
|
+
* its RPC channel, falling back to `ps`. Those two are not the same
|
|
2707
|
+
* quantities:
|
|
2708
|
+
*
|
|
2709
|
+
* - `cpu` from the child is `(cpuUsage().user + .system) / 1e6` — CPU
|
|
2710
|
+
* SECONDS CONSUMED SINCE START, a monotonically growing counter. `ps`
|
|
2711
|
+
* `%cpu` is a percentage. The dashboard column is labelled `CPU %`,
|
|
2712
|
+
* so an app running for a day would have read `86400`. Observed
|
|
2713
|
+
* climbing live: 12.845 to 13.297 across twelve seconds of idling.
|
|
2714
|
+
* - `memory` from the child is `memoryUsage().heapUsed` — the V8 heap.
|
|
2715
|
+
* `ps` reports RSS, which is what "memory this app uses" means to
|
|
2716
|
+
* whoever is sizing a host. Observed: 66 MB heap against 149 MB
|
|
2717
|
+
* resident for the same process.
|
|
2718
|
+
*
|
|
2719
|
+
* Mixing them per-child, depending on which source answered first, gave a
|
|
2720
|
+
* column that was neither quantity. `ps` is asked for both figures here so
|
|
2721
|
+
* that every number under one label means one thing.
|
|
2722
|
+
*/
|
|
2723
|
+
async sampleAppMetrics(handle) {
|
|
2724
|
+
const topology = handle.topologyProcesses ?? [];
|
|
2725
|
+
const childNames = handle.supervisor?.getChildNames() ?? [];
|
|
2726
|
+
// Which pids belong to which topology entry. Everything is sampled in a
|
|
2727
|
+
// single `ps` call: sampling one child at a time would fork a process per
|
|
2728
|
+
// child per polling interval, and a monitor that costs more than what it
|
|
2729
|
+
// monitors is its own kind of defect.
|
|
2730
|
+
const pidsByEntry = new Map();
|
|
2731
|
+
for (const topo of topology) {
|
|
2732
|
+
pidsByEntry.set(topo.name, this.resolveTopologyPids(handle, topo));
|
|
2733
|
+
}
|
|
2734
|
+
if (topology.length === 0) {
|
|
2735
|
+
// No declared topology: fall back to the supervisor's children, and
|
|
2736
|
+
// failing that to the handle's own pid.
|
|
2737
|
+
for (const childName of childNames) {
|
|
2738
|
+
const processId = handle.supervisor?.getChildProcessId(childName);
|
|
2739
|
+
const pid = processId ? this.pm.getWorkerHandle(processId)?.pid : undefined;
|
|
2740
|
+
if (typeof pid === 'number')
|
|
2741
|
+
pidsByEntry.set(childName, [pid]);
|
|
2742
|
+
}
|
|
2743
|
+
if (pidsByEntry.size === 0 && handle.pid) {
|
|
2744
|
+
pidsByEntry.set(handle.name, [handle.pid]);
|
|
2745
|
+
}
|
|
2746
|
+
}
|
|
2747
|
+
const batch = await this.sampleProcessMetricsBatch([...pidsByEntry.values()].flat());
|
|
2748
|
+
let cpu = 0;
|
|
2749
|
+
let memory = 0;
|
|
2750
|
+
const sampled = new Map();
|
|
2751
|
+
for (const [name, pids] of pidsByEntry) {
|
|
2752
|
+
const sample = { cpu: 0, memory: 0 };
|
|
2753
|
+
for (const pid of pids) {
|
|
2754
|
+
const os = batch.get(pid);
|
|
2755
|
+
if (!os)
|
|
2756
|
+
continue;
|
|
2757
|
+
sample.cpu += os.cpu;
|
|
2758
|
+
sample.memory += os.memory;
|
|
2759
|
+
}
|
|
2760
|
+
sampled.set(name, sample);
|
|
2761
|
+
cpu += sample.cpu;
|
|
2762
|
+
memory += sample.memory;
|
|
2763
|
+
}
|
|
2764
|
+
// Traffic can only come from inside the child; `ps` knows nothing about
|
|
2765
|
+
// it. Its CPU and memory figures are deliberately NOT used — see the note
|
|
2766
|
+
// on this method. Only what a child reports as TRAFFIC is summed: the
|
|
2767
|
+
// `requests`/`errors` a child on an older runtime reports are the call
|
|
2768
|
+
// counters of its wrapper — the supervisor's own health and metrics
|
|
2769
|
+
// calls — and adding them up counted this daemon's questions as the
|
|
2770
|
+
// app's requests (`combineProcessTraffic`).
|
|
2771
|
+
const reports = [];
|
|
2772
|
+
for (const childName of childNames) {
|
|
2773
|
+
const reported = await handle.supervisor?.getChildMetrics(childName);
|
|
2774
|
+
reports.push(reported?.traffic);
|
|
2775
|
+
}
|
|
2776
|
+
const traffic = combineProcessTraffic(reports);
|
|
2777
|
+
// A sample that produced nothing for an app that HAS live processes is a
|
|
2778
|
+
// failed sample, not a measurement of zero — `ps` can time out under load,
|
|
2779
|
+
// and it did: an app was observed reporting 0 MB one second and 143 MB the
|
|
2780
|
+
// next. Overwriting the last known reading with zeros would put the exact
|
|
2781
|
+
// placeholder this method exists to remove back on the dashboard, so the
|
|
2782
|
+
// previous reading is kept instead.
|
|
2783
|
+
const sampledNothing = batch.size === 0 && [...pidsByEntry.values()].some((pids) => pids.length > 0);
|
|
2784
|
+
if (sampledNothing && handle.lastMetrics) {
|
|
2785
|
+
return handle.lastMetrics;
|
|
2786
|
+
}
|
|
2787
|
+
handle.childMetrics = sampled;
|
|
2788
|
+
return traffic
|
|
2789
|
+
? {
|
|
2790
|
+
cpu,
|
|
2791
|
+
memory,
|
|
2792
|
+
requests: traffic.requests,
|
|
2793
|
+
errors: traffic.serverErrors,
|
|
2794
|
+
traffic,
|
|
2795
|
+
...(traffic.latency
|
|
2796
|
+
? {
|
|
2797
|
+
latency: {
|
|
2798
|
+
p50: traffic.latency.p50,
|
|
2799
|
+
p75: traffic.latency.p75,
|
|
2800
|
+
p90: traffic.latency.p90,
|
|
2801
|
+
p95: traffic.latency.p95,
|
|
2802
|
+
p99: traffic.latency.p99,
|
|
2803
|
+
mean: traffic.latency.mean,
|
|
2804
|
+
},
|
|
2805
|
+
}
|
|
2806
|
+
: {}),
|
|
2807
|
+
}
|
|
2808
|
+
: { cpu, memory };
|
|
2809
|
+
}
|
|
2810
|
+
/**
|
|
2811
|
+
* Sample several processes in one `ps` call.
|
|
2812
|
+
*
|
|
2813
|
+
* Pids the OS no longer knows are simply absent from the result — an
|
|
2814
|
+
* omission the caller can see, unlike a zeroed reading it cannot tell
|
|
2815
|
+
* apart from an idle process.
|
|
2816
|
+
*/
|
|
2817
|
+
async sampleProcessMetricsBatch(pids) {
|
|
2818
|
+
const out = new Map();
|
|
2819
|
+
const unique = [...new Set(pids)];
|
|
2820
|
+
if (unique.length === 0)
|
|
2821
|
+
return out;
|
|
2822
|
+
try {
|
|
2823
|
+
const psOutput = await new Promise((resolve, reject) => {
|
|
2824
|
+
execFile('ps', ['-p', unique.join(','), '-o', 'pid=,rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
|
|
2825
|
+
// `ps` exits non-zero when NONE of the pids exist, but still
|
|
2826
|
+
// prints the ones that do — so stdout is worth reading either way.
|
|
2827
|
+
if (err && !stdout)
|
|
2828
|
+
reject(err);
|
|
2829
|
+
else
|
|
2830
|
+
resolve(stdout.trim());
|
|
2831
|
+
});
|
|
2832
|
+
});
|
|
2833
|
+
for (const [pid, sample] of parsePsBatch(psOutput))
|
|
2834
|
+
out.set(pid, sample);
|
|
2835
|
+
}
|
|
2836
|
+
catch {
|
|
2837
|
+
// Leave the map empty: no reading is honest, a zero reading is not.
|
|
2838
|
+
}
|
|
2839
|
+
return out;
|
|
2840
|
+
}
|
|
2841
|
+
/**
|
|
2842
|
+
* Sample CPU/memory for a single OS process via `ps`.
|
|
2843
|
+
*
|
|
2844
|
+
* `null` when it could not be read. This used to return
|
|
2845
|
+
* `{ cpu: 0, memory: 0 }` on any failure, and the caller wrote that
|
|
2846
|
+
* straight over `handle.lastMetrics` — so a `ps` that timed out under load
|
|
2847
|
+
* made a running classic-mode app read as idle, indistinguishable from one
|
|
2848
|
+
* that genuinely was. The batch path above already treated absence as
|
|
2849
|
+
* absence; these two now agree.
|
|
2850
|
+
*/
|
|
1035
2851
|
async sampleProcessMetrics(pid) {
|
|
1036
2852
|
try {
|
|
1037
|
-
const { execFile } = await import('node:child_process');
|
|
1038
2853
|
const psOutput = await new Promise((resolve, reject) => {
|
|
1039
|
-
execFile('ps', ['-p', String(pid), '-o', 'rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
|
|
1040
|
-
if (err)
|
|
2854
|
+
execFile('ps', ['-p', String(pid), '-o', 'pid=,rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
|
|
2855
|
+
if (err && !stdout)
|
|
1041
2856
|
reject(err);
|
|
1042
2857
|
else
|
|
1043
2858
|
resolve(stdout.trim());
|
|
1044
2859
|
});
|
|
1045
2860
|
});
|
|
1046
|
-
|
|
1047
|
-
return {
|
|
1048
|
-
cpu: parseFloat(cpuPercent ?? '0'),
|
|
1049
|
-
memory: parseInt(rssKb ?? '0', 10) * 1024,
|
|
1050
|
-
};
|
|
2861
|
+
return parsePsLine(psOutput)?.sample ?? null;
|
|
1051
2862
|
}
|
|
1052
2863
|
catch {
|
|
1053
|
-
return
|
|
2864
|
+
return null;
|
|
1054
2865
|
}
|
|
1055
2866
|
}
|
|
1056
|
-
/**
|
|
2867
|
+
/**
|
|
2868
|
+
* Periodic CPU/memory sampling for every managed process.
|
|
2869
|
+
*
|
|
2870
|
+
* Idempotent: re-arming replaces the previous timer rather than stacking a
|
|
2871
|
+
* second one, so `setConfig` can call it on every config reload.
|
|
2872
|
+
*/
|
|
1057
2873
|
startMetricsPolling(interval) {
|
|
2874
|
+
if (this.metricsTimer) {
|
|
2875
|
+
clearInterval(this.metricsTimer);
|
|
2876
|
+
this.metricsTimer = null;
|
|
2877
|
+
}
|
|
1058
2878
|
this.metricsTimer = setInterval(async () => {
|
|
1059
2879
|
for (const handle of this.handles.values()) {
|
|
1060
2880
|
if (handle.status !== 'online')
|
|
1061
2881
|
continue;
|
|
1062
2882
|
if (handle.mode === 'classic' && handle.childProcess?.pid) {
|
|
1063
2883
|
const m = await this.sampleProcessMetrics(handle.childProcess.pid);
|
|
1064
|
-
|
|
2884
|
+
// A failed read keeps the previous numbers rather than replacing
|
|
2885
|
+
// them with zeros — the same rule the bootstrap path below uses.
|
|
2886
|
+
if (m) {
|
|
2887
|
+
handle.lastMetrics = {
|
|
2888
|
+
...m,
|
|
2889
|
+
requests: handle.lastMetrics?.requests ?? 0,
|
|
2890
|
+
errors: handle.lastMetrics?.errors ?? 0,
|
|
2891
|
+
};
|
|
2892
|
+
}
|
|
1065
2893
|
}
|
|
1066
|
-
else if (handle.mode === 'bootstrap'
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
2894
|
+
else if (handle.mode === 'bootstrap') {
|
|
2895
|
+
// Every process the app owns, not just the one on the handle.
|
|
2896
|
+
// As sampled. The counts used to fall back to the previous reading
|
|
2897
|
+
// whenever a sample said 0 — which kept a stale figure alive
|
|
2898
|
+
// across a process restart (traffic counts restart from zero with
|
|
2899
|
+
// the process) and, while `requests` was the wrapper's call count,
|
|
2900
|
+
// mixed the two meanings in one field.
|
|
2901
|
+
handle.lastMetrics = await this.sampleAppMetrics(handle);
|
|
1073
2902
|
}
|
|
1074
2903
|
}
|
|
1075
2904
|
}, interval);
|
|
2905
|
+
this.metricsTimer.unref();
|
|
1076
2906
|
}
|
|
1077
2907
|
// ============================================================================
|
|
1078
2908
|
// Private — Utilities
|
|
1079
2909
|
// ============================================================================
|
|
1080
2910
|
toProcessInfo(handle) {
|
|
2911
|
+
// Ghost-online guard. The in-memory `handle.status` reflects the
|
|
2912
|
+
// last event the supervisor observed. If a child died between
|
|
2913
|
+
// the last sweep and now (kill -9 by an operator, OOM kill,
|
|
2914
|
+
// container restart) the handle still claims 'online' until the
|
|
2915
|
+
// next event lands. A cheap signal-0 verification at DTO-mapping
|
|
2916
|
+
// time keeps `omnitron list` honest: when the OS reports the pid
|
|
2917
|
+
// is gone, surface 'crashed' instead of lying to the operator.
|
|
2918
|
+
// `isAlive` covers EPERM as alive (correct — process exists, we
|
|
2919
|
+
// just can't signal it).
|
|
2920
|
+
let status = handle.status;
|
|
2921
|
+
if (status === 'online' && handle.pid != null && !isAlive(handle.pid)) {
|
|
2922
|
+
status = 'crashed';
|
|
2923
|
+
}
|
|
1081
2924
|
const info = {
|
|
1082
2925
|
name: handle.name,
|
|
1083
2926
|
pid: handle.pid,
|
|
1084
|
-
status
|
|
2927
|
+
status,
|
|
1085
2928
|
cpu: handle.cpu,
|
|
1086
2929
|
memory: handle.memory,
|
|
1087
2930
|
uptime: handle.uptime,
|
|
@@ -1095,28 +2938,49 @@ export class OrchestratorService extends EventEmitter {
|
|
|
1095
2938
|
if (handle.topologyProcesses && handle.topologyProcesses.length > 0 && handle.supervisor) {
|
|
1096
2939
|
info.processes = [];
|
|
1097
2940
|
for (const topo of handle.topologyProcesses) {
|
|
1098
|
-
const
|
|
1099
|
-
//
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
2941
|
+
const isPoolEntry = (topo.instances ?? 1) > 1;
|
|
2942
|
+
// The same ghost-online guard the parent gets a few lines above.
|
|
2943
|
+
// Without it the pool branch reported `online` from the pool's own
|
|
2944
|
+
// bookkeeping, never checking whether the pid it printed alongside
|
|
2945
|
+
// still existed.
|
|
2946
|
+
const { members, childName } = this.resolveTopologyMembers(handle, topo);
|
|
2947
|
+
const alive = members.filter((m) => isAlive(m.pid));
|
|
2948
|
+
const declaredInstances = topo.instances ?? 1;
|
|
2949
|
+
const liveInstances = isPoolEntry
|
|
2950
|
+
? (handle.topologyPools.get(topo.name)?.size ?? 0)
|
|
2951
|
+
: alive.length;
|
|
2952
|
+
const chosen = alive[0] ?? members[0];
|
|
2953
|
+
const childPid = chosen?.pid ?? null;
|
|
2954
|
+
const childStatus = alive.length > 0 ? 'online' : members.length > 0 ? 'crashed' : 'stopped';
|
|
2955
|
+
// The row prints one pid, so the row's clock is that pid's clock.
|
|
2956
|
+
// `startTime` is written when the process id is registered and a
|
|
2957
|
+
// restart registers a new one, so it moves when the process does.
|
|
2958
|
+
// (`IProcessInfo.restartCount` does NOT: it is assigned 0 there and
|
|
2959
|
+
// incremented nowhere. The supervisor keeps the real count, by child
|
|
2960
|
+
// name — reading the field instead would have replaced a wrong
|
|
2961
|
+
// constant with a wrong lookup.)
|
|
2962
|
+
const chosenInfo = chosen ? this.pm.getProcess(chosen.processId) : undefined;
|
|
2963
|
+
const childUptime = typeof chosenInfo?.startTime === 'number' && chosenInfo.startTime > 0
|
|
2964
|
+
? Math.max(0, Date.now() - chosenInfo.startTime)
|
|
2965
|
+
: 0;
|
|
2966
|
+
// A pool is not a supervisor child and nothing counts its restarts.
|
|
2967
|
+
// `null` says so; `0` would be absence written as a value from the
|
|
2968
|
+
// domain, leaving "never restarted" and "not tracked" identical.
|
|
2969
|
+
const childRestarts = isPoolEntry || !childName ? null : (handle.supervisor?.getRestartCount(childName) ?? null);
|
|
1109
2970
|
// Derive process type from declarations for backward-compatible DTO
|
|
1110
|
-
const derivedType = topo.transports ? 'server' :
|
|
2971
|
+
const derivedType = topo.transports ? 'server' : isPoolEntry ? 'worker' : 'custom';
|
|
2972
|
+
const sample = handle.childMetrics.get(topo.name);
|
|
1111
2973
|
info.processes.push({
|
|
1112
2974
|
name: topo.name,
|
|
1113
2975
|
type: derivedType,
|
|
1114
2976
|
pid: childPid,
|
|
1115
2977
|
status: childStatus,
|
|
1116
|
-
cpu: 0,
|
|
1117
|
-
memory: 0,
|
|
1118
|
-
uptime:
|
|
1119
|
-
restarts:
|
|
2978
|
+
cpu: sample?.cpu ?? 0,
|
|
2979
|
+
memory: sample?.memory ?? 0,
|
|
2980
|
+
uptime: childUptime,
|
|
2981
|
+
restarts: childRestarts,
|
|
2982
|
+
instances: liveInstances,
|
|
2983
|
+
declaredInstances,
|
|
1120
2984
|
});
|
|
1121
2985
|
}
|
|
1122
2986
|
}
|