@omnitron-dev/omnitron 0.1.3 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1253) hide show
  1. package/dist/BUILD.json +5 -0
  2. package/dist/cli/omnitron.js +378 -25
  3. package/dist/cli/omnitron.js.map +1 -1
  4. package/dist/cluster/cluster.rpc-service.d.ts +40 -0
  5. package/dist/cluster/cluster.rpc-service.d.ts.map +1 -1
  6. package/dist/cluster/cluster.rpc-service.js +49 -3
  7. package/dist/cluster/cluster.rpc-service.js.map +1 -1
  8. package/dist/cluster/config-sync.d.ts.map +1 -1
  9. package/dist/cluster/config-sync.js.map +1 -1
  10. package/dist/cluster/leader-election.d.ts +90 -2
  11. package/dist/cluster/leader-election.d.ts.map +1 -1
  12. package/dist/cluster/leader-election.js +176 -24
  13. package/dist/cluster/leader-election.js.map +1 -1
  14. package/dist/cluster/mesh-link.d.ts +136 -0
  15. package/dist/cluster/mesh-link.d.ts.map +1 -0
  16. package/dist/cluster/mesh-link.js +212 -0
  17. package/dist/cluster/mesh-link.js.map +1 -0
  18. package/dist/cluster/mesh.d.ts +59 -0
  19. package/dist/cluster/mesh.d.ts.map +1 -0
  20. package/dist/cluster/mesh.js +117 -0
  21. package/dist/cluster/mesh.js.map +1 -0
  22. package/dist/cluster/slave-connector.d.ts +193 -9
  23. package/dist/cluster/slave-connector.d.ts.map +1 -1
  24. package/dist/cluster/slave-connector.js +409 -37
  25. package/dist/cluster/slave-connector.js.map +1 -1
  26. package/dist/commands/audit.d.ts +65 -0
  27. package/dist/commands/audit.d.ts.map +1 -0
  28. package/dist/commands/audit.js +233 -0
  29. package/dist/commands/audit.js.map +1 -0
  30. package/dist/commands/backup.d.ts +4 -0
  31. package/dist/commands/backup.d.ts.map +1 -1
  32. package/dist/commands/backup.js +255 -33
  33. package/dist/commands/backup.js.map +1 -1
  34. package/dist/commands/cluster.d.ts.map +1 -1
  35. package/dist/commands/cluster.js +78 -47
  36. package/dist/commands/cluster.js.map +1 -1
  37. package/dist/commands/daemon-cmd.d.ts +7 -1
  38. package/dist/commands/daemon-cmd.d.ts.map +1 -1
  39. package/dist/commands/daemon-cmd.js +87 -21
  40. package/dist/commands/daemon-cmd.js.map +1 -1
  41. package/dist/commands/daemon-required.d.ts +34 -0
  42. package/dist/commands/daemon-required.d.ts.map +1 -0
  43. package/dist/commands/daemon-required.js +92 -0
  44. package/dist/commands/daemon-required.js.map +1 -0
  45. package/dist/commands/deploy.d.ts +70 -8
  46. package/dist/commands/deploy.d.ts.map +1 -1
  47. package/dist/commands/deploy.js +99 -64
  48. package/dist/commands/deploy.js.map +1 -1
  49. package/dist/commands/discover.d.ts +25 -4
  50. package/dist/commands/discover.d.ts.map +1 -1
  51. package/dist/commands/discover.js +155 -102
  52. package/dist/commands/discover.js.map +1 -1
  53. package/dist/commands/doctor.d.ts +293 -0
  54. package/dist/commands/doctor.d.ts.map +1 -0
  55. package/dist/commands/doctor.js +1706 -0
  56. package/dist/commands/doctor.js.map +1 -0
  57. package/dist/commands/env.d.ts +8 -1
  58. package/dist/commands/env.d.ts.map +1 -1
  59. package/dist/commands/env.js +18 -5
  60. package/dist/commands/env.js.map +1 -1
  61. package/dist/commands/exec.d.ts.map +1 -1
  62. package/dist/commands/exec.js +3 -2
  63. package/dist/commands/exec.js.map +1 -1
  64. package/dist/commands/fleet-asking.d.ts +87 -0
  65. package/dist/commands/fleet-asking.d.ts.map +1 -0
  66. package/dist/commands/fleet-asking.js +176 -0
  67. package/dist/commands/fleet-asking.js.map +1 -0
  68. package/dist/commands/fleet.d.ts +18 -0
  69. package/dist/commands/fleet.d.ts.map +1 -1
  70. package/dist/commands/fleet.js +314 -74
  71. package/dist/commands/fleet.js.map +1 -1
  72. package/dist/commands/health-check.d.ts +8 -6
  73. package/dist/commands/health-check.d.ts.map +1 -1
  74. package/dist/commands/health-check.js +26 -23
  75. package/dist/commands/health-check.js.map +1 -1
  76. package/dist/commands/health.d.ts.map +1 -1
  77. package/dist/commands/health.js +26 -4
  78. package/dist/commands/health.js.map +1 -1
  79. package/dist/commands/infra.d.ts +42 -1
  80. package/dist/commands/infra.d.ts.map +1 -1
  81. package/dist/commands/infra.js +418 -49
  82. package/dist/commands/infra.js.map +1 -1
  83. package/dist/commands/init.d.ts.map +1 -1
  84. package/dist/commands/init.js +21 -36
  85. package/dist/commands/init.js.map +1 -1
  86. package/dist/commands/inspect-graph.d.ts +20 -0
  87. package/dist/commands/inspect-graph.d.ts.map +1 -0
  88. package/dist/commands/inspect-graph.js +74 -0
  89. package/dist/commands/inspect-graph.js.map +1 -0
  90. package/dist/commands/inspect.d.ts.map +1 -1
  91. package/dist/commands/inspect.js +115 -11
  92. package/dist/commands/inspect.js.map +1 -1
  93. package/dist/commands/k8s.d.ts.map +1 -1
  94. package/dist/commands/k8s.js +23 -18
  95. package/dist/commands/k8s.js.map +1 -1
  96. package/dist/commands/kb.d.ts.map +1 -1
  97. package/dist/commands/kb.js +88 -29
  98. package/dist/commands/kb.js.map +1 -1
  99. package/dist/commands/list.d.ts.map +1 -1
  100. package/dist/commands/list.js +25 -3
  101. package/dist/commands/list.js.map +1 -1
  102. package/dist/commands/log-health.d.ts +120 -0
  103. package/dist/commands/log-health.d.ts.map +1 -0
  104. package/dist/commands/log-health.js +212 -0
  105. package/dist/commands/log-health.js.map +1 -0
  106. package/dist/commands/logs.d.ts +96 -2
  107. package/dist/commands/logs.d.ts.map +1 -1
  108. package/dist/commands/logs.js +532 -192
  109. package/dist/commands/logs.js.map +1 -1
  110. package/dist/commands/metrics.d.ts.map +1 -1
  111. package/dist/commands/metrics.js +40 -14
  112. package/dist/commands/metrics.js.map +1 -1
  113. package/dist/commands/monit.d.ts.map +1 -1
  114. package/dist/commands/monit.js +12 -2
  115. package/dist/commands/monit.js.map +1 -1
  116. package/dist/commands/node.d.ts +68 -0
  117. package/dist/commands/node.d.ts.map +1 -1
  118. package/dist/commands/node.js +285 -68
  119. package/dist/commands/node.js.map +1 -1
  120. package/dist/commands/output.d.ts +45 -0
  121. package/dist/commands/output.d.ts.map +1 -0
  122. package/dist/commands/output.js +103 -0
  123. package/dist/commands/output.js.map +1 -0
  124. package/dist/commands/pipeline.d.ts.map +1 -1
  125. package/dist/commands/pipeline.js +26 -19
  126. package/dist/commands/pipeline.js.map +1 -1
  127. package/dist/commands/project.d.ts +19 -0
  128. package/dist/commands/project.d.ts.map +1 -1
  129. package/dist/commands/project.js +125 -27
  130. package/dist/commands/project.js.map +1 -1
  131. package/dist/commands/release-store.d.ts +28 -0
  132. package/dist/commands/release-store.d.ts.map +1 -0
  133. package/dist/commands/release-store.js +158 -0
  134. package/dist/commands/release-store.js.map +1 -0
  135. package/dist/commands/release.d.ts +61 -0
  136. package/dist/commands/release.d.ts.map +1 -0
  137. package/dist/commands/release.js +555 -0
  138. package/dist/commands/release.js.map +1 -0
  139. package/dist/commands/reload.d.ts +16 -0
  140. package/dist/commands/reload.d.ts.map +1 -1
  141. package/dist/commands/reload.js +21 -5
  142. package/dist/commands/reload.js.map +1 -1
  143. package/dist/commands/remote.d.ts +40 -0
  144. package/dist/commands/remote.d.ts.map +1 -1
  145. package/dist/commands/remote.js +146 -35
  146. package/dist/commands/remote.js.map +1 -1
  147. package/dist/commands/restart.d.ts +16 -0
  148. package/dist/commands/restart.d.ts.map +1 -1
  149. package/dist/commands/restart.js +21 -5
  150. package/dist/commands/restart.js.map +1 -1
  151. package/dist/commands/scale.d.ts.map +1 -1
  152. package/dist/commands/scale.js +4 -3
  153. package/dist/commands/scale.js.map +1 -1
  154. package/dist/commands/secret.d.ts +54 -0
  155. package/dist/commands/secret.d.ts.map +1 -1
  156. package/dist/commands/secret.js +236 -34
  157. package/dist/commands/secret.js.map +1 -1
  158. package/dist/commands/service-units.d.ts +181 -0
  159. package/dist/commands/service-units.d.ts.map +1 -0
  160. package/dist/commands/service-units.js +291 -0
  161. package/dist/commands/service-units.js.map +1 -0
  162. package/dist/commands/service.d.ts +39 -0
  163. package/dist/commands/service.d.ts.map +1 -0
  164. package/dist/commands/service.js +370 -0
  165. package/dist/commands/service.js.map +1 -0
  166. package/dist/commands/spinner.d.ts +29 -0
  167. package/dist/commands/spinner.d.ts.map +1 -0
  168. package/dist/commands/spinner.js +44 -0
  169. package/dist/commands/spinner.js.map +1 -0
  170. package/dist/commands/stack.d.ts +49 -1
  171. package/dist/commands/stack.d.ts.map +1 -1
  172. package/dist/commands/stack.js +341 -13
  173. package/dist/commands/stack.js.map +1 -1
  174. package/dist/commands/start.d.ts +16 -0
  175. package/dist/commands/start.d.ts.map +1 -1
  176. package/dist/commands/start.js +20 -3
  177. package/dist/commands/start.js.map +1 -1
  178. package/dist/commands/status.d.ts +9 -0
  179. package/dist/commands/status.d.ts.map +1 -1
  180. package/dist/commands/status.js +62 -29
  181. package/dist/commands/status.js.map +1 -1
  182. package/dist/commands/stop.d.ts +16 -0
  183. package/dist/commands/stop.d.ts.map +1 -1
  184. package/dist/commands/stop.js +34 -6
  185. package/dist/commands/stop.js.map +1 -1
  186. package/dist/commands/tor.d.ts +20 -0
  187. package/dist/commands/tor.d.ts.map +1 -0
  188. package/dist/commands/tor.js +180 -0
  189. package/dist/commands/tor.js.map +1 -0
  190. package/dist/commands/up.d.ts +85 -1
  191. package/dist/commands/up.d.ts.map +1 -1
  192. package/dist/commands/up.js +181 -33
  193. package/dist/commands/up.js.map +1 -1
  194. package/dist/commands/webapp.d.ts.map +1 -1
  195. package/dist/commands/webapp.js +2 -1
  196. package/dist/commands/webapp.js.map +1 -1
  197. package/dist/config/defaults.d.ts +16 -0
  198. package/dist/config/defaults.d.ts.map +1 -1
  199. package/dist/config/defaults.js +26 -3
  200. package/dist/config/defaults.js.map +1 -1
  201. package/dist/config/define-ecosystem.d.ts.map +1 -1
  202. package/dist/config/define-ecosystem.js +2 -0
  203. package/dist/config/define-ecosystem.js.map +1 -1
  204. package/dist/config/index.d.ts +1 -1
  205. package/dist/config/index.d.ts.map +1 -1
  206. package/dist/config/index.js +1 -1
  207. package/dist/config/index.js.map +1 -1
  208. package/dist/config/loader.d.ts +45 -1
  209. package/dist/config/loader.d.ts.map +1 -1
  210. package/dist/config/loader.js +93 -11
  211. package/dist/config/loader.js.map +1 -1
  212. package/dist/config/omnitron-config.schema.d.ts.map +1 -1
  213. package/dist/config/omnitron-config.schema.js +16 -0
  214. package/dist/config/omnitron-config.schema.js.map +1 -1
  215. package/dist/config/types.d.ts +425 -42
  216. package/dist/config/types.d.ts.map +1 -1
  217. package/dist/daemon/config-reload.d.ts +27 -0
  218. package/dist/daemon/config-reload.d.ts.map +1 -0
  219. package/dist/daemon/config-reload.js +42 -0
  220. package/dist/daemon/config-reload.js.map +1 -0
  221. package/dist/daemon/daemon-client.d.ts +150 -2
  222. package/dist/daemon/daemon-client.d.ts.map +1 -1
  223. package/dist/daemon/daemon-client.js +311 -21
  224. package/dist/daemon/daemon-client.js.map +1 -1
  225. package/dist/daemon/daemon-entry.js +92 -38
  226. package/dist/daemon/daemon-entry.js.map +1 -1
  227. package/dist/daemon/daemon-scheduler.d.ts +4 -0
  228. package/dist/daemon/daemon-scheduler.d.ts.map +1 -1
  229. package/dist/daemon/daemon-scheduler.js +137 -7
  230. package/dist/daemon/daemon-scheduler.js.map +1 -1
  231. package/dist/daemon/daemon-state-store.service.d.ts +263 -0
  232. package/dist/daemon/daemon-state-store.service.d.ts.map +1 -0
  233. package/dist/daemon/daemon-state-store.service.js +446 -0
  234. package/dist/daemon/daemon-state-store.service.js.map +1 -0
  235. package/dist/daemon/daemon.d.ts +207 -1
  236. package/dist/daemon/daemon.d.ts.map +1 -1
  237. package/dist/daemon/daemon.js +1254 -143
  238. package/dist/daemon/daemon.js.map +1 -1
  239. package/dist/daemon/daemon.module.d.ts.map +1 -1
  240. package/dist/daemon/daemon.module.js +186 -79
  241. package/dist/daemon/daemon.module.js.map +1 -1
  242. package/dist/daemon/daemon.rpc-service.d.ts +109 -1
  243. package/dist/daemon/daemon.rpc-service.d.ts.map +1 -1
  244. package/dist/daemon/daemon.rpc-service.js +451 -95
  245. package/dist/daemon/daemon.rpc-service.js.map +1 -1
  246. package/dist/daemon/pid-manager.d.ts +69 -5
  247. package/dist/daemon/pid-manager.d.ts.map +1 -1
  248. package/dist/daemon/pid-manager.js +151 -17
  249. package/dist/daemon/pid-manager.js.map +1 -1
  250. package/dist/daemon/spawn-daemon.d.ts +73 -0
  251. package/dist/daemon/spawn-daemon.d.ts.map +1 -0
  252. package/dist/daemon/spawn-daemon.js +101 -0
  253. package/dist/daemon/spawn-daemon.js.map +1 -0
  254. package/dist/daemon/state-store.d.ts +65 -3
  255. package/dist/daemon/state-store.d.ts.map +1 -1
  256. package/dist/daemon/state-store.js +103 -19
  257. package/dist/daemon/state-store.js.map +1 -1
  258. package/dist/database/connection.d.ts +71 -0
  259. package/dist/database/connection.d.ts.map +1 -0
  260. package/dist/database/connection.js +129 -0
  261. package/dist/database/connection.js.map +1 -0
  262. package/dist/database/dialect.d.ts +60 -0
  263. package/dist/database/dialect.d.ts.map +1 -0
  264. package/dist/database/dialect.js +80 -0
  265. package/dist/database/dialect.js.map +1 -0
  266. package/dist/database/migrate.d.ts +5 -2
  267. package/dist/database/migrate.d.ts.map +1 -1
  268. package/dist/database/migrate.js +23 -33
  269. package/dist/database/migrate.js.map +1 -1
  270. package/dist/database/migration-runner.d.ts +50 -0
  271. package/dist/database/migration-runner.d.ts.map +1 -0
  272. package/dist/database/migration-runner.js +143 -0
  273. package/dist/database/migration-runner.js.map +1 -0
  274. package/dist/database/migrations/006_login_throttle.d.ts +17 -0
  275. package/dist/database/migrations/006_login_throttle.d.ts.map +1 -0
  276. package/dist/database/migrations/006_login_throttle.js +33 -0
  277. package/dist/database/migrations/006_login_throttle.js.map +1 -0
  278. package/dist/database/migrations/007_sync_ingested.d.ts +21 -0
  279. package/dist/database/migrations/007_sync_ingested.d.ts.map +1 -0
  280. package/dist/database/migrations/007_sync_ingested.js +39 -0
  281. package/dist/database/migrations/007_sync_ingested.js.map +1 -0
  282. package/dist/database/migrations/008_node_health_version_text.d.ts +24 -0
  283. package/dist/database/migrations/008_node_health_version_text.d.ts.map +1 -0
  284. package/dist/database/migrations/008_node_health_version_text.js +35 -0
  285. package/dist/database/migrations/008_node_health_version_text.js.map +1 -0
  286. package/dist/database/migrations/009_node_health_unmeasured.d.ts +40 -0
  287. package/dist/database/migrations/009_node_health_unmeasured.d.ts.map +1 -0
  288. package/dist/database/migrations/009_node_health_unmeasured.js +63 -0
  289. package/dist/database/migrations/009_node_health_unmeasured.js.map +1 -0
  290. package/dist/database/migrations/010_metrics_raw_had_no_reader.d.ts +22 -0
  291. package/dist/database/migrations/010_metrics_raw_had_no_reader.d.ts.map +1 -0
  292. package/dist/database/migrations/010_metrics_raw_had_no_reader.js +26 -0
  293. package/dist/database/migrations/010_metrics_raw_had_no_reader.js.map +1 -0
  294. package/dist/database/migrations/011_alerts_nobody_had_to_write.d.ts +56 -0
  295. package/dist/database/migrations/011_alerts_nobody_had_to_write.d.ts.map +1 -0
  296. package/dist/database/migrations/011_alerts_nobody_had_to_write.js +79 -0
  297. package/dist/database/migrations/011_alerts_nobody_had_to_write.js.map +1 -0
  298. package/dist/database/migrations/012_a_disk_nobody_watched.d.ts +26 -0
  299. package/dist/database/migrations/012_a_disk_nobody_watched.d.ts.map +1 -0
  300. package/dist/database/migrations/012_a_disk_nobody_watched.js +40 -0
  301. package/dist/database/migrations/012_a_disk_nobody_watched.js.map +1 -0
  302. package/dist/database/migrations/013_an_alarm_delivered.d.ts +14 -0
  303. package/dist/database/migrations/013_an_alarm_delivered.d.ts.map +1 -0
  304. package/dist/database/migrations/013_an_alarm_delivered.js +28 -0
  305. package/dist/database/migrations/013_an_alarm_delivered.js.map +1 -0
  306. package/dist/database/migrations/index.d.ts +21 -0
  307. package/dist/database/migrations/index.d.ts.map +1 -0
  308. package/dist/database/migrations/index.js +47 -0
  309. package/dist/database/migrations/index.js.map +1 -0
  310. package/dist/database/schema.d.ts +27 -11
  311. package/dist/database/schema.d.ts.map +1 -1
  312. package/dist/database/sqlite-date-binding.d.ts +71 -0
  313. package/dist/database/sqlite-date-binding.d.ts.map +1 -0
  314. package/dist/database/sqlite-date-binding.js +174 -0
  315. package/dist/database/sqlite-date-binding.js.map +1 -0
  316. package/dist/execution/data-channel.d.ts +48 -0
  317. package/dist/execution/data-channel.d.ts.map +1 -0
  318. package/dist/execution/data-channel.js +55 -0
  319. package/dist/execution/data-channel.js.map +1 -0
  320. package/dist/execution/execution.service.d.ts +139 -0
  321. package/dist/execution/execution.service.d.ts.map +1 -1
  322. package/dist/execution/execution.service.js +273 -10
  323. package/dist/execution/execution.service.js.map +1 -1
  324. package/dist/index.d.ts +1 -1
  325. package/dist/index.d.ts.map +1 -1
  326. package/dist/infrastructure/bare-metal-plan.d.ts +208 -0
  327. package/dist/infrastructure/bare-metal-plan.d.ts.map +1 -0
  328. package/dist/infrastructure/bare-metal-plan.js +363 -0
  329. package/dist/infrastructure/bare-metal-plan.js.map +1 -0
  330. package/dist/infrastructure/bare-metal-runner.d.ts +65 -0
  331. package/dist/infrastructure/bare-metal-runner.d.ts.map +1 -0
  332. package/dist/infrastructure/bare-metal-runner.js +354 -0
  333. package/dist/infrastructure/bare-metal-runner.js.map +1 -0
  334. package/dist/infrastructure/config-payload.d.ts +149 -0
  335. package/dist/infrastructure/config-payload.d.ts.map +1 -0
  336. package/dist/infrastructure/config-payload.js +235 -0
  337. package/dist/infrastructure/config-payload.js.map +1 -0
  338. package/dist/infrastructure/container-runtime.d.ts +179 -1
  339. package/dist/infrastructure/container-runtime.d.ts.map +1 -1
  340. package/dist/infrastructure/container-runtime.js +612 -45
  341. package/dist/infrastructure/container-runtime.js.map +1 -1
  342. package/dist/infrastructure/gateway-reachability.d.ts +85 -0
  343. package/dist/infrastructure/gateway-reachability.d.ts.map +1 -0
  344. package/dist/infrastructure/gateway-reachability.js +116 -0
  345. package/dist/infrastructure/gateway-reachability.js.map +1 -0
  346. package/dist/infrastructure/gateway-redis-db.d.ts +29 -0
  347. package/dist/infrastructure/gateway-redis-db.d.ts.map +1 -0
  348. package/dist/infrastructure/gateway-redis-db.js +32 -0
  349. package/dist/infrastructure/gateway-redis-db.js.map +1 -0
  350. package/dist/infrastructure/host-inspection.d.ts +223 -0
  351. package/dist/infrastructure/host-inspection.d.ts.map +1 -0
  352. package/dist/infrastructure/host-inspection.js +501 -0
  353. package/dist/infrastructure/host-inspection.js.map +1 -0
  354. package/dist/infrastructure/infrastructure-gate.d.ts.map +1 -1
  355. package/dist/infrastructure/infrastructure.service.d.ts +247 -6
  356. package/dist/infrastructure/infrastructure.service.d.ts.map +1 -1
  357. package/dist/infrastructure/infrastructure.service.js +571 -96
  358. package/dist/infrastructure/infrastructure.service.js.map +1 -1
  359. package/dist/infrastructure/known-machines.d.ts +76 -0
  360. package/dist/infrastructure/known-machines.d.ts.map +1 -0
  361. package/dist/infrastructure/known-machines.js +94 -0
  362. package/dist/infrastructure/known-machines.js.map +1 -0
  363. package/dist/infrastructure/node-credentials.d.ts +45 -0
  364. package/dist/infrastructure/node-credentials.d.ts.map +1 -0
  365. package/dist/infrastructure/node-credentials.js +80 -0
  366. package/dist/infrastructure/node-credentials.js.map +1 -0
  367. package/dist/infrastructure/orphan-containers.d.ts +99 -0
  368. package/dist/infrastructure/orphan-containers.d.ts.map +1 -0
  369. package/dist/infrastructure/orphan-containers.js +106 -0
  370. package/dist/infrastructure/orphan-containers.js.map +1 -0
  371. package/dist/infrastructure/phantom-endpoint-janitor.d.ts +92 -0
  372. package/dist/infrastructure/phantom-endpoint-janitor.d.ts.map +1 -0
  373. package/dist/infrastructure/phantom-endpoint-janitor.js +199 -0
  374. package/dist/infrastructure/phantom-endpoint-janitor.js.map +1 -0
  375. package/dist/infrastructure/presets/gateway-openresty.d.ts.map +1 -1
  376. package/dist/infrastructure/presets/gateway-openresty.js +29 -3
  377. package/dist/infrastructure/presets/gateway-openresty.js.map +1 -1
  378. package/dist/infrastructure/presets/index.d.ts.map +1 -1
  379. package/dist/infrastructure/presets/index.js +2 -0
  380. package/dist/infrastructure/presets/index.js.map +1 -1
  381. package/dist/infrastructure/presets/minio.d.ts.map +1 -1
  382. package/dist/infrastructure/presets/minio.js +36 -3
  383. package/dist/infrastructure/presets/minio.js.map +1 -1
  384. package/dist/infrastructure/presets/postgres.d.ts.map +1 -1
  385. package/dist/infrastructure/presets/postgres.js +78 -5
  386. package/dist/infrastructure/presets/postgres.js.map +1 -1
  387. package/dist/infrastructure/presets/redis.d.ts.map +1 -1
  388. package/dist/infrastructure/presets/redis.js +23 -4
  389. package/dist/infrastructure/presets/redis.js.map +1 -1
  390. package/dist/infrastructure/presets/registry.d.ts.map +1 -1
  391. package/dist/infrastructure/presets/registry.js +85 -4
  392. package/dist/infrastructure/presets/registry.js.map +1 -1
  393. package/dist/infrastructure/presets/tor.d.ts +112 -0
  394. package/dist/infrastructure/presets/tor.d.ts.map +1 -0
  395. package/dist/infrastructure/presets/tor.js +185 -0
  396. package/dist/infrastructure/presets/tor.js.map +1 -0
  397. package/dist/infrastructure/presets/torrc.d.ts +54 -0
  398. package/dist/infrastructure/presets/torrc.d.ts.map +1 -0
  399. package/dist/infrastructure/presets/torrc.js +153 -0
  400. package/dist/infrastructure/presets/torrc.js.map +1 -0
  401. package/dist/infrastructure/presets/types.d.ts +17 -0
  402. package/dist/infrastructure/presets/types.d.ts.map +1 -1
  403. package/dist/infrastructure/provisioning-outcome.d.ts +85 -0
  404. package/dist/infrastructure/provisioning-outcome.d.ts.map +1 -0
  405. package/dist/infrastructure/provisioning-outcome.js +122 -0
  406. package/dist/infrastructure/provisioning-outcome.js.map +1 -0
  407. package/dist/infrastructure/server-registry.d.ts +27 -0
  408. package/dist/infrastructure/server-registry.d.ts.map +1 -1
  409. package/dist/infrastructure/server-registry.js +60 -5
  410. package/dist/infrastructure/server-registry.js.map +1 -1
  411. package/dist/infrastructure/service-binding.d.ts +60 -0
  412. package/dist/infrastructure/service-binding.d.ts.map +1 -0
  413. package/dist/infrastructure/service-binding.js +90 -0
  414. package/dist/infrastructure/service-binding.js.map +1 -0
  415. package/dist/infrastructure/service-credentials.d.ts +105 -0
  416. package/dist/infrastructure/service-credentials.d.ts.map +1 -0
  417. package/dist/infrastructure/service-credentials.js +141 -0
  418. package/dist/infrastructure/service-credentials.js.map +1 -0
  419. package/dist/infrastructure/service-resolver.d.ts +191 -12
  420. package/dist/infrastructure/service-resolver.d.ts.map +1 -1
  421. package/dist/infrastructure/service-resolver.js +521 -60
  422. package/dist/infrastructure/service-resolver.js.map +1 -1
  423. package/dist/infrastructure/shipped-config.d.ts +43 -0
  424. package/dist/infrastructure/shipped-config.d.ts.map +1 -0
  425. package/dist/infrastructure/shipped-config.js +159 -0
  426. package/dist/infrastructure/shipped-config.js.map +1 -0
  427. package/dist/infrastructure/stack-infra-manager.d.ts +7 -1
  428. package/dist/infrastructure/stack-infra-manager.d.ts.map +1 -1
  429. package/dist/infrastructure/stack-infra-manager.js +32 -14
  430. package/dist/infrastructure/stack-infra-manager.js.map +1 -1
  431. package/dist/infrastructure/types.d.ts +278 -6
  432. package/dist/infrastructure/types.d.ts.map +1 -1
  433. package/dist/mcp/mcp-bridge.d.ts.map +1 -1
  434. package/dist/mcp/mcp-bridge.js.map +1 -1
  435. package/dist/mcp/tool-groups/apps.tools.d.ts +2 -1
  436. package/dist/mcp/tool-groups/apps.tools.d.ts.map +1 -1
  437. package/dist/mcp/tool-groups/apps.tools.js +2 -2
  438. package/dist/mcp/tool-groups/apps.tools.js.map +1 -1
  439. package/dist/mcp/tool-groups/infra.tools.d.ts +16 -2
  440. package/dist/mcp/tool-groups/infra.tools.d.ts.map +1 -1
  441. package/dist/mcp/tool-groups/infra.tools.js +51 -41
  442. package/dist/mcp/tool-groups/infra.tools.js.map +1 -1
  443. package/dist/mcp/tool-groups/kb.tools.d.ts.map +1 -1
  444. package/dist/mcp/tool-groups/kb.tools.js +4 -6
  445. package/dist/mcp/tool-groups/kb.tools.js.map +1 -1
  446. package/dist/mcp/tool-groups/management.tools.d.ts +20 -3
  447. package/dist/mcp/tool-groups/management.tools.d.ts.map +1 -1
  448. package/dist/mcp/tool-groups/management.tools.js +158 -125
  449. package/dist/mcp/tool-groups/management.tools.js.map +1 -1
  450. package/dist/mcp/tool-groups/monitoring.tools.d.ts +10 -1
  451. package/dist/mcp/tool-groups/monitoring.tools.d.ts.map +1 -1
  452. package/dist/mcp/tool-groups/monitoring.tools.js +65 -18
  453. package/dist/mcp/tool-groups/monitoring.tools.js.map +1 -1
  454. package/dist/mcp/unavailable-tools.d.ts +35 -0
  455. package/dist/mcp/unavailable-tools.d.ts.map +1 -0
  456. package/dist/mcp/unavailable-tools.js +59 -0
  457. package/dist/mcp/unavailable-tools.js.map +1 -0
  458. package/dist/monitoring/app-health.indicator.d.ts.map +1 -1
  459. package/dist/monitoring/app-health.indicator.js +12 -0
  460. package/dist/monitoring/app-health.indicator.js.map +1 -1
  461. package/dist/monitoring/desktop-notifier.d.ts +26 -0
  462. package/dist/monitoring/desktop-notifier.d.ts.map +1 -0
  463. package/dist/monitoring/desktop-notifier.js +49 -0
  464. package/dist/monitoring/desktop-notifier.js.map +1 -0
  465. package/dist/monitoring/docker-health.indicator.d.ts +17 -1
  466. package/dist/monitoring/docker-health.indicator.d.ts.map +1 -1
  467. package/dist/monitoring/docker-health.indicator.js +38 -8
  468. package/dist/monitoring/docker-health.indicator.js.map +1 -1
  469. package/dist/monitoring/event-loop-stall.indicator.d.ts +26 -0
  470. package/dist/monitoring/event-loop-stall.indicator.d.ts.map +1 -0
  471. package/dist/monitoring/event-loop-stall.indicator.js +47 -0
  472. package/dist/monitoring/event-loop-stall.indicator.js.map +1 -0
  473. package/dist/monitoring/event-loop-watch.d.ts +124 -0
  474. package/dist/monitoring/event-loop-watch.d.ts.map +1 -0
  475. package/dist/monitoring/event-loop-watch.js +179 -0
  476. package/dist/monitoring/event-loop-watch.js.map +1 -0
  477. package/dist/monitoring/host-disk.d.ts +6 -0
  478. package/dist/monitoring/host-disk.d.ts.map +1 -0
  479. package/dist/monitoring/host-disk.js +26 -0
  480. package/dist/monitoring/host-disk.js.map +1 -0
  481. package/dist/monitoring/log-line.d.ts +188 -0
  482. package/dist/monitoring/log-line.d.ts.map +1 -0
  483. package/dist/monitoring/log-line.js +458 -0
  484. package/dist/monitoring/log-line.js.map +1 -0
  485. package/dist/monitoring/log-manager.d.ts +89 -8
  486. package/dist/monitoring/log-manager.d.ts.map +1 -1
  487. package/dist/monitoring/log-manager.js +246 -77
  488. package/dist/monitoring/log-manager.js.map +1 -1
  489. package/dist/observability/index.d.ts +4 -0
  490. package/dist/observability/index.d.ts.map +1 -0
  491. package/dist/observability/index.js +3 -0
  492. package/dist/observability/index.js.map +1 -0
  493. package/dist/observability/metrics-bridge.d.ts +87 -0
  494. package/dist/observability/metrics-bridge.d.ts.map +1 -0
  495. package/dist/observability/metrics-bridge.js +295 -0
  496. package/dist/observability/metrics-bridge.js.map +1 -0
  497. package/dist/observability/metrics-server.d.ts +68 -0
  498. package/dist/observability/metrics-server.d.ts.map +1 -0
  499. package/dist/observability/metrics-server.js +146 -0
  500. package/dist/observability/metrics-server.js.map +1 -0
  501. package/dist/orchestrator/app-handle.d.ts +170 -3
  502. package/dist/orchestrator/app-handle.d.ts.map +1 -1
  503. package/dist/orchestrator/app-handle.js +177 -3
  504. package/dist/orchestrator/app-handle.js.map +1 -1
  505. package/dist/orchestrator/app-health.d.ts +33 -0
  506. package/dist/orchestrator/app-health.d.ts.map +1 -0
  507. package/dist/orchestrator/app-health.js +74 -0
  508. package/dist/orchestrator/app-health.js.map +1 -0
  509. package/dist/orchestrator/app-traffic.d.ts +14 -0
  510. package/dist/orchestrator/app-traffic.d.ts.map +1 -0
  511. package/dist/orchestrator/app-traffic.js +33 -0
  512. package/dist/orchestrator/app-traffic.js.map +1 -0
  513. package/dist/orchestrator/bootstrap-loader-child.d.ts +19 -0
  514. package/dist/orchestrator/bootstrap-loader-child.d.ts.map +1 -0
  515. package/dist/orchestrator/bootstrap-loader-child.js +43 -0
  516. package/dist/orchestrator/bootstrap-loader-child.js.map +1 -0
  517. package/dist/orchestrator/bootstrap-loader.d.ts +72 -0
  518. package/dist/orchestrator/bootstrap-loader.d.ts.map +1 -1
  519. package/dist/orchestrator/bootstrap-loader.js +350 -5
  520. package/dist/orchestrator/bootstrap-loader.js.map +1 -1
  521. package/dist/orchestrator/bootstrap-process.d.ts +148 -4
  522. package/dist/orchestrator/bootstrap-process.d.ts.map +1 -1
  523. package/dist/orchestrator/bootstrap-process.js +496 -18
  524. package/dist/orchestrator/bootstrap-process.js.map +1 -1
  525. package/dist/orchestrator/build-service.d.ts +50 -3
  526. package/dist/orchestrator/build-service.d.ts.map +1 -1
  527. package/dist/orchestrator/build-service.js +357 -85
  528. package/dist/orchestrator/build-service.js.map +1 -1
  529. package/dist/orchestrator/call-exposed-service.d.ts +7 -0
  530. package/dist/orchestrator/call-exposed-service.d.ts.map +1 -0
  531. package/dist/orchestrator/call-exposed-service.js +50 -0
  532. package/dist/orchestrator/call-exposed-service.js.map +1 -0
  533. package/dist/orchestrator/classic-launcher.d.ts +1 -0
  534. package/dist/orchestrator/classic-launcher.d.ts.map +1 -1
  535. package/dist/orchestrator/classic-launcher.js +34 -3
  536. package/dist/orchestrator/classic-launcher.js.map +1 -1
  537. package/dist/orchestrator/file-watcher.d.ts +78 -1
  538. package/dist/orchestrator/file-watcher.d.ts.map +1 -1
  539. package/dist/orchestrator/file-watcher.js +240 -43
  540. package/dist/orchestrator/file-watcher.js.map +1 -1
  541. package/dist/orchestrator/module-worker-process.d.ts.map +1 -1
  542. package/dist/orchestrator/module-worker-process.js +10 -0
  543. package/dist/orchestrator/module-worker-process.js.map +1 -1
  544. package/dist/orchestrator/orchestrator.service.d.ts +540 -7
  545. package/dist/orchestrator/orchestrator.service.d.ts.map +1 -1
  546. package/dist/orchestrator/orchestrator.service.js +2106 -242
  547. package/dist/orchestrator/orchestrator.service.js.map +1 -1
  548. package/dist/orchestrator/owned-pids.d.ts +83 -0
  549. package/dist/orchestrator/owned-pids.d.ts.map +1 -0
  550. package/dist/orchestrator/owned-pids.js +93 -0
  551. package/dist/orchestrator/owned-pids.js.map +1 -0
  552. package/dist/orchestrator/port-conflict.d.ts +102 -0
  553. package/dist/orchestrator/port-conflict.d.ts.map +1 -0
  554. package/dist/orchestrator/port-conflict.js +131 -0
  555. package/dist/orchestrator/port-conflict.js.map +1 -0
  556. package/dist/orchestrator/process-janitor.d.ts +227 -0
  557. package/dist/orchestrator/process-janitor.d.ts.map +1 -0
  558. package/dist/orchestrator/process-janitor.js +495 -0
  559. package/dist/orchestrator/process-janitor.js.map +1 -0
  560. package/dist/orchestrator/ps-metrics.d.ts +37 -0
  561. package/dist/orchestrator/ps-metrics.d.ts.map +1 -0
  562. package/dist/orchestrator/ps-metrics.js +60 -0
  563. package/dist/orchestrator/ps-metrics.js.map +1 -0
  564. package/dist/orchestrator/service-router.d.ts +58 -2
  565. package/dist/orchestrator/service-router.d.ts.map +1 -1
  566. package/dist/orchestrator/service-router.js +157 -25
  567. package/dist/orchestrator/service-router.js.map +1 -1
  568. package/dist/orchestrator/topology-query.d.ts +49 -0
  569. package/dist/orchestrator/topology-query.d.ts.map +1 -0
  570. package/dist/orchestrator/topology-query.js +54 -0
  571. package/dist/orchestrator/topology-query.js.map +1 -0
  572. package/dist/project/artifact-builder.d.ts +174 -3
  573. package/dist/project/artifact-builder.d.ts.map +1 -1
  574. package/dist/project/artifact-builder.js +454 -42
  575. package/dist/project/artifact-builder.js.map +1 -1
  576. package/dist/project/build-decision.d.ts +58 -0
  577. package/dist/project/build-decision.d.ts.map +1 -0
  578. package/dist/project/build-decision.js +59 -0
  579. package/dist/project/build-decision.js.map +1 -0
  580. package/dist/project/config-resolver.d.ts +6 -4
  581. package/dist/project/config-resolver.d.ts.map +1 -1
  582. package/dist/project/config-resolver.js +158 -78
  583. package/dist/project/config-resolver.js.map +1 -1
  584. package/dist/project/declared-config.d.ts +42 -0
  585. package/dist/project/declared-config.d.ts.map +1 -0
  586. package/dist/project/declared-config.js +60 -0
  587. package/dist/project/declared-config.js.map +1 -0
  588. package/dist/project/dependency-analyzer.d.ts.map +1 -1
  589. package/dist/project/dependency-analyzer.js.map +1 -1
  590. package/dist/project/deploy-phases.d.ts +35 -0
  591. package/dist/project/deploy-phases.d.ts.map +1 -0
  592. package/dist/project/deploy-phases.js +75 -0
  593. package/dist/project/deploy-phases.js.map +1 -0
  594. package/dist/project/exec-in-group.d.ts +49 -0
  595. package/dist/project/exec-in-group.d.ts.map +1 -0
  596. package/dist/project/exec-in-group.js +100 -0
  597. package/dist/project/exec-in-group.js.map +1 -0
  598. package/dist/project/node-app-config.d.ts +101 -0
  599. package/dist/project/node-app-config.d.ts.map +1 -0
  600. package/dist/project/node-app-config.js +134 -0
  601. package/dist/project/node-app-config.js.map +1 -0
  602. package/dist/project/node-app-health.d.ts +76 -0
  603. package/dist/project/node-app-health.d.ts.map +1 -0
  604. package/dist/project/node-app-health.js +119 -0
  605. package/dist/project/node-app-health.js.map +1 -0
  606. package/dist/project/operator-account.d.ts +287 -0
  607. package/dist/project/operator-account.d.ts.map +1 -0
  608. package/dist/project/operator-account.js +373 -0
  609. package/dist/project/operator-account.js.map +1 -0
  610. package/dist/project/registry.d.ts +46 -8
  611. package/dist/project/registry.d.ts.map +1 -1
  612. package/dist/project/registry.js +238 -40
  613. package/dist/project/registry.js.map +1 -1
  614. package/dist/project/requirements-scanner.d.ts +1 -0
  615. package/dist/project/requirements-scanner.d.ts.map +1 -1
  616. package/dist/project/requirements-scanner.js +28 -3
  617. package/dist/project/requirements-scanner.js.map +1 -1
  618. package/dist/project/stack-env.d.ts +37 -0
  619. package/dist/project/stack-env.d.ts.map +1 -0
  620. package/dist/project/stack-env.js +85 -0
  621. package/dist/project/stack-env.js.map +1 -0
  622. package/dist/project/stand-containers.d.ts +4 -0
  623. package/dist/project/stand-containers.d.ts.map +1 -0
  624. package/dist/project/stand-containers.js +27 -0
  625. package/dist/project/stand-containers.js.map +1 -0
  626. package/dist/project/working-tree.d.ts +62 -0
  627. package/dist/project/working-tree.d.ts.map +1 -0
  628. package/dist/project/working-tree.js +101 -0
  629. package/dist/project/working-tree.js.map +1 -0
  630. package/dist/release/artifact-store.d.ts +360 -0
  631. package/dist/release/artifact-store.d.ts.map +1 -0
  632. package/dist/release/artifact-store.js +1040 -0
  633. package/dist/release/artifact-store.js.map +1 -0
  634. package/dist/release/attest-on-node.d.ts +92 -0
  635. package/dist/release/attest-on-node.d.ts.map +1 -0
  636. package/dist/release/attest-on-node.js +207 -0
  637. package/dist/release/attest-on-node.js.map +1 -0
  638. package/dist/release/attest.d.ts +135 -0
  639. package/dist/release/attest.d.ts.map +1 -0
  640. package/dist/release/attest.js +199 -0
  641. package/dist/release/attest.js.map +1 -0
  642. package/dist/release/build-run.d.ts +151 -0
  643. package/dist/release/build-run.d.ts.map +1 -0
  644. package/dist/release/build-run.js +609 -0
  645. package/dist/release/build-run.js.map +1 -0
  646. package/dist/release/builder.d.ts +57 -0
  647. package/dist/release/builder.d.ts.map +1 -0
  648. package/dist/release/builder.js +120 -0
  649. package/dist/release/builder.js.map +1 -0
  650. package/dist/release/daemon-omni.d.ts +41 -0
  651. package/dist/release/daemon-omni.d.ts.map +1 -0
  652. package/dist/release/daemon-omni.js +59 -0
  653. package/dist/release/daemon-omni.js.map +1 -0
  654. package/dist/release/definition-inputs.d.ts +34 -0
  655. package/dist/release/definition-inputs.d.ts.map +1 -0
  656. package/dist/release/definition-inputs.js +90 -0
  657. package/dist/release/definition-inputs.js.map +1 -0
  658. package/dist/release/delivered.d.ts +53 -0
  659. package/dist/release/delivered.d.ts.map +1 -0
  660. package/dist/release/delivered.js +81 -0
  661. package/dist/release/delivered.js.map +1 -0
  662. package/dist/release/layout.d.ts +61 -0
  663. package/dist/release/layout.d.ts.map +1 -0
  664. package/dist/release/layout.js +114 -0
  665. package/dist/release/layout.js.map +1 -0
  666. package/dist/release/load.d.ts +44 -0
  667. package/dist/release/load.d.ts.map +1 -0
  668. package/dist/release/load.js +84 -0
  669. package/dist/release/load.js.map +1 -0
  670. package/dist/release/manifest.d.ts +299 -0
  671. package/dist/release/manifest.d.ts.map +1 -0
  672. package/dist/release/manifest.js +176 -0
  673. package/dist/release/manifest.js.map +1 -0
  674. package/dist/release/migrations.d.ts +32 -0
  675. package/dist/release/migrations.d.ts.map +1 -0
  676. package/dist/release/migrations.js +51 -0
  677. package/dist/release/migrations.js.map +1 -0
  678. package/dist/release/publish.d.ts +71 -0
  679. package/dist/release/publish.d.ts.map +1 -0
  680. package/dist/release/publish.js +102 -0
  681. package/dist/release/publish.js.map +1 -0
  682. package/dist/release/store.d.ts +174 -0
  683. package/dist/release/store.d.ts.map +1 -0
  684. package/dist/release/store.js +257 -0
  685. package/dist/release/store.js.map +1 -0
  686. package/dist/services/alert.rpc-service.d.ts +11 -15
  687. package/dist/services/alert.rpc-service.d.ts.map +1 -1
  688. package/dist/services/alert.rpc-service.js +29 -14
  689. package/dist/services/alert.rpc-service.js.map +1 -1
  690. package/dist/services/alert.service.d.ts +96 -59
  691. package/dist/services/alert.service.d.ts.map +1 -1
  692. package/dist/services/alert.service.js +455 -70
  693. package/dist/services/alert.service.js.map +1 -1
  694. package/dist/services/anonymous-input.d.ts +26 -0
  695. package/dist/services/anonymous-input.d.ts.map +1 -0
  696. package/dist/services/anonymous-input.js +44 -0
  697. package/dist/services/anonymous-input.js.map +1 -0
  698. package/dist/services/audit.rpc-service.d.ts +27 -0
  699. package/dist/services/audit.rpc-service.d.ts.map +1 -0
  700. package/dist/services/audit.rpc-service.js +57 -0
  701. package/dist/services/audit.rpc-service.js.map +1 -0
  702. package/dist/services/audit.service.d.ts +165 -0
  703. package/dist/services/audit.service.d.ts.map +1 -0
  704. package/dist/services/audit.service.js +242 -0
  705. package/dist/services/audit.service.js.map +1 -0
  706. package/dist/services/auth-context.d.ts +21 -1
  707. package/dist/services/auth-context.d.ts.map +1 -1
  708. package/dist/services/auth-context.js +51 -5
  709. package/dist/services/auth-context.js.map +1 -1
  710. package/dist/services/auth.rpc-service.d.ts +0 -1
  711. package/dist/services/auth.rpc-service.d.ts.map +1 -1
  712. package/dist/services/auth.rpc-service.js +23 -9
  713. package/dist/services/auth.rpc-service.js.map +1 -1
  714. package/dist/services/auth.service.d.ts +47 -33
  715. package/dist/services/auth.service.d.ts.map +1 -1
  716. package/dist/services/auth.service.js +163 -24
  717. package/dist/services/auth.service.js.map +1 -1
  718. package/dist/services/backup-pipeline.d.ts +97 -0
  719. package/dist/services/backup-pipeline.d.ts.map +1 -0
  720. package/dist/services/backup-pipeline.js +286 -0
  721. package/dist/services/backup-pipeline.js.map +1 -0
  722. package/dist/services/backup-schedule.d.ts +69 -0
  723. package/dist/services/backup-schedule.d.ts.map +1 -0
  724. package/dist/services/backup-schedule.js +132 -0
  725. package/dist/services/backup-schedule.js.map +1 -0
  726. package/dist/services/backup.rpc-service.d.ts +31 -2
  727. package/dist/services/backup.rpc-service.d.ts.map +1 -1
  728. package/dist/services/backup.rpc-service.js +53 -0
  729. package/dist/services/backup.rpc-service.js.map +1 -1
  730. package/dist/services/backup.service.d.ts +458 -14
  731. package/dist/services/backup.service.d.ts.map +1 -1
  732. package/dist/services/backup.service.js +1368 -170
  733. package/dist/services/backup.service.js.map +1 -1
  734. package/dist/services/bundle-build-worker.d.ts +19 -0
  735. package/dist/services/bundle-build-worker.d.ts.map +1 -0
  736. package/dist/services/bundle-build-worker.js +40 -0
  737. package/dist/services/bundle-build-worker.js.map +1 -0
  738. package/dist/services/bundle-builder.d.ts +487 -0
  739. package/dist/services/bundle-builder.d.ts.map +1 -0
  740. package/dist/services/bundle-builder.js +1083 -0
  741. package/dist/services/bundle-builder.js.map +1 -0
  742. package/dist/services/bundle-worker-protocol.d.ts +19 -0
  743. package/dist/services/bundle-worker-protocol.d.ts.map +1 -0
  744. package/dist/services/bundle-worker-protocol.js +32 -0
  745. package/dist/services/bundle-worker-protocol.js.map +1 -0
  746. package/dist/services/deploy.rpc-service.d.ts +4 -1
  747. package/dist/services/deploy.rpc-service.d.ts.map +1 -1
  748. package/dist/services/deploy.rpc-service.js +10 -0
  749. package/dist/services/deploy.rpc-service.js.map +1 -1
  750. package/dist/services/deploy.service.d.ts +14 -24
  751. package/dist/services/deploy.service.d.ts.map +1 -1
  752. package/dist/services/deploy.service.js +46 -5
  753. package/dist/services/deploy.service.js.map +1 -1
  754. package/dist/services/discovery.rpc-service.d.ts +2 -1
  755. package/dist/services/discovery.rpc-service.d.ts.map +1 -1
  756. package/dist/services/discovery.rpc-service.js.map +1 -1
  757. package/dist/services/discovery.service.d.ts +32 -28
  758. package/dist/services/discovery.service.d.ts.map +1 -1
  759. package/dist/services/discovery.service.js +108 -133
  760. package/dist/services/discovery.service.js.map +1 -1
  761. package/dist/services/event-broadcaster.rpc-service.d.ts +36 -23
  762. package/dist/services/event-broadcaster.rpc-service.d.ts.map +1 -1
  763. package/dist/services/event-broadcaster.rpc-service.js +70 -32
  764. package/dist/services/event-broadcaster.rpc-service.js.map +1 -1
  765. package/dist/services/event-broadcaster.service.d.ts.map +1 -1
  766. package/dist/services/event-broadcaster.service.js.map +1 -1
  767. package/dist/services/fleet.rpc-service.d.ts +2 -1
  768. package/dist/services/fleet.rpc-service.d.ts.map +1 -1
  769. package/dist/services/fleet.rpc-service.js +3 -1
  770. package/dist/services/fleet.rpc-service.js.map +1 -1
  771. package/dist/services/fleet.service.d.ts +32 -34
  772. package/dist/services/fleet.service.d.ts.map +1 -1
  773. package/dist/services/fleet.service.js +66 -10
  774. package/dist/services/fleet.service.js.map +1 -1
  775. package/dist/services/health-check.rpc-service.d.ts +2 -1
  776. package/dist/services/health-check.rpc-service.d.ts.map +1 -1
  777. package/dist/services/health-check.rpc-service.js.map +1 -1
  778. package/dist/services/health-check.service.d.ts +50 -36
  779. package/dist/services/health-check.service.d.ts.map +1 -1
  780. package/dist/services/health-check.service.js +179 -182
  781. package/dist/services/health-check.service.js.map +1 -1
  782. package/dist/services/infrastructure.rpc-service.d.ts +297 -2
  783. package/dist/services/infrastructure.rpc-service.d.ts.map +1 -1
  784. package/dist/services/infrastructure.rpc-service.js +523 -11
  785. package/dist/services/infrastructure.rpc-service.js.map +1 -1
  786. package/dist/services/kubernetes.rpc-service.d.ts +2 -1
  787. package/dist/services/kubernetes.rpc-service.d.ts.map +1 -1
  788. package/dist/services/kubernetes.rpc-service.js.map +1 -1
  789. package/dist/services/kubernetes.service.d.ts +5 -39
  790. package/dist/services/kubernetes.service.d.ts.map +1 -1
  791. package/dist/services/kubernetes.service.js +29 -5
  792. package/dist/services/kubernetes.service.js.map +1 -1
  793. package/dist/services/local-bundle.d.ts +151 -0
  794. package/dist/services/local-bundle.d.ts.map +1 -0
  795. package/dist/services/local-bundle.js +233 -0
  796. package/dist/services/local-bundle.js.map +1 -0
  797. package/dist/services/lockout.d.ts +28 -0
  798. package/dist/services/lockout.d.ts.map +1 -0
  799. package/dist/services/lockout.js +41 -0
  800. package/dist/services/lockout.js.map +1 -0
  801. package/dist/services/log-collector.rpc-service.d.ts +16 -0
  802. package/dist/services/log-collector.rpc-service.d.ts.map +1 -1
  803. package/dist/services/log-collector.rpc-service.js +16 -0
  804. package/dist/services/log-collector.rpc-service.js.map +1 -1
  805. package/dist/services/log-collector.service.d.ts +172 -54
  806. package/dist/services/log-collector.service.d.ts.map +1 -1
  807. package/dist/services/log-collector.service.js +492 -102
  808. package/dist/services/log-collector.service.js.map +1 -1
  809. package/dist/services/log-retention.d.ts +50 -0
  810. package/dist/services/log-retention.d.ts.map +1 -0
  811. package/dist/services/log-retention.js +52 -0
  812. package/dist/services/log-retention.js.map +1 -0
  813. package/dist/services/maintenance-lock.d.ts +162 -0
  814. package/dist/services/maintenance-lock.d.ts.map +1 -0
  815. package/dist/services/maintenance-lock.js +316 -0
  816. package/dist/services/maintenance-lock.js.map +1 -0
  817. package/dist/services/master-address.d.ts +107 -0
  818. package/dist/services/master-address.d.ts.map +1 -0
  819. package/dist/services/master-address.js +154 -0
  820. package/dist/services/master-address.js.map +1 -0
  821. package/dist/services/migration-output.d.ts +16 -0
  822. package/dist/services/migration-output.d.ts.map +1 -0
  823. package/dist/services/migration-output.js +22 -0
  824. package/dist/services/migration-output.js.map +1 -0
  825. package/dist/services/netron-telemetry-transport.d.ts.map +1 -1
  826. package/dist/services/netron-telemetry-transport.js +0 -4
  827. package/dist/services/netron-telemetry-transport.js.map +1 -1
  828. package/dist/services/node-deploy-lease.d.ts +154 -0
  829. package/dist/services/node-deploy-lease.d.ts.map +1 -0
  830. package/dist/services/node-deploy-lease.js +386 -0
  831. package/dist/services/node-deploy-lease.js.map +1 -0
  832. package/dist/services/node-health.repository.d.ts +63 -4
  833. package/dist/services/node-health.repository.d.ts.map +1 -1
  834. package/dist/services/node-health.repository.js +108 -67
  835. package/dist/services/node-health.repository.js.map +1 -1
  836. package/dist/services/node-manager.rpc-service.d.ts +355 -10
  837. package/dist/services/node-manager.rpc-service.d.ts.map +1 -1
  838. package/dist/services/node-manager.rpc-service.js +730 -34
  839. package/dist/services/node-manager.rpc-service.js.map +1 -1
  840. package/dist/services/node-manager.service.d.ts +267 -95
  841. package/dist/services/node-manager.service.d.ts.map +1 -1
  842. package/dist/services/node-manager.service.js +806 -104
  843. package/dist/services/node-manager.service.js.map +1 -1
  844. package/dist/services/node-upgrade.d.ts +129 -0
  845. package/dist/services/node-upgrade.d.ts.map +1 -0
  846. package/dist/services/node-upgrade.js +161 -0
  847. package/dist/services/node-upgrade.js.map +1 -0
  848. package/dist/services/node-upgrade.service.d.ts +232 -0
  849. package/dist/services/node-upgrade.service.d.ts.map +1 -0
  850. package/dist/services/node-upgrade.service.js +468 -0
  851. package/dist/services/node-upgrade.service.js.map +1 -0
  852. package/dist/services/pipeline.rpc-service.d.ts +2 -1
  853. package/dist/services/pipeline.rpc-service.d.ts.map +1 -1
  854. package/dist/services/pipeline.rpc-service.js.map +1 -1
  855. package/dist/services/pipeline.service.d.ts +30 -46
  856. package/dist/services/pipeline.service.d.ts.map +1 -1
  857. package/dist/services/pipeline.service.js +125 -11
  858. package/dist/services/pipeline.service.js.map +1 -1
  859. package/dist/services/project.rpc-service.d.ts +137 -5
  860. package/dist/services/project.rpc-service.d.ts.map +1 -1
  861. package/dist/services/project.rpc-service.js +258 -22
  862. package/dist/services/project.rpc-service.js.map +1 -1
  863. package/dist/services/project.service.d.ts +680 -9
  864. package/dist/services/project.service.d.ts.map +1 -1
  865. package/dist/services/project.service.js +2914 -293
  866. package/dist/services/project.service.js.map +1 -1
  867. package/dist/services/redeploy-decision.d.ts +176 -0
  868. package/dist/services/redeploy-decision.d.ts.map +1 -0
  869. package/dist/services/redeploy-decision.js +142 -0
  870. package/dist/services/redeploy-decision.js.map +1 -0
  871. package/dist/services/release.rpc-service.d.ts +159 -0
  872. package/dist/services/release.rpc-service.d.ts.map +1 -0
  873. package/dist/services/release.rpc-service.js +395 -0
  874. package/dist/services/release.rpc-service.js.map +1 -0
  875. package/dist/services/release.service.d.ts +114 -0
  876. package/dist/services/release.service.d.ts.map +1 -0
  877. package/dist/services/release.service.js +208 -0
  878. package/dist/services/release.service.js.map +1 -0
  879. package/dist/services/remote-deployer.service.d.ts +569 -8
  880. package/dist/services/remote-deployer.service.d.ts.map +1 -1
  881. package/dist/services/remote-deployer.service.js +1618 -169
  882. package/dist/services/remote-deployer.service.js.map +1 -1
  883. package/dist/services/remote-ops.service.d.ts +51 -9
  884. package/dist/services/remote-ops.service.d.ts.map +1 -1
  885. package/dist/services/remote-ops.service.js +235 -39
  886. package/dist/services/remote-ops.service.js.map +1 -1
  887. package/dist/services/remote-provisioner.d.ts +122 -0
  888. package/dist/services/remote-provisioner.d.ts.map +1 -0
  889. package/dist/services/remote-provisioner.js +258 -0
  890. package/dist/services/remote-provisioner.js.map +1 -0
  891. package/dist/services/secrets.rpc-service.d.ts +51 -2
  892. package/dist/services/secrets.rpc-service.d.ts.map +1 -1
  893. package/dist/services/secrets.rpc-service.js +81 -2
  894. package/dist/services/secrets.rpc-service.js.map +1 -1
  895. package/dist/services/secrets.service.d.ts +108 -9
  896. package/dist/services/secrets.service.d.ts.map +1 -1
  897. package/dist/services/secrets.service.js +313 -48
  898. package/dist/services/secrets.service.js.map +1 -1
  899. package/dist/services/slave-storage.service.d.ts +107 -0
  900. package/dist/services/slave-storage.service.d.ts.map +1 -1
  901. package/dist/services/slave-storage.service.js +171 -2
  902. package/dist/services/slave-storage.service.js.map +1 -1
  903. package/dist/services/static-carry-forward.d.ts +85 -0
  904. package/dist/services/static-carry-forward.d.ts.map +1 -0
  905. package/dist/services/static-carry-forward.js +159 -0
  906. package/dist/services/static-carry-forward.js.map +1 -0
  907. package/dist/services/sync-policy.d.ts +167 -0
  908. package/dist/services/sync-policy.d.ts.map +1 -0
  909. package/dist/services/sync-policy.js +161 -0
  910. package/dist/services/sync-policy.js.map +1 -0
  911. package/dist/services/sync-summary.d.ts +39 -0
  912. package/dist/services/sync-summary.d.ts.map +1 -0
  913. package/dist/services/sync-summary.js +62 -0
  914. package/dist/services/sync-summary.js.map +1 -0
  915. package/dist/services/sync.rpc-service.d.ts +19 -7
  916. package/dist/services/sync.rpc-service.d.ts.map +1 -1
  917. package/dist/services/sync.rpc-service.js +36 -5
  918. package/dist/services/sync.rpc-service.js.map +1 -1
  919. package/dist/services/sync.service.d.ts +350 -18
  920. package/dist/services/sync.service.d.ts.map +1 -1
  921. package/dist/services/sync.service.js +864 -116
  922. package/dist/services/sync.service.js.map +1 -1
  923. package/dist/services/system-info.rpc-service.d.ts +2 -1
  924. package/dist/services/system-info.rpc-service.d.ts.map +1 -1
  925. package/dist/services/system-info.rpc-service.js.map +1 -1
  926. package/dist/services/system-info.service.d.ts +56 -68
  927. package/dist/services/system-info.service.d.ts.map +1 -1
  928. package/dist/services/system-info.service.js +240 -127
  929. package/dist/services/system-info.service.js.map +1 -1
  930. package/dist/services/telemetry.rpc-service.d.ts.map +1 -1
  931. package/dist/services/telemetry.rpc-service.js +9 -3
  932. package/dist/services/telemetry.rpc-service.js.map +1 -1
  933. package/dist/services/telemetry.service.d.ts +4 -2
  934. package/dist/services/telemetry.service.d.ts.map +1 -1
  935. package/dist/services/telemetry.service.js +4 -2
  936. package/dist/services/telemetry.service.js.map +1 -1
  937. package/dist/services/trace-collector.rpc-service.d.ts +2 -1
  938. package/dist/services/trace-collector.rpc-service.d.ts.map +1 -1
  939. package/dist/services/trace-collector.rpc-service.js.map +1 -1
  940. package/dist/services/trace-collector.service.d.ts +2 -41
  941. package/dist/services/trace-collector.service.d.ts.map +1 -1
  942. package/dist/services/trace-collector.service.js +27 -7
  943. package/dist/services/trace-collector.service.js.map +1 -1
  944. package/dist/services/wait-for-postgres.d.ts +28 -0
  945. package/dist/services/wait-for-postgres.d.ts.map +1 -0
  946. package/dist/services/wait-for-postgres.js +77 -0
  947. package/dist/services/wait-for-postgres.js.map +1 -0
  948. package/dist/shared/alert-expression.d.ts +60 -0
  949. package/dist/shared/alert-expression.d.ts.map +1 -0
  950. package/dist/shared/alert-expression.js +124 -0
  951. package/dist/shared/alert-expression.js.map +1 -0
  952. package/dist/shared/audit-outcome.d.ts +39 -0
  953. package/dist/shared/audit-outcome.d.ts.map +1 -0
  954. package/dist/shared/audit-outcome.js +45 -0
  955. package/dist/shared/audit-outcome.js.map +1 -0
  956. package/dist/shared/build-freshness.d.ts +59 -0
  957. package/dist/shared/build-freshness.d.ts.map +1 -0
  958. package/dist/shared/build-freshness.js +113 -0
  959. package/dist/shared/build-freshness.js.map +1 -0
  960. package/dist/shared/build-stamp.d.ts +26 -0
  961. package/dist/shared/build-stamp.d.ts.map +1 -0
  962. package/dist/shared/build-stamp.js +36 -0
  963. package/dist/shared/build-stamp.js.map +1 -0
  964. package/dist/shared/describe-error.d.ts +22 -0
  965. package/dist/shared/describe-error.d.ts.map +1 -0
  966. package/dist/shared/describe-error.js +39 -0
  967. package/dist/shared/describe-error.js.map +1 -0
  968. package/dist/shared/dto/alerts.d.ts +101 -0
  969. package/dist/shared/dto/alerts.d.ts.map +1 -0
  970. package/dist/shared/dto/alerts.js +9 -0
  971. package/dist/shared/dto/alerts.js.map +1 -0
  972. package/dist/shared/dto/auth.d.ts +49 -0
  973. package/dist/shared/dto/auth.d.ts.map +1 -0
  974. package/dist/shared/dto/auth.js +14 -0
  975. package/dist/shared/dto/auth.js.map +1 -0
  976. package/dist/shared/dto/backups.d.ts +16 -0
  977. package/dist/shared/dto/backups.d.ts.map +1 -0
  978. package/dist/shared/dto/backups.js +9 -0
  979. package/dist/shared/dto/backups.js.map +1 -0
  980. package/dist/shared/dto/deploy.d.ts +28 -0
  981. package/dist/shared/dto/deploy.d.ts.map +1 -0
  982. package/dist/shared/dto/deploy.js +8 -0
  983. package/dist/shared/dto/deploy.js.map +1 -0
  984. package/dist/shared/dto/discovery.d.ts +24 -0
  985. package/dist/shared/dto/discovery.d.ts.map +1 -0
  986. package/dist/shared/dto/discovery.js +9 -0
  987. package/dist/shared/dto/discovery.js.map +1 -0
  988. package/dist/shared/dto/fleet.d.ts +35 -0
  989. package/dist/shared/dto/fleet.d.ts.map +1 -0
  990. package/dist/shared/dto/fleet.js +8 -0
  991. package/dist/shared/dto/fleet.js.map +1 -0
  992. package/dist/shared/dto/health.d.ts +26 -0
  993. package/dist/shared/dto/health.d.ts.map +1 -0
  994. package/dist/shared/dto/health.js +9 -0
  995. package/dist/shared/dto/health.js.map +1 -0
  996. package/dist/shared/dto/kubernetes.d.ts +44 -0
  997. package/dist/shared/dto/kubernetes.d.ts.map +1 -0
  998. package/dist/shared/dto/kubernetes.js +9 -0
  999. package/dist/shared/dto/kubernetes.js.map +1 -0
  1000. package/dist/shared/dto/logs.d.ts +77 -0
  1001. package/dist/shared/dto/logs.d.ts.map +1 -0
  1002. package/dist/shared/dto/logs.js +10 -0
  1003. package/dist/shared/dto/logs.js.map +1 -0
  1004. package/dist/shared/dto/metrics.d.ts +65 -0
  1005. package/dist/shared/dto/metrics.d.ts.map +1 -0
  1006. package/dist/shared/dto/metrics.js +17 -0
  1007. package/dist/shared/dto/metrics.js.map +1 -0
  1008. package/dist/shared/dto/nodes.d.ts +347 -0
  1009. package/dist/shared/dto/nodes.d.ts.map +1 -0
  1010. package/dist/shared/dto/nodes.js +8 -0
  1011. package/dist/shared/dto/nodes.js.map +1 -0
  1012. package/dist/shared/dto/pipelines.d.ts +53 -0
  1013. package/dist/shared/dto/pipelines.d.ts.map +1 -0
  1014. package/dist/shared/dto/pipelines.js +9 -0
  1015. package/dist/shared/dto/pipelines.js.map +1 -0
  1016. package/dist/shared/dto/project.d.ts +143 -9
  1017. package/dist/shared/dto/project.d.ts.map +1 -1
  1018. package/dist/shared/dto/services.d.ts +855 -7
  1019. package/dist/shared/dto/services.d.ts.map +1 -1
  1020. package/dist/shared/dto/system-info.d.ts +93 -0
  1021. package/dist/shared/dto/system-info.d.ts.map +1 -0
  1022. package/dist/shared/dto/system-info.js +9 -0
  1023. package/dist/shared/dto/system-info.js.map +1 -0
  1024. package/dist/shared/dto/traces.d.ts +49 -0
  1025. package/dist/shared/dto/traces.d.ts.map +1 -0
  1026. package/dist/shared/dto/traces.js +9 -0
  1027. package/dist/shared/dto/traces.js.map +1 -0
  1028. package/dist/shared/env-config.d.ts +74 -0
  1029. package/dist/shared/env-config.d.ts.map +1 -0
  1030. package/dist/shared/env-config.js +99 -0
  1031. package/dist/shared/env-config.js.map +1 -0
  1032. package/dist/shared/events.d.ts +45 -39
  1033. package/dist/shared/events.d.ts.map +1 -1
  1034. package/dist/shared/events.js +6 -0
  1035. package/dist/shared/events.js.map +1 -1
  1036. package/dist/shared/format.d.ts +1 -1
  1037. package/dist/shared/format.d.ts.map +1 -1
  1038. package/dist/shared/format.js +4 -0
  1039. package/dist/shared/format.js.map +1 -1
  1040. package/dist/shared/local-infra-host.d.ts +28 -0
  1041. package/dist/shared/local-infra-host.d.ts.map +1 -0
  1042. package/dist/shared/local-infra-host.js +28 -0
  1043. package/dist/shared/local-infra-host.js.map +1 -0
  1044. package/dist/shared/node-check.d.ts +37 -0
  1045. package/dist/shared/node-check.d.ts.map +1 -0
  1046. package/dist/shared/node-check.js +41 -0
  1047. package/dist/shared/node-check.js.map +1 -0
  1048. package/dist/shared/paths.d.ts +29 -0
  1049. package/dist/shared/paths.d.ts.map +1 -0
  1050. package/dist/shared/paths.js +54 -0
  1051. package/dist/shared/paths.js.map +1 -0
  1052. package/dist/shared/pnpm.d.ts +29 -0
  1053. package/dist/shared/pnpm.d.ts.map +1 -0
  1054. package/dist/shared/pnpm.js +57 -0
  1055. package/dist/shared/pnpm.js.map +1 -0
  1056. package/dist/shared/private-files.d.ts +29 -0
  1057. package/dist/shared/private-files.d.ts.map +1 -0
  1058. package/dist/shared/private-files.js +109 -0
  1059. package/dist/shared/private-files.js.map +1 -0
  1060. package/dist/shared/redact-env.d.ts +24 -0
  1061. package/dist/shared/redact-env.d.ts.map +1 -0
  1062. package/dist/shared/redact-env.js +36 -0
  1063. package/dist/shared/redact-env.js.map +1 -0
  1064. package/dist/shared/release-reading.d.ts +85 -0
  1065. package/dist/shared/release-reading.d.ts.map +1 -0
  1066. package/dist/shared/release-reading.js +81 -0
  1067. package/dist/shared/release-reading.js.map +1 -0
  1068. package/dist/shared/roles.d.ts +58 -3
  1069. package/dist/shared/roles.d.ts.map +1 -1
  1070. package/dist/shared/roles.js +55 -0
  1071. package/dist/shared/roles.js.map +1 -1
  1072. package/dist/shared/rpcauth.d.ts +13 -0
  1073. package/dist/shared/rpcauth.d.ts.map +1 -0
  1074. package/dist/shared/rpcauth.js +23 -0
  1075. package/dist/shared/rpcauth.js.map +1 -0
  1076. package/dist/shared/shell-escape.d.ts +9 -0
  1077. package/dist/shared/shell-escape.d.ts.map +1 -0
  1078. package/dist/shared/shell-escape.js +11 -0
  1079. package/dist/shared/shell-escape.js.map +1 -0
  1080. package/dist/shared/sync-reading.d.ts +45 -0
  1081. package/dist/shared/sync-reading.d.ts.map +1 -0
  1082. package/dist/shared/sync-reading.js +42 -0
  1083. package/dist/shared/sync-reading.js.map +1 -0
  1084. package/dist/shared/tokens.d.ts +19 -0
  1085. package/dist/shared/tokens.d.ts.map +1 -1
  1086. package/dist/shared/tokens.js +28 -0
  1087. package/dist/shared/tokens.js.map +1 -1
  1088. package/dist/webapp/webapp.service.d.ts +33 -2
  1089. package/dist/webapp/webapp.service.d.ts.map +1 -1
  1090. package/dist/webapp/webapp.service.js +143 -31
  1091. package/dist/webapp/webapp.service.js.map +1 -1
  1092. package/dist/workers/health-monitor-process.d.ts.map +1 -1
  1093. package/dist/workers/health-monitor-process.js +10 -0
  1094. package/dist/workers/health-monitor-process.js.map +1 -1
  1095. package/dist/workers/health-monitor.service.d.ts +45 -3
  1096. package/dist/workers/health-monitor.service.d.ts.map +1 -1
  1097. package/dist/workers/health-monitor.service.js +156 -26
  1098. package/dist/workers/health-monitor.service.js.map +1 -1
  1099. package/dist/workers/system-worker-manager.d.ts +43 -0
  1100. package/dist/workers/system-worker-manager.d.ts.map +1 -1
  1101. package/dist/workers/system-worker-manager.js +150 -3
  1102. package/dist/workers/system-worker-manager.js.map +1 -1
  1103. package/dist/workers/types.d.ts +11 -2
  1104. package/dist/workers/types.d.ts.map +1 -1
  1105. package/package.json +64 -29
  1106. package/webapp/dist/BUILD.json +5 -0
  1107. package/webapp/dist/assets/FormControlLabel-BrH2diyl.js +1 -0
  1108. package/webapp/dist/assets/Grid-BZ7xhoHQ.js +1 -0
  1109. package/webapp/dist/assets/Switch-DGSAxQPM.js +1 -0
  1110. package/webapp/dist/assets/alerts-sXqHJbsa.js +1 -0
  1111. package/webapp/dist/assets/apps-B_Ah8Flh.js +1 -0
  1112. package/webapp/dist/assets/audit-YXihwyui.js +1 -0
  1113. package/webapp/dist/assets/client-CGbBvy9k.js +1 -0
  1114. package/webapp/dist/assets/constants-veKGFkC6.js +1 -0
  1115. package/webapp/dist/assets/containers-Di4NOTLf.js +2 -0
  1116. package/webapp/dist/assets/daemon-memory-CCPZ4cV_.js +1 -0
  1117. package/webapp/dist/assets/dashboard-Cfuf95N3.js +1 -0
  1118. package/webapp/dist/assets/dashboard-builder-CowaYb-M.js +1 -0
  1119. package/webapp/dist/assets/deploy-release-dialog-D6C-XcKY.js +1 -0
  1120. package/webapp/dist/assets/deployments-ChTq5kCw.js +1 -0
  1121. package/webapp/dist/assets/detail-BsGWv7Dj.js +1 -0
  1122. package/webapp/dist/assets/detail-CyVf_cw5.js +2 -0
  1123. package/webapp/dist/assets/detail-D027j6aO.js +8 -0
  1124. package/webapp/dist/assets/formatters-BqjL73N3.js +1 -0
  1125. package/webapp/dist/assets/index-B8AHIPc-.css +1 -0
  1126. package/webapp/dist/assets/index-DiZbrwxT.js +414 -0
  1127. package/webapp/dist/assets/logs-L2b8Oqmo.js +12 -0
  1128. package/webapp/dist/assets/maplibre-gl-MlmG2kbn.js +801 -0
  1129. package/webapp/dist/assets/metrics-CQgjz2Lz.js +2 -0
  1130. package/webapp/dist/assets/nodes-Nwia2Ty6.js +7 -0
  1131. package/webapp/dist/assets/pipelines-DCu_rXOr.js +4 -0
  1132. package/webapp/dist/assets/projects-Bb4aV76h.js +1 -0
  1133. package/webapp/dist/assets/react-CwJFpaho.js +1 -0
  1134. package/webapp/dist/assets/react-apexcharts.esm-Dy0Fez42.js +1956 -0
  1135. package/webapp/dist/assets/release-bits-Dj4EOhhb.js +1 -0
  1136. package/webapp/dist/assets/release-build-panel-D6OCVlhT.js +1 -0
  1137. package/webapp/dist/assets/releases-CnAt3yO6.js +2 -0
  1138. package/webapp/dist/assets/rolldown-runtime-hePW80VL.js +1 -0
  1139. package/webapp/dist/assets/settings-DU-mjiwb.js +1 -0
  1140. package/webapp/dist/assets/settled-pair-C7SjZuns.js +1 -0
  1141. package/webapp/dist/assets/sign-in-Bcvjrr4C.js +1 -0
  1142. package/webapp/dist/assets/stacks-B-e8p50I.js +1 -0
  1143. package/webapp/dist/assets/sync-reading-C3PnJ7AX.js +1 -0
  1144. package/webapp/dist/assets/system-info-BtE3dgfw.js +1 -0
  1145. package/webapp/dist/assets/topology-Cwch-uF1.js +12 -0
  1146. package/webapp/dist/assets/topology-DLioOiRN.css +1 -0
  1147. package/webapp/dist/assets/traces-BbShR-4P.js +1 -0
  1148. package/webapp/dist/assets/use-stack-context-CZUgPq88.js +1 -0
  1149. package/webapp/dist/index.html +6 -7
  1150. package/dist/commands/dev.d.ts +0 -23
  1151. package/dist/commands/dev.d.ts.map +0 -1
  1152. package/dist/commands/dev.js +0 -76
  1153. package/dist/commands/dev.js.map +0 -1
  1154. package/dist/config/define-app.d.ts +0 -6
  1155. package/dist/config/define-app.d.ts.map +0 -1
  1156. package/dist/config/define-app.js +0 -38
  1157. package/dist/config/define-app.js.map +0 -1
  1158. package/dist/infrastructure/gateway-generator.d.ts +0 -63
  1159. package/dist/infrastructure/gateway-generator.d.ts.map +0 -1
  1160. package/dist/infrastructure/gateway-generator.js +0 -253
  1161. package/dist/infrastructure/gateway-generator.js.map +0 -1
  1162. package/dist/monitoring/health.service.d.ts +0 -14
  1163. package/dist/monitoring/health.service.d.ts.map +0 -1
  1164. package/dist/monitoring/health.service.js +0 -50
  1165. package/dist/monitoring/health.service.js.map +0 -1
  1166. package/dist/monitoring/metrics.service.d.ts +0 -14
  1167. package/dist/monitoring/metrics.service.d.ts.map +0 -1
  1168. package/dist/monitoring/metrics.service.js +0 -40
  1169. package/dist/monitoring/metrics.service.js.map +0 -1
  1170. package/dist/orchestrator/bootstrap-runner.d.ts +0 -15
  1171. package/dist/orchestrator/bootstrap-runner.d.ts.map +0 -1
  1172. package/dist/orchestrator/bootstrap-runner.js +0 -152
  1173. package/dist/orchestrator/bootstrap-runner.js.map +0 -1
  1174. package/dist/orchestrator/topology-proxy.d.ts +0 -65
  1175. package/dist/orchestrator/topology-proxy.d.ts.map +0 -1
  1176. package/dist/orchestrator/topology-proxy.js +0 -209
  1177. package/dist/orchestrator/topology-proxy.js.map +0 -1
  1178. package/dist/orchestrator/topology-relay.d.ts +0 -111
  1179. package/dist/orchestrator/topology-relay.d.ts.map +0 -1
  1180. package/dist/orchestrator/topology-relay.js +0 -255
  1181. package/dist/orchestrator/topology-relay.js.map +0 -1
  1182. package/dist/services/auth-middleware.d.ts +0 -47
  1183. package/dist/services/auth-middleware.d.ts.map +0 -1
  1184. package/dist/services/auth-middleware.js +0 -158
  1185. package/dist/services/auth-middleware.js.map +0 -1
  1186. package/dist/services/environment.rpc-service.d.ts +0 -22
  1187. package/dist/services/environment.rpc-service.d.ts.map +0 -1
  1188. package/dist/services/environment.rpc-service.js +0 -65
  1189. package/dist/services/environment.rpc-service.js.map +0 -1
  1190. package/dist/services/environment.service.d.ts +0 -87
  1191. package/dist/services/environment.service.d.ts.map +0 -1
  1192. package/dist/services/environment.service.js +0 -220
  1193. package/dist/services/environment.service.js.map +0 -1
  1194. package/dist/services/metrics-collector.service.d.ts +0 -103
  1195. package/dist/services/metrics-collector.service.d.ts.map +0 -1
  1196. package/dist/services/metrics-collector.service.js +0 -235
  1197. package/dist/services/metrics-collector.service.js.map +0 -1
  1198. package/dist/services/metrics.rpc-service.d.ts +0 -41
  1199. package/dist/services/metrics.rpc-service.d.ts.map +0 -1
  1200. package/dist/services/metrics.rpc-service.js +0 -90
  1201. package/dist/services/metrics.rpc-service.js.map +0 -1
  1202. package/dist/services/metrics.service.d.ts +0 -72
  1203. package/dist/services/metrics.service.d.ts.map +0 -1
  1204. package/dist/services/metrics.service.js +0 -185
  1205. package/dist/services/metrics.service.js.map +0 -1
  1206. package/dist/supervisor/omnitron-supervisor.d.ts +0 -21
  1207. package/dist/supervisor/omnitron-supervisor.d.ts.map +0 -1
  1208. package/dist/supervisor/omnitron-supervisor.js +0 -43
  1209. package/dist/supervisor/omnitron-supervisor.js.map +0 -1
  1210. package/dist/webapp/static-server.d.ts +0 -24
  1211. package/dist/webapp/static-server.d.ts.map +0 -1
  1212. package/dist/webapp/static-server.js +0 -152
  1213. package/dist/webapp/static-server.js.map +0 -1
  1214. package/webapp/dist/assets/Chip-BySxTjyN.js +0 -1
  1215. package/webapp/dist/assets/DialogContentText-YQWiAVrU.js +0 -1
  1216. package/webapp/dist/assets/DialogTitle-CDkgUIwp.js +0 -1
  1217. package/webapp/dist/assets/FormControlLabel-Bu3CCsBM.js +0 -1
  1218. package/webapp/dist/assets/Grid-BFY8hAzL.js +0 -1
  1219. package/webapp/dist/assets/InputAdornment-_Dayk-3y.js +0 -3
  1220. package/webapp/dist/assets/Skeleton-BGj6Sw8m.js +0 -37
  1221. package/webapp/dist/assets/Switch-CA6682vv.js +0 -1
  1222. package/webapp/dist/assets/TableContainer-4epq10KJ.js +0 -1
  1223. package/webapp/dist/assets/TextField-C73KHMVN.js +0 -1
  1224. package/webapp/dist/assets/alerts-JEQALS7Y.js +0 -1
  1225. package/webapp/dist/assets/apps-DMPysEKB.js +0 -1
  1226. package/webapp/dist/assets/client-DlGPter5.js +0 -110
  1227. package/webapp/dist/assets/constants-DqFQJ27U.js +0 -1
  1228. package/webapp/dist/assets/containers-CvkC7do8.js +0 -1
  1229. package/webapp/dist/assets/dashboard-C0Fre9pC.js +0 -1
  1230. package/webapp/dist/assets/dashboard-builder-DAghqdpE.js +0 -1
  1231. package/webapp/dist/assets/deployments-Cg1lI1uT.js +0 -1
  1232. package/webapp/dist/assets/detail-CyFtEson.js +0 -1
  1233. package/webapp/dist/assets/detail-DqoSGKT1.js +0 -8
  1234. package/webapp/dist/assets/formatters-J3Xqn6BQ.js +0 -1
  1235. package/webapp/dist/assets/icons-ynIWYfBP.js +0 -3
  1236. package/webapp/dist/assets/index-CHVXOsTS.css +0 -1
  1237. package/webapp/dist/assets/index-vKrDmEA_.js +0 -259
  1238. package/webapp/dist/assets/isMuiElement-BRL3YuCT.js +0 -1
  1239. package/webapp/dist/assets/logs-IovIij1C.js +0 -11
  1240. package/webapp/dist/assets/metrics-Dufwtwjq.js +0 -1
  1241. package/webapp/dist/assets/nodes-Brvu8W3j.js +0 -7
  1242. package/webapp/dist/assets/pipelines-t40X4MuC.js +0 -4
  1243. package/webapp/dist/assets/prop-types-CvVtHxp4.js +0 -1
  1244. package/webapp/dist/assets/react-apexcharts.esm-CLO6gala.js +0 -954
  1245. package/webapp/dist/assets/settings-DTAIST3v.js +0 -1
  1246. package/webapp/dist/assets/sign-in-DQEAkJum.js +0 -1
  1247. package/webapp/dist/assets/stacks-BjM1f45N.js +0 -1
  1248. package/webapp/dist/assets/system-info-BQSbSEK-.js +0 -1
  1249. package/webapp/dist/assets/topology-6N8-Az_Z.js +0 -12
  1250. package/webapp/dist/assets/topology-CHpVij2M.css +0 -1
  1251. package/webapp/dist/assets/traces-DVXyb-4m.js +0 -1
  1252. package/webapp/dist/assets/use-stack-context-CbE8fxBA.js +0 -1
  1253. /package/webapp/dist/assets/{fs-uP8eiZAR.js → fs-Ccl3sAcp.js} +0 -0
@@ -11,7 +11,13 @@
11
11
  *
12
12
  * Classic mode:
13
13
  * - child_process.fork() with manual exit handling (legacy compat)
14
- * - Real CPU/memory metrics via pidusage
14
+ *
15
+ * Metrics for both modes come from one `ps -o pid=,rss=,%cpu=` call covering
16
+ * every managed pid — supervisor children and pool workers alike. This line
17
+ * used to say "real CPU/memory metrics via pidusage"; `pidusage` is not a
18
+ * dependency of this package and appears nowhere else in the source. The
19
+ * sentence outlived the code it described, in the one area where a stale
20
+ * claim is hardest to catch: metrics that are wrong still look like metrics.
15
21
  *
16
22
  * Architecture:
17
23
  * pm.supervisor({ children: [{ process: bootstrap-process.js, ... }] })
@@ -21,17 +27,59 @@
21
27
  * → Two Netron instances: PM management plane + app HTTP data plane
22
28
  */
23
29
  import fs from 'node:fs';
30
+ import { readDeclaredConfig, withDeclaredConfig } from '../project/declared-config.js';
24
31
  import path from 'node:path';
25
32
  import { fileURLToPath } from 'node:url';
26
- import { PoolStrategy, SupervisionStrategy, RestartDecision, } from '@omnitron-dev/titan-pm';
33
+ import { PoolStrategy, SupervisionStrategy, ProcessLifecycleQueue, LifecyclePreempted, computeBackoff, isAlive, discoverManagedProcesses, } from '@omnitron-dev/titan-pm';
27
34
  import { EventEmitter } from '@omnitron-dev/eventemitter';
28
35
  import { AppHandle } from './app-handle.js';
36
+ import { parsePsLine, parsePsBatch } from './ps-metrics.js';
29
37
  import { launchClassic } from './classic-launcher.js';
30
38
  import { resolveStartupOrder, resolveShutdownOrder } from './dependency-resolver.js';
31
39
  import { buildRestartPolicy } from '../supervisor/restart-policy.js';
40
+ /**
41
+ * How long to wait for a pool worker to finish exposing its `@Service`
42
+ * classes before giving up on `topology.expose`. A worker reports ready
43
+ * before its Application starts, and exposure happens during start, so the
44
+ * gap is a real one — measured at ~7s for a worker that opens database
45
+ * connections first. The consumer that needs the service starts right after
46
+ * this returns, so the wait is paid once at boot and buys a feature that
47
+ * otherwise stays off until the next restart.
48
+ */
49
+ /**
50
+ * `childName` is sometimes bare ('http') and sometimes already carries the app
51
+ * ('acme/dev/pricing/http'), depending on which supervisor path raised the
52
+ * event. Prefixing unconditionally produced
53
+ * `acme/dev/pricing/acme/dev/pricing/http` in the log file.
54
+ */
55
+ function qualifyChildName(appName, childName) {
56
+ return childName.startsWith(`${appName}/`) || childName === appName
57
+ ? childName
58
+ : `${appName}/${childName}`;
59
+ }
60
+ const TOPOLOGY_EXPOSE_TIMEOUT_MS = 30_000;
61
+ const TOPOLOGY_EXPOSE_POLL_MS = 250;
32
62
  import { ServiceRouter } from './service-router.js';
33
63
  import { loadBootstrapConfig, clearCacheFor } from './bootstrap-loader.js';
64
+ /** Levels a pino logger accepts. A child is spawned with nothing else. */
65
+ const PINO_LEVELS = new Set(['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent']);
66
+ /**
67
+ * Process-manager states in which a record no longer describes a live process.
68
+ *
69
+ * Read by `managerOwnedPids`, where the distinction decides whether a pid is
70
+ * protected from the janitor. Written as strings rather than imported from
71
+ * `ProcessStatus` because that export is a const object whose values are
72
+ * these same literals; the set has to hold values, not keys, and a typo here
73
+ * would silently protect a dead pid — so it is pinned by a test.
74
+ */
75
+ const TERMINAL_PROCESS_STATUSES = new Set(['stopped', 'failed', 'crashed']);
34
76
  import { BuildService } from './build-service.js';
77
+ import { execFile } from 'node:child_process';
78
+ import { ProcessJanitor } from './process-janitor.js';
79
+ import { addressInUse, holderCommand, parseHolders, explainConflict } from './port-conflict.js';
80
+ import { collectOwnedPids } from './owned-pids.js';
81
+ import { combineProcessHealth, notRunningHealth, poolHealth } from './app-health.js';
82
+ import { combineProcessTraffic } from './app-traffic.js';
35
83
  import { CLI_VERSION } from '../config/defaults.js';
36
84
  const __dirname = path.dirname(fileURLToPath(import.meta.url));
37
85
  // Resolve bootstrap-process path — .ts in dev (tsx), .js when compiled
@@ -55,11 +103,124 @@ function mapStrategy(strategy) {
55
103
  return SupervisionStrategy.ONE_FOR_ONE;
56
104
  }
57
105
  }
106
+ /**
107
+ * Strip the `${project}/${stack}/` prefix from a handle key (or
108
+ * leave a bare name untouched). Used to identify "the same physical
109
+ * app under any registration form" so duplicate registrations from
110
+ * stack mode + bare-name RPC paths can be deduplicated.
111
+ *
112
+ * Examples:
113
+ * effectiveAppName('omni/dev/payments') === 'payments'
114
+ * effectiveAppName('payments') === 'payments'
115
+ * effectiveAppName('a/b/c/payments') === 'payments' (last segment)
116
+ */
117
+ export function effectiveAppName(key) {
118
+ const idx = key.lastIndexOf('/');
119
+ return idx >= 0 ? key.slice(idx + 1) : key;
120
+ }
121
+ /**
122
+ * Auto-promote a bare `entry.name` to canonical `${project}/${stack}/${name}`
123
+ * form when the env carries OMNITRON_PROJECT + OMNITRON_STACK. The stack-mode
124
+ * boot path (ProjectService.startStack) already prefixes the name explicitly;
125
+ * this helper closes every other entry point that could otherwise register a
126
+ * bare-name handle which then becomes invisible to the stack-prefix-based
127
+ * UI lookup (ProjectService.toStackInfo filters on `${project}/${stack}/`).
128
+ *
129
+ * Idempotent: if the entry already looks namespaced (or env lacks the keys),
130
+ * the input is returned unchanged. An entry qualified under a DIFFERENT
131
+ * project/stack is preserved too — that's operator intent (cross-project
132
+ * administrative run), not a bug.
133
+ */
134
+ export function ensureNamespacedEntry(entry) {
135
+ const project = entry.env?.['OMNITRON_PROJECT'];
136
+ const stack = entry.env?.['OMNITRON_STACK'];
137
+ if (!project || !stack)
138
+ return entry;
139
+ const canonicalPrefix = `${project}/${stack}/`;
140
+ if (entry.name.startsWith(canonicalPrefix))
141
+ return entry;
142
+ if (entry.name.includes('/'))
143
+ return entry;
144
+ return { ...entry, name: `${canonicalPrefix}${entry.name}` };
145
+ }
146
+ const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
147
+ /**
148
+ * Startup budget for a child process that declares none of its own.
149
+ *
150
+ * There were TWO of these and they disagreed — `?? 60_000` on the pooled path
151
+ * and `?? 30_000` on the single-process path — and, worse, the pooled one did
152
+ * not consult `entry.startupTimeout` at all. So an operator who set
153
+ * `startupTimeout: 120_000` on an app had it honoured for the app and
154
+ * SILENTLY IGNORED for its child processes, which kept 60 seconds and could
155
+ * not be given more by any setting.
156
+ *
157
+ * Measured on this host: an app whose own successful boot takes 120.6 seconds
158
+ * carried a 120 s budget, and its `deposit-worker` child — a deposit scanner
159
+ * — stayed `stopped` while the app itself came up. Two copies of one decision,
160
+ * one of which discarded the only knob an operator has.
161
+ *
162
+ * 60 s rather than 30 s, for the reason the project's own config already
163
+ * writes down: an over-long ceiling costs a slower failure report, a short one
164
+ * costs the whole stack.
165
+ */
166
+ const DEFAULT_CHILD_STARTUP_TIMEOUT = 60_000;
167
+ /**
168
+ * Deadline for an RPC the daemon makes to a child it supervises.
169
+ *
170
+ * A call across this connection is WORK — aggregate five minutes of candles,
171
+ * answer a health probe that touches a pool, hand back metrics — not a wire
172
+ * request, and netron's default (5000 ms) is sized for the latter. The pool
173
+ * path stated this number and the single-child path stated nothing, so a
174
+ * client built for a single child fell back to that default: measured, 94
175
+ * timeouts «after 5000ms» in the daemon log, every one on
176
+ * `callExposedService`, which lives on exactly those single children.
177
+ *
178
+ * It belongs here rather than in titan-pm because the orchestrator is what
179
+ * knows the nature of the work; the library's own fallback is 30 s
180
+ * (`process-pool.ts:212`) and applies when a caller states nothing — which,
181
+ * after this, no caller here does.
182
+ */
183
+ const CHILD_REQUEST_TIMEOUT = 120_000;
58
184
  export class OrchestratorService extends EventEmitter {
59
185
  logger;
60
186
  pm;
61
187
  stateStore;
62
188
  handles = new Map();
189
+ /**
190
+ * Per-name in-flight start lock (T#57). When two `startApp()` calls
191
+ * arrive concurrently for the same app, the first one's promise is
192
+ * stored here; the second one awaits and returns the same handle
193
+ * rather than racing through to a parallel `launchBootstrapMode` /
194
+ * `launchClassicMode` that would spawn a second worker pool and
195
+ * leak the original. The map is cleared in a `finally` once the
196
+ * launch settles (success or failure) so the next legitimate call
197
+ * for the same name isn't permanently locked out.
198
+ */
199
+ startingApps = new Map();
200
+ /**
201
+ * Per-app async serial-execution queue covering start / stop / restart
202
+ * / crash-restart-timer (P0-E + P0-I + P2-B in the audit). Pre-this-
203
+ * queue each path had its own coalescer:
204
+ * - startingApps: same-name start coalescing only — `stopApp` did
205
+ * NOT consult it, so stop racing in-flight start left an orphan
206
+ * supervisor when stop arrived before start wrote childProcess.
207
+ * - restartCoalescer: same-name restart coalescing; independent of
208
+ * stop+start.
209
+ * - handleClassicCrash setTimeout: a raw timer that could fire
210
+ * during the `await stopChild(...)` window and spawn a zombie
211
+ * against an app the operator just stopped.
212
+ * The queue collapses all four into one primitive ordered by
213
+ * priority (stop > restart > start > crash-restart). The same-key
214
+ * mutex means stop ALWAYS observes a completed start, and an
215
+ * explicit stop ALWAYS preempts a pending crash-restart timer.
216
+ */
217
+ lifecycleQueue = new ProcessLifecycleQueue();
218
+ /**
219
+ * Pending post-escalation cool-down timers, keyed by app name.
220
+ * Held here rather than inside a lifecycle-queue slot so that a
221
+ * crashed app still answers operator commands while it waits.
222
+ */
223
+ coolDownTimers = new Map();
63
224
  config = null;
64
225
  cwd;
65
226
  metricsTimer = null;
@@ -70,16 +231,84 @@ export class OrchestratorService extends EventEmitter {
70
231
  buildService = null;
71
232
  /** Cached build results per app name */
72
233
  buildResults = new Map();
234
+ /**
235
+ * Per-app restart coalescer state.
236
+ *
237
+ * `inFlight` — the currently-running restart promise (so a concurrent
238
+ * trigger doesn't kick off a parallel restart and tear down the half-
239
+ * started state from the previous one).
240
+ * `triggerSerial` — monotonically incremented by external trigger
241
+ * entry points (BuildService onRebuild, RPC restartApp, manual
242
+ * `omnitron restart`, deploy.service). Internal coalescer arms
243
+ * that queue themselves DO NOT bump it.
244
+ * `firedSerial` — value of `triggerSerial` that the most recently
245
+ * completed restart consumed. A trailing coalesced restart only
246
+ * fires when `triggerSerial > firedSerial`, i.e. a genuinely new
247
+ * external trigger arrived during the in-flight window. Two
248
+ * coalescer arms colliding can no longer cause a self-perpetuating
249
+ * loop because neither bumps `triggerSerial`.
250
+ * `cooldownUntil` — absolute epoch ms timestamp. Any trigger received
251
+ * before this clears is queued instead of fired immediately.
252
+ * Prevents the storm where esbuild's onEnd fires for a freshly-
253
+ * spawned context milliseconds after the app reports online.
254
+ */
255
+ restartCoalescer = new Map();
256
+ /**
257
+ * Wall-clock window in ms during which restart triggers from
258
+ * BuildService.watchApp are deferred after `app:online`. Sized so a
259
+ * normal bundle build + child spawn + DI bootstrap fits comfortably
260
+ * inside; anything legitimately changing more often than this is a
261
+ * developer typing very fast, which the regular debounce handles.
262
+ */
263
+ static POST_ONLINE_COOLDOWN_MS = 3_000;
264
+ /**
265
+ * Orphan-process reaper. Lazily created on first reconcile so unit
266
+ * tests that hit `startApp` directly don't get a periodic timer
267
+ * until the first real reconcile runs (which they bypass by
268
+ * stubbing `bootReconciled = true`).
269
+ */
270
+ janitor = null;
271
+ /**
272
+ * One-shot guard for cold-start reconciliation. T#63 — every code
273
+ * path that lands an app on this orchestrator goes through
274
+ * `ensureBootReconciled()`, regardless of whether the entry-point
275
+ * is `startAll` (daemon-config) or `startApp` (project/stack
276
+ * mode). Pre-T#63 the cold-start sweep only fired inside
277
+ * `startAll`, so project-mode boots silently inherited orphans
278
+ * from the previous daemon generation.
279
+ */
280
+ bootReconciled = false;
281
+ bootReconcilePromise = null;
73
282
  /** Daemon's Netron instance for native service routing */
74
283
  daemonNetron = null;
75
284
  /** Daemon Unix socket path for child-to-daemon Netron connections */
76
285
  daemonSocketPath = null;
286
+ /**
287
+ * Promise that resolves once the daemon's Netron is wired up. Apps that
288
+ * declare topology cannot launch until this is ready — instead of failing
289
+ * with "daemon Netron not available" the launcher awaits this promise
290
+ * (with a generous timeout) so transient startup-order interleavings
291
+ * resolve themselves instead of becoming user-visible errors.
292
+ */
293
+ daemonNetronReady;
294
+ resolveDaemonNetronReady = () => undefined;
295
+ /**
296
+ * Optional metrics sink. Set by the daemon at boot via
297
+ * `setMetricsBridge` so the orchestrator can forward janitor sweeps,
298
+ * esbuild rebuilds, and lifecycle events without coupling itself to
299
+ * the metrics module. Null means metrics are disabled (tests, slim
300
+ * bootstraps).
301
+ */
302
+ metricsBridge = null;
77
303
  constructor(logger, pm, stateStore, cwd) {
78
304
  super();
79
305
  this.logger = logger;
80
306
  this.pm = pm;
81
307
  this.stateStore = stateStore;
82
308
  this.cwd = cwd ?? process.cwd();
309
+ this.daemonNetronReady = new Promise((resolve) => {
310
+ this.resolveDaemonNetronReady = resolve;
311
+ });
83
312
  }
84
313
  /**
85
314
  * Set the daemon Netron instance and socket path for native topology routing.
@@ -88,28 +317,356 @@ export class OrchestratorService extends EventEmitter {
88
317
  setDaemonNetron(netron, socketPath) {
89
318
  this.daemonNetron = netron;
90
319
  this.daemonSocketPath = socketPath;
320
+ this.resolveDaemonNetronReady();
91
321
  this.logger.info({ socketPath }, 'Daemon Netron set for native topology routing');
92
322
  }
323
+ /**
324
+ * Wire the metrics bridge. Called once during daemon bootstrap.
325
+ * After this, janitor sweeps, esbuild rebuilds, and orchestrator
326
+ * lifecycle events are forwarded to Prometheus.
327
+ */
328
+ setMetricsBridge(bridge) {
329
+ this.metricsBridge = bridge;
330
+ bridge.attachOrchestrator(this);
331
+ this.wireOrchestratorTelemetry();
332
+ }
333
+ /**
334
+ * Subscribe to our own EventEmitter to forward state-change events to
335
+ * the metrics bridge. Idempotent so repeated `setMetricsBridge` calls
336
+ * (mostly tests) don't multiply listeners.
337
+ */
338
+ telemetryWired = false;
339
+ wireOrchestratorTelemetry() {
340
+ if (this.telemetryWired)
341
+ return;
342
+ this.telemetryWired = true;
343
+ this.on('app:restart', (app, attempt) => {
344
+ this.metricsBridge?.recordAppRestart(app, attempt, 'crash');
345
+ });
346
+ this.on('app:crash', (app) => {
347
+ this.metricsBridge?.recordAppCrash(app);
348
+ });
349
+ this.on('app:escalated', (app) => {
350
+ this.metricsBridge?.recordAppEscalated(app);
351
+ });
352
+ }
353
+ /**
354
+ * Wait until the daemon Netron is available, with a hard ceiling so we
355
+ * never hang forever if `setDaemonNetron()` was never wired in.
356
+ */
357
+ async ensureDaemonNetronReady(timeoutMs = 30_000) {
358
+ if (this.daemonNetron && this.daemonSocketPath)
359
+ return;
360
+ const timeout = new Promise((_resolve, reject) => {
361
+ const timer = setTimeout(() => reject(new Error(`Daemon Netron not initialized within ${timeoutMs}ms — wiring bug?`)), timeoutMs);
362
+ timer.unref?.();
363
+ });
364
+ await Promise.race([this.daemonNetronReady, timeout]);
365
+ }
93
366
  // ============================================================================
94
367
  // Lifecycle
95
368
  // ============================================================================
369
+ /**
370
+ * Refresh the orchestrator's view of the ecosystem config. Called
371
+ * from `daemon.reloadConfig` so a subsequent `restartApp` reads the
372
+ * new entry definition (mode, env, dependsOn, restart policy)
373
+ * instead of the stale copy captured at first `startAll`. Pre-fix
374
+ * the two refs diverged silently after every reload — operators
375
+ * editing ecosystem.config.ts and running `omnitron reload`
376
+ * believed the daemon picked up the changes; in fact only daemon's
377
+ * own RPC service did, and individual app restarts kept running
378
+ * against the pre-reload definition until the daemon itself
379
+ * restarted. P0-G in the audit.
380
+ */
381
+ setConfig(config) {
382
+ this.config = config;
383
+ // Metrics polling is armed here rather than in `startAll` alone.
384
+ //
385
+ // It used to run only from `startAll`, which is the `omnitron start`
386
+ // path. Apps brought up any other way — `stack start`, a single
387
+ // `omnitron start <app>`, or the daemon resuming a stack at boot — were
388
+ // never sampled, so `omnitron list`, `status` and the console's app
389
+ // cards reported cpu 0 / memory 0 for processes that were serving
390
+ // traffic. Observed live: six apps online, every one showing 0 while
391
+ // `ps` reported ~105 MB RSS apiece.
392
+ //
393
+ // `setConfig` is the one point every path passes through, and the call
394
+ // is idempotent — an existing timer is cleared first.
395
+ this.ensureMetricsPolling();
396
+ }
397
+ /**
398
+ * Start metrics sampling if it is not already running.
399
+ *
400
+ * Called from every path that can produce a managed process, so no app can
401
+ * end up unsampled because of how it was started.
402
+ */
403
+ ensureMetricsPolling() {
404
+ if (this.metricsTimer)
405
+ return;
406
+ this.startMetricsPolling(this.config?.monitoring?.metrics?.interval ?? 5_000);
407
+ }
408
+ /**
409
+ * One-shot cold-start reconciliation: discover pre-existing managed
410
+ * processes, reap fork-workers left over from a previous daemon
411
+ * generation, and arm the periodic janitor sweep. Idempotent —
412
+ * concurrent callers join the in-flight promise; subsequent calls
413
+ * after success are no-ops.
414
+ *
415
+ * Called from BOTH `startAll` (top-level daemon-config boot) and
416
+ * `startApp` (project/stack mode boot), so every code path that
417
+ * lands an app on this orchestrator runs the cleanup. Pre-T#63 the
418
+ * cold-start sweep was gated on `startAll` and never ran in
419
+ * project mode, so an orphan from a previous daemon survived
420
+ * silently and held the listen port hostage from its successor.
421
+ */
422
+ async ensureBootReconciled(config) {
423
+ if (this.bootReconciled)
424
+ return;
425
+ if (this.bootReconcilePromise) {
426
+ await this.bootReconcilePromise;
427
+ return;
428
+ }
429
+ this.bootReconcilePromise = (async () => {
430
+ // Discovery: surface OMNITRON_MANAGED-tagged processes that match
431
+ // a configured app so the operator can see what's about to be
432
+ // reaped (or — once the ChildContract handshake lands — adopted).
433
+ // P2-A in the audit.
434
+ try {
435
+ const discovered = discoverManagedProcesses();
436
+ const adopted = [];
437
+ const knownApps = new Set(config?.apps.map((a) => a.name) ?? []);
438
+ for (const proc of discovered) {
439
+ if (proc.ppid === process.pid)
440
+ continue; // our own freshly-spawned child
441
+ const fqn = proc.fullyQualifiedName;
442
+ const bare = proc.appName;
443
+ if (knownApps.has(fqn) || knownApps.has(bare)) {
444
+ adopted.push(`${fqn}#${proc.pid}`);
445
+ }
446
+ }
447
+ if (adopted.length > 0) {
448
+ this.logger.warn({ discovered: adopted.length, apps: adopted }, 'Found pre-existing managed processes — they will be reaped by the janitor in 60s unless adopted');
449
+ }
450
+ }
451
+ catch (err) {
452
+ this.logger.debug({ err: err.message }, 'Process discovery failed — proceeding without reconcile');
453
+ }
454
+ // Reap any fork-workers left over from a previous daemon BEFORE
455
+ // launching apps. Without this, a hard-killed daemon's children
456
+ // are still alive (now reparented to init); they hold DB
457
+ // connections and listen ports that the new apps will fight
458
+ // for. The reap is bounded by `gracefulMs` (default 3s).
459
+ await this.ensureJanitor().coldStartSweep();
460
+ // Arm the periodic sweep so runtime leaks from partially-failed
461
+ // restart cycles (esbuild-rebuild → spawn-fail → orphan) get
462
+ // collected on every interval.
463
+ this.ensureJanitor().start();
464
+ })();
465
+ try {
466
+ await this.bootReconcilePromise;
467
+ this.bootReconciled = true;
468
+ }
469
+ finally {
470
+ this.bootReconcilePromise = null;
471
+ }
472
+ }
96
473
  async startAll(config) {
97
474
  this.config = config;
475
+ // Cold-start orphan reap + periodic janitor arming. Idempotent —
476
+ // safe to call on every boot path. Project-mode flows that call
477
+ // `startApp` directly trigger the same one-shot reconcile, so
478
+ // every boot path lands in the same post-condition. (T#63)
479
+ await this.ensureBootReconciled(config);
480
+ // Dependency-aware skip: if app A fails to start, every app that
481
+ // declares A in `dependsOn` (transitively) is skipped instead of
482
+ // attempted. Without this, a downstream app would launch, fail to
483
+ // reach its missing dependency, crash, and chew through its restart
484
+ // budget — masking the real root cause.
485
+ const failed = new Set();
486
+ const blocked = new Set();
98
487
  for (const batch of resolveStartupOrder(config.apps)) {
99
488
  const enabled = batch.filter((e) => e.enabled !== false);
100
489
  await Promise.all(enabled.map(async (e) => {
490
+ const blockingDep = (e.dependsOn ?? []).find((dep) => failed.has(dep) || blocked.has(dep));
491
+ if (blockingDep) {
492
+ blocked.add(e.name);
493
+ this.logger.warn({ app: e.name, blockedBy: blockingDep }, 'Skipping app startup — depends on a failed/blocked app');
494
+ return;
495
+ }
101
496
  try {
102
497
  await this.startApp(e, config);
103
498
  }
104
499
  catch (err) {
500
+ failed.add(e.name);
105
501
  this.logger.error({ app: e.name, error: err.message }, 'Failed to start app — continuing with remaining apps');
106
502
  }
107
503
  }));
108
504
  }
109
- this.startMetricsPolling(config.monitoring.metrics.interval);
505
+ if (failed.size > 0 || blocked.size > 0) {
506
+ this.logger.warn({ failed: [...failed], blocked: [...blocked] }, 'Startup completed with failures — some apps did not launch');
507
+ }
508
+ // Periodic sweep was armed by `ensureBootReconciled` above, and metrics
509
+ // polling by `setConfig` — no second call needed for either.
110
510
  this.persistState();
111
511
  }
512
+ /**
513
+ * Lazily create the orphan-process janitor with the live "owned
514
+ * pids" snapshot wired to the orchestrator's handle map.
515
+ */
516
+ ensureJanitor() {
517
+ if (this.janitor)
518
+ return this.janitor;
519
+ this.janitor = new ProcessJanitor({
520
+ logger: this.logger,
521
+ getOwnedPids: () => this.collectOwnedPids(),
522
+ onMetrics: (m) => this.metricsBridge?.recordJanitorSweep(m),
523
+ });
524
+ return this.janitor;
525
+ }
526
+ /**
527
+ * Every OS pid the orchestrator currently considers ours. Cheap — called
528
+ * on every janitor sweep, and the janitor kills whatever this set omits.
529
+ *
530
+ * The rule itself lives in `owned-pids.ts`, where it can be tested: the
531
+ * janitor's own tests supply this set as a fixture, so they could never
532
+ * catch it being incomplete — and it was, by a whole class of process.
533
+ */
534
+ /**
535
+ * Who is holding the port an app could not bind, in one sentence.
536
+ *
537
+ * Called where a start or restart failed, because that is the moment the
538
+ * answer is worth having and the moment nobody has it: the error names the
539
+ * address and never the holder. Measured on this daemon — four
540
+ * `EADDRINUSE` failures on 3005 in thirty-one seconds, `omnitron list`
541
+ * reporting the app crashed with no pid, and the port held the whole time
542
+ * by a child of this very daemon that nothing in its records claimed.
543
+ *
544
+ * Returns null when the failure was not a port clash, and says so out loud
545
+ * when it could not read the machine — a reading that failed and a port
546
+ * nobody holds are different findings.
547
+ */
548
+ async explainPortConflict(message) {
549
+ const address = addressInUse(message);
550
+ if (!address)
551
+ return null;
552
+ const { file, args } = holderCommand(address.port, process.platform);
553
+ let output;
554
+ try {
555
+ output = await new Promise((resolve, reject) => {
556
+ execFile(file, args, { encoding: 'utf8', timeout: 5_000 }, (err, stdout) => {
557
+ // `ss` and `lsof` both exit non-zero when nothing matches, which is
558
+ // an answer rather than a failure; the empty stdout carries it.
559
+ if (err && !stdout)
560
+ reject(err);
561
+ else
562
+ resolve(stdout);
563
+ });
564
+ });
565
+ }
566
+ catch (err) {
567
+ return `could not read who holds port ${address.port}: ${file} ${err.message}`;
568
+ }
569
+ const holders = parseHolders(output, process.platform);
570
+ // One `ps` for the parents, before deciding: the decision is pure and
571
+ // takes the answers, so that it can be tested without a machine.
572
+ const parents = new Map();
573
+ for (const holder of holders) {
574
+ const ppid = await this.parentOf(holder.pid);
575
+ if (ppid !== undefined)
576
+ parents.set(holder.pid, ppid);
577
+ }
578
+ const verdict = explainConflict({
579
+ port: address.port,
580
+ holders,
581
+ parentOf: (pid) => parents.get(pid),
582
+ daemonPid: process.pid,
583
+ owned: this.collectOwnedPids(),
584
+ });
585
+ return verdict.because;
586
+ }
587
+ /** The parent of one process, or undefined when it cannot be read. */
588
+ async parentOf(pid) {
589
+ try {
590
+ const out = await new Promise((resolve, reject) => {
591
+ execFile('ps', ['-o', 'ppid=', '-p', String(pid)], { encoding: 'utf8', timeout: 5_000 }, (err, stdout) => {
592
+ if (err && !stdout)
593
+ reject(err);
594
+ else
595
+ resolve(stdout);
596
+ });
597
+ });
598
+ const parsed = Number(out.trim());
599
+ return Number.isInteger(parsed) ? parsed : undefined;
600
+ }
601
+ catch {
602
+ // A process that exited between the two readings has no parent to
603
+ // report, and that is not an error worth a line of its own — the
604
+ // verdict already says the holder was gone.
605
+ return undefined;
606
+ }
607
+ }
608
+ collectOwnedPids() {
609
+ return collectOwnedPids(this.handles.values(), (workerId) => this.pm.getWorkerHandle(workerId)?.pid, this.managerOwnedPids());
610
+ }
611
+ /**
612
+ * Every pid the process manager currently vouches for.
613
+ *
614
+ * Two windows, and a process is only safe if both are asked. `getForkedPids`
615
+ * holds a claim from `fork()` until `spawn()` resolves and releases it —
616
+ * after that its own docstring says "the WorkerHandle registry is the
617
+ * authority". `listProcesses()` IS that registry, and it is the only source
618
+ * that answers for a process the orchestrator did not start: the daemon's
619
+ * system workers go through this same manager but belong to no app handle,
620
+ * so walking `this.handles` can never see them.
621
+ *
622
+ * Terminal records are excluded deliberately. They keep the pid they had,
623
+ * the OS reuses pids, and a stale one here would vouch for exactly the
624
+ * leaked worker the janitor exists to reap.
625
+ */
626
+ managerOwnedPids() {
627
+ const pids = [...(this.pm.getForkedPids?.() ?? [])];
628
+ for (const info of this.pm.listProcesses?.() ?? []) {
629
+ if (typeof info.pid !== 'number')
630
+ continue;
631
+ if (TERMINAL_PROCESS_STATUSES.has(info.status))
632
+ continue;
633
+ pids.push(info.pid);
634
+ }
635
+ return pids;
636
+ }
112
637
  async startApp(entry, config) {
638
+ // T#63: run the one-shot cold-start reconcile (orphan reap +
639
+ // janitor arming) before ANY app spawns. Project-mode boots flow
640
+ // through `projectService.startStack → orchestrator.startApp`
641
+ // and historically bypassed the reap entirely — a previous
642
+ // daemon's children (now reparented to init) kept their HTTP
643
+ // listen ports and the fresh child failed to bind silently.
644
+ // No-op after first call thanks to `bootReconciled`.
645
+ await this.ensureBootReconciled(config);
646
+ // Defensive auto-namespacing — promotes a bare entry.name to the
647
+ // canonical `${project}/${stack}/${name}` form when the env carries
648
+ // OMNITRON_PROJECT + OMNITRON_STACK. Catches every code path that
649
+ // bypasses ProjectService.startStack:
650
+ // - direct CLI `omnitron start <name>` (no stack context)
651
+ // - legacy ecosystem configs without per-app namespace
652
+ // - daemon startAll() flows over a not-yet-namespaced config
653
+ entry = ensureNamespacedEntry(entry);
654
+ const inflight = this.startingApps.get(entry.name);
655
+ if (inflight) {
656
+ this.logger.debug({ app: entry.name }, 'Coalescing concurrent startApp into in-flight launch (T#57)');
657
+ return inflight;
658
+ }
659
+ // Serialise against stop/restart for the same app via the
660
+ // lifecycle queue. A stopApp arriving while this start is queued
661
+ // (or in-flight) now waits its turn instead of racing through
662
+ // and leaving an orphan supervisor.
663
+ const promise = this.lifecycleQueue.enqueue(entry.name, 'start', () => this.startAppInternal(entry, config)).finally(() => {
664
+ this.startingApps.delete(entry.name);
665
+ });
666
+ this.startingApps.set(entry.name, promise);
667
+ return promise;
668
+ }
669
+ async startAppInternal(entry, config) {
113
670
  // Persist ecosystem config for restartApp — stack mode passes config per-app,
114
671
  // but restartApp calls startApp without it.
115
672
  if (config && !this.config) {
@@ -118,14 +675,106 @@ export class OrchestratorService extends EventEmitter {
118
675
  const cfg = config ?? this.config;
119
676
  if (!cfg)
120
677
  throw new Error('Ecosystem config not loaded');
678
+ // Deduplicate by effective app identity. Project-mode registers
679
+ // apps as `${project}/${stack}/${app}`; non-project paths use the
680
+ // bare `${app}` short name. They refer to the SAME physical app
681
+ // and must never coexist as separate handles — the duplicate
682
+ // would compete for the port and the orphan-reaper's owned-pid
683
+ // accounting would split between them.
684
+ //
685
+ // Policy:
686
+ // - exact key already exists, online → return it (idempotent)
687
+ // - exact key already exists, not online → fall through and re-launch
688
+ // under the same key
689
+ // - DIFFERENT key with the same effective short name:
690
+ // - new is MORE specific (namespaced) and existing is bare → stop
691
+ // the bare one and re-launch under the namespaced key. Without
692
+ // this, a boot-race where bare-name auto-start beats the
693
+ // project/stack flow strands the app under the wrong key and
694
+ // webapp/CLI lookups by namespaced key see it as stopped.
695
+ // - new is LESS-or-equally specific → refuse: operator should
696
+ // restart by canonical name, not register an alias.
697
+ const effective = effectiveAppName(entry.name);
698
+ const newIsNamespaced = entry.name.includes('/');
699
+ for (const [key, h] of this.handles) {
700
+ if (key === entry.name)
701
+ continue;
702
+ if (effectiveAppName(key) !== effective)
703
+ continue;
704
+ const existingIsNamespaced = key.includes('/');
705
+ // Same effective app under a different key.
706
+ if (h.status === 'online') {
707
+ if (newIsNamespaced && !existingIsNamespaced) {
708
+ this.logger.info({ stale: key, replacing_with: entry.name, effective }, 'Stopping bare-name handle to re-register under namespaced key');
709
+ try {
710
+ await this.stopApp(key);
711
+ }
712
+ catch (err) {
713
+ this.logger.warn({ stale: key, error: err.message }, 'stopApp during dedup failed — clearing handle anyway');
714
+ }
715
+ this.handles.delete(key);
716
+ if (key !== entry.name)
717
+ this.metricsBridge?.evictApp(key);
718
+ continue;
719
+ }
720
+ this.logger.warn({ existing: key, attempted: entry.name, effective }, 'App already running under a canonical name — refusing duplicate registration');
721
+ return h;
722
+ }
723
+ // Existing handle is errored/stopped — replace silently so a
724
+ // failed previous launch doesn't permanently block the canonical
725
+ // form.
726
+ this.logger.info({ stale: key, replacing_with: entry.name, effective }, 'Replacing stale duplicate handle');
727
+ // The handle is errored or stopped, so `stopApp` never ran for it and
728
+ // its services are still registered on the daemon's Netron. Dropping the
729
+ // handle here is the last moment anything knows they exist — or that its
730
+ // processes still run: an errored app's other children and its pools.
731
+ await this.releaseExposedServices(h);
732
+ await this.tearDownProcesses(h);
733
+ this.handles.delete(key);
734
+ // Drop any metrics that were collected under the stale alias so it
735
+ // doesn't haunt future snapshots as a ghost offline app.
736
+ if (key !== entry.name)
737
+ this.metricsBridge?.evictApp(key);
738
+ }
121
739
  const existing = this.handles.get(entry.name);
122
740
  if (existing && existing.status === 'online') {
123
741
  this.logger.warn({ app: entry.name }, 'App already running');
124
742
  return existing;
125
743
  }
744
+ // A handle that is not online may still run processes. A crash restart and
745
+ // a cool-down recovery only flip its status before coming here; one child
746
+ // crashing leaves its siblings — the server among them — up; and its pools
747
+ // were never any supervisor's to stop. Stopped before the new handle
748
+ // starts, or the new one finds its own predecessor on its port: on
749
+ // daos/test (2026-09-25) paysys's crash restarts failed five times on
750
+ // `EADDRINUSE :3004`, «held by … a child this daemon still accounts for».
751
+ if (existing) {
752
+ await this.releaseExposedServices(existing);
753
+ await this.tearDownProcesses(existing);
754
+ }
126
755
  const mode = entry.bootstrap ? 'bootstrap' : 'classic';
127
756
  const handle = new AppHandle(entry, mode);
757
+ // The restart budget belongs to the APP, not to this handle instance.
758
+ // Every start replaces the handle, so a counter that starts at zero here
759
+ // can never reach `maxRestarts` — which made the budget, the escalation
760
+ // and the cool-down unreachable even once the restart itself worked. The
761
+ // window is enforced by the policy, not by forgetting.
762
+ if (existing)
763
+ handle.restarts = existing.restarts;
128
764
  this.handles.set(entry.name, handle);
765
+ // Arm metrics sampling here — the one place an app enters the registry.
766
+ //
767
+ // It used to be armed only from `startAll` (the `omnitron start` path).
768
+ // Anything else — `stack start`, a single `omnitron start <app>`, the
769
+ // daemon resuming a stack at boot — was never sampled, so `list`,
770
+ // `status` and the console's app cards reported cpu 0 / memory 0 for
771
+ // processes serving traffic. Observed: six apps online, every one at
772
+ // zero, while `ps` showed ~105 MB RSS apiece.
773
+ //
774
+ // `setConfig` looked like the right seam but is only called on config
775
+ // RELOAD, so it never fired on a normal start. This is the seam that
776
+ // actually covers every path. Idempotent: re-arming replaces the timer.
777
+ this.ensureMetricsPolling();
129
778
  this.logger.info({ app: entry.name, mode }, 'Starting app');
130
779
  try {
131
780
  if (mode === 'classic') {
@@ -134,42 +783,113 @@ export class OrchestratorService extends EventEmitter {
134
783
  else {
135
784
  await this.launchBootstrapMode(entry, handle, cfg);
136
785
  }
137
- this.logger.info({ app: entry.name, pid: handle.pid, mode }, 'App started');
786
+ // «App started» only for an app that is. It was logged in the same
787
+ // millisecond as a child's «Process failed to start», for an app whose
788
+ // processes never all came up.
789
+ if (handle.status === 'online') {
790
+ this.logger.info({ app: entry.name, pid: handle.pid, mode }, 'App started');
791
+ }
792
+ else {
793
+ this.logger.error({ app: entry.name, status: handle.status, mode }, 'App did not come up — not every declared process started');
794
+ }
138
795
  this.persistState();
139
796
  return handle;
140
797
  }
141
798
  catch (err) {
799
+ // `errored` rather than `crashed` is the right REPORT — the app never
800
+ // ran. It was also, until now, the end of the story: `'errored'` is read
801
+ // by no code in this repository outside formatters, `status` and a
802
+ // metrics label, so a start that failed was never attempted again. One
803
+ // try, then errored until a human noticed.
804
+ //
805
+ // That asymmetry is backwards. A failure to start is MORE likely to be
806
+ // transient than a crash after a successful boot — a dependency still
807
+ // coming up, a disk still cold, a machine briefly loaded. Six bootstrap
808
+ // apps here timed out at `config:loading` under a load average of 149
809
+ // and stayed down after the load passed, because nothing asked again.
810
+ //
811
+ // The counter is bumped by hand because `markErrored()` deliberately
812
+ // does not (an app that never ran has not "restarted"), and without it
813
+ // the budget below can never trip and the retry would be unbounded.
142
814
  handle.markErrored();
815
+ handle.restarts++;
143
816
  this.persistState();
817
+ this.scheduleRecovery(entry, handle, cfg);
144
818
  throw err;
145
819
  }
146
820
  }
147
- async stopApp(name, force = false, timeout = 10_000) {
148
- const handle = this.handles.get(name);
149
- if (!handle || handle.status === 'stopped')
150
- return;
151
- handle.markStopping();
152
- // Unexpose topology services from daemon Netron before stopping processes
153
- if (handle.serviceRouter) {
154
- try {
155
- for (const svcName of handle.serviceRouter.getServiceNames()) {
156
- await handle.serviceRouter.unexposeService(svcName);
157
- }
821
+ /**
822
+ * Everything a handle still runs, stopped: its supervisor's children (or its
823
+ * classic child process) AND its topology pools.
824
+ *
825
+ * The pools were missed. A pool's workers are not the supervisor's children
826
+ * — they are the daemon's, siblings of the server, recorded only in
827
+ * `topologyPools` — so `supervisor.stop()` never reached them, and nothing
828
+ * in the orchestrator destroyed a pool: only the daemon's own shutdown did.
829
+ * Every restart of an app with `instances > 1` left the previous workers
830
+ * running. On daos/test (2026-09-25): priceverse's `ohlcv-aggregator` and
831
+ * storage's `transform` in three generations at once.
832
+ *
833
+ * The pools go even when the supervisor's stop throws: a stop that failed
834
+ * half-way must not also leave the half nobody asked about.
835
+ */
836
+ async tearDownProcesses(handle, force = false, timeout = 10_000) {
837
+ try {
838
+ if (handle.mode === 'bootstrap' && handle.supervisor) {
839
+ await handle.supervisor.stop();
158
840
  }
159
- catch {
160
- // Best-effort cleanup
841
+ else if (handle.childProcess) {
842
+ await this.stopChildProcess(handle, force, timeout);
161
843
  }
162
- handle.serviceRouter = null;
163
844
  }
164
- if (handle.mode === 'bootstrap' && handle.supervisor) {
165
- await handle.supervisor.stop();
845
+ finally {
846
+ for (const [name, pool] of handle.topologyPools) {
847
+ try {
848
+ await pool.destroy();
849
+ }
850
+ catch (err) {
851
+ this.logger.warn({ app: handle.name, pool: name, error: err.message }, 'Destroying a topology pool failed — its workers may still run');
852
+ }
853
+ }
854
+ handle.topologyPools.clear();
166
855
  }
167
- else if (handle.childProcess) {
168
- await this.stopChildProcess(handle, force, timeout);
856
+ }
857
+ /**
858
+ * Give back every service an app exposed on the daemon's Netron.
859
+ *
860
+ * A registration outlives the process behind it: the daemon keeps answering
861
+ * `queryInterface` with the full method list and every call through the name
862
+ * fails at the socket. `takeOverExisting` rescues a name the next launch
863
+ * re-exposes; one it does not stays advertised for the life of the daemon.
864
+ */
865
+ async releaseExposedServices(handle) {
866
+ if (!handle.serviceRouter)
867
+ return;
868
+ try {
869
+ await handle.serviceRouter.releaseAll();
169
870
  }
170
- handle.markStopped();
171
- this.persistState();
172
- this.logger.info({ app: name }, 'App stopped');
871
+ catch (err) {
872
+ this.logger.warn({ app: handle.name, error: err.message }, 'Releasing exposed services failed — the daemon may still advertise them');
873
+ }
874
+ handle.serviceRouter = null;
875
+ }
876
+ async stopApp(name, force = false, timeout = 10_000) {
877
+ const canonical = this.resolveAppName(name) ?? name;
878
+ // Outside the queue on purpose: the operator's intent lands the
879
+ // moment the command arrives, not whenever the queue drains.
880
+ this.clearCoolDown(canonical);
881
+ return this.lifecycleQueue.enqueue(canonical, 'stop', async () => {
882
+ const handle = this.handles.get(canonical);
883
+ if (!handle || handle.status === 'stopped')
884
+ return;
885
+ handle.markStopping();
886
+ // Unexpose topology services from daemon Netron before stopping processes
887
+ await this.releaseExposedServices(handle);
888
+ await this.tearDownProcesses(handle, force, timeout);
889
+ handle.markStopped();
890
+ this.persistState();
891
+ this.logger.info({ app: name }, 'App stopped');
892
+ });
173
893
  }
174
894
  async stopAll(force = false) {
175
895
  if (!this.config)
@@ -178,6 +898,11 @@ export class OrchestratorService extends EventEmitter {
178
898
  clearInterval(this.metricsTimer);
179
899
  this.metricsTimer = null;
180
900
  }
901
+ // Stop the janitor first so it doesn't race the controlled
902
+ // shutdown by trying to "reap" children that are mid-stop.
903
+ if (this.janitor) {
904
+ this.janitor.stop();
905
+ }
181
906
  let count = 0;
182
907
  // When force=true (daemon shutdown), use short timeout per app (3s SIGTERM, then SIGKILL)
183
908
  const perAppTimeout = force ? 3_000 : 10_000;
@@ -209,36 +934,270 @@ export class OrchestratorService extends EventEmitter {
209
934
  this.buildResults.clear();
210
935
  return count;
211
936
  }
937
+ /**
938
+ * Give a supervised app the definition its next start will use — its
939
+ * environment, its command. The running process is not touched; the next
940
+ * `restartApp` starts from this. `false` when nothing by that name runs
941
+ * here. See `AppHandle.redefine`.
942
+ */
943
+ redefineApp(name, entry) {
944
+ const handle = this.handles.get(this.resolveAppName(name) ?? name);
945
+ if (!handle)
946
+ return false;
947
+ handle.redefine(entry);
948
+ return true;
949
+ }
950
+ /**
951
+ * Restart an app, coalescing concurrent triggers.
952
+ *
953
+ * The dev-mode hot-reload path can deliver many restart triggers per
954
+ * second (file save → esbuild rebuild → onEnd → debounced → restart),
955
+ * and naive parallelism here ends in cross-stomping state mutations
956
+ * (one restart's stopApp tearing down the other's freshly-spawned
957
+ * child). This wrapper serialises restarts per app:
958
+ *
959
+ * - If a restart is already in flight, we set the `pending` bit and
960
+ * await the current restart. When it completes we run exactly one
961
+ * follow-up — extra triggers received in the meantime collapse onto
962
+ * the same bit, so a burst of N triggers produces at most 2 actual
963
+ * restarts (the current one + one trailing).
964
+ * - If we're inside the post-online cooldown window, the trigger is
965
+ * queued (via `pending`) and fires after the cooldown clears.
966
+ *
967
+ * The hot path is `restartAppNow()` which contains the actual work and
968
+ * is what tests / non-coalescing callers can hit directly.
969
+ */
212
970
  async restartApp(name) {
971
+ // External entry — record one new trigger and dispatch.
972
+ return this.restartAppCoalesced(name, /* fromExternal */ true);
973
+ }
974
+ /**
975
+ * Internal coalescer dispatcher.
976
+ *
977
+ * `fromExternal=true` bumps `triggerSerial` (a user / file-watcher /
978
+ * RPC asked us to restart). `fromExternal=false` is used by the
979
+ * recursive arms (cooldown-wait, await-inflight) which are NOT new
980
+ * triggers, just continuations — they must not bump or the trailing
981
+ * coalesced restart would re-fire forever.
982
+ */
983
+ async restartAppCoalesced(name, fromExternal) {
984
+ const canonical = this.resolveAppName(name) ?? name;
985
+ const handle = this.handles.get(canonical);
986
+ if (!handle) {
987
+ // A PROJECT-QUALIFIED name with no live handle is not this daemon's to
988
+ // start, and matching it against the daemon's own config starts the
989
+ // WRONG APP.
990
+ //
991
+ // The names collide in practice. omnitron's own `omnitron.config.ts`
992
+ // declares `main`, `payments`, `messaging`, `storage`, `pricing` — the
993
+ // same five a downstream stack uses. On 2026-09-10, after payments died,
994
+ // `omnitron restart acme/dev/payments` matched omnitron's sample entry by
995
+ // its last path segment and launched `apps/payments/src/main.ts` relative
996
+ // to the DAEMON's cwd. It died with ERR_MODULE_NOT_FOUND on a path that
997
+ // exists in neither repository, and the backend stayed down while the
998
+ // command reported that it had restarted it.
999
+ //
1000
+ // Starting the wrong app is worse than refusing: the operator is told
1001
+ // their service is coming back, and the log that says otherwise is the
1002
+ // one they are not reading yet. `startApp` on the daemon already refuses
1003
+ // this exact shape and names the command that works; this is the same
1004
+ // refusal on the same reasoning, at the point where "no live handle" is
1005
+ // already known.
1006
+ const qualified = name.includes('/');
1007
+ const ownProject = this.config?.project;
1008
+ if (qualified && !ownProject) {
1009
+ const [project, stack] = name.split('/');
1010
+ throw new Error(`'${name}' belongs to a registered project, which this daemon restarts through its ` +
1011
+ `stack. Use \`omnitron stack start ${project} ${stack}\`, or restart the app by its ` +
1012
+ `bare name if you mean the daemon's own config.`);
1013
+ }
1014
+ // P1-F — fall through to startApp when there's no live handle but
1015
+ // the ecosystem config declares the app. Operators expect
1016
+ // `omnitron restart foo` to work even when `foo` was never started
1017
+ // (or was stopped, or crashed and was reaped). Previously this
1018
+ // threw "Unknown app", forcing the operator to first run start
1019
+ // explicitly.
1020
+ const effective = effectiveAppName(name);
1021
+ const entry = this.config?.apps.find((a) => a.name === name || a.name === effective || effectiveAppName(a.name) === effective);
1022
+ if (entry) {
1023
+ this.logger.info({ app: name }, 'restartApp: no live handle — falling through to startApp');
1024
+ // Restart under the name the OPERATOR asked for. The match above
1025
+ // accepts a bare entry for a qualified request, and handing it to
1026
+ // `startApp` unchanged re-registers the app bare —
1027
+ // `ensureNamespacedEntry` leaves an entry with no OMNITRON_PROJECT /
1028
+ // OMNITRON_STACK env alone by design. The app then silently leaves its
1029
+ // stack, because `ProjectService.toStackInfo` filters handles on the
1030
+ // `${project}/${stack}/` prefix.
1031
+ const requested = name.includes('/') && !entry.name.includes('/')
1032
+ ? { ...entry, name }
1033
+ : entry;
1034
+ return this.startApp(requested);
1035
+ }
1036
+ throw new Error(`Unknown app: ${name}`);
1037
+ }
1038
+ name = canonical;
1039
+ const state = this.getCoalescerState(name);
1040
+ if (fromExternal)
1041
+ state.triggerSerial += 1;
1042
+ const observedSerial = state.triggerSerial;
1043
+ // Cooldown: the app reported online recently. Wait it out, then
1044
+ // proceed as a non-external continuation (we already counted this
1045
+ // trigger when we first arrived).
1046
+ const now = Date.now();
1047
+ if (now < state.cooldownUntil) {
1048
+ const waitMs = state.cooldownUntil - now;
1049
+ this.logger.debug({ app: name, cooldownRemainingMs: waitMs }, 'Restart trigger received during post-online cooldown — queued');
1050
+ await sleep(waitMs);
1051
+ return this.restartAppCoalesced(name, /* fromExternal */ false);
1052
+ }
1053
+ // Already restarting — wait for it, then re-enter as a continuation.
1054
+ // The trailing-fire gate at the end of the in-flight finally block
1055
+ // is what re-fires us if our serial outranks what got consumed.
1056
+ if (state.inFlight) {
1057
+ await state.inFlight.catch(() => { });
1058
+ // If the in-flight restart already consumed our serial (or a
1059
+ // later one), we have nothing new to do — return its result by
1060
+ // re-reading the current handle. Otherwise fall through and
1061
+ // claim the slot ourselves.
1062
+ if (state.firedSerial >= observedSerial) {
1063
+ const refreshed = this.handles.get(name);
1064
+ if (refreshed)
1065
+ return refreshed;
1066
+ }
1067
+ return this.restartAppCoalesced(name, /* fromExternal */ false);
1068
+ }
1069
+ // Take the slot. Capture which serial this run is consuming —
1070
+ // anything bumped after this is the next run's problem.
1071
+ const consumedSerial = state.triggerSerial;
1072
+ const promise = this.restartAppNow(name);
1073
+ state.inFlight = promise;
1074
+ try {
1075
+ // Cooldown window is started from the `app:online` transition
1076
+ // inside startApp(), not here, so the timer is tied to the moment
1077
+ // the new child reports ready, not the moment our promise resolves.
1078
+ return await promise;
1079
+ }
1080
+ finally {
1081
+ state.inFlight = null;
1082
+ state.firedSerial = consumedSerial;
1083
+ // Trailing coalesced fire — only if a GENUINELY NEW external
1084
+ // trigger arrived between when we claimed the slot and now.
1085
+ // Two coalescer arms colliding can no longer cause this branch
1086
+ // to fire because they don't bump triggerSerial.
1087
+ if (state.triggerSerial > consumedSerial) {
1088
+ this.restartAppCoalesced(name, /* fromExternal */ false).catch((err) => {
1089
+ this.logger.error({ app: name, error: err.message }, 'Trailing coalesced restart failed');
1090
+ });
1091
+ }
1092
+ }
1093
+ }
1094
+ /**
1095
+ * Internal restart hot path — assumes coalescer has cleared us to proceed.
1096
+ *
1097
+ * Architecture note (Fix #247): we deliberately do NOT tear down the
1098
+ * esbuild watch or invalidate buildResults here. The watch is what
1099
+ * triggered us in the first place — its bundle is already fresh, and
1100
+ * disposing the context creates a window where the next save can be
1101
+ * missed (esbuild's metafile doesn't carry over between contexts).
1102
+ *
1103
+ * `launchBootstrapMode` sees the live buildResult + watch and skips
1104
+ * the cold rebuild, so a restart cycle is just: kill old child →
1105
+ * spawn new child with the bundle that was already on disk. The
1106
+ * bootstrap-config cache is cleared because changes to `bootstrap.ts`
1107
+ * (topology definition) need to take effect on the next start.
1108
+ */
1109
+ /**
1110
+ * Start an app this daemon already supervises, using the entry its handle
1111
+ * carries.
1112
+ *
1113
+ * `startApp(entry)` needs an ecosystem entry, which the daemon's own config
1114
+ * is only one source of — a project stack registers handles this daemon
1115
+ * never declared. `restartAppNow` has always taken the entry off the handle
1116
+ * for exactly that reason; this is the same move for a start, so the name
1117
+ * `list` prints is a name `start` accepts.
1118
+ *
1119
+ * Returns `undefined` when the app is unknown, leaving the caller to decide
1120
+ * what to say about a name that names nothing.
1121
+ */
1122
+ async startKnownApp(name) {
1123
+ const canonical = this.resolveAppName(name) ?? name;
1124
+ const handle = this.handles.get(canonical);
1125
+ if (!handle)
1126
+ return undefined;
1127
+ return this.startApp(handle.entry);
1128
+ }
1129
+ async restartAppNow(name) {
213
1130
  const handle = this.handles.get(name);
214
1131
  if (!handle)
215
1132
  throw new Error(`Unknown app: ${name}`);
216
1133
  const entry = handle.entry;
217
- // In dev mode, clear bootstrap config cache so topology changes are picked up
218
1134
  if (this.devMode && entry.bootstrap) {
219
- clearCacheFor(path.resolve(this.cwd, entry.bootstrap));
220
- // Stop esbuild watch and clear stale build result — launchBootstrapMode will rebuild + re-watch
221
- if (this.buildService) {
222
- await this.buildService.unwatchApp(name);
223
- }
224
- this.buildResults.delete(name);
1135
+ clearCacheFor(path.resolve(entry.cwd ?? this.cwd, entry.bootstrap));
225
1136
  }
226
1137
  await this.stopApp(name);
227
1138
  return this.startApp(entry);
228
1139
  }
1140
+ getCoalescerState(name) {
1141
+ let state = this.restartCoalescer.get(name);
1142
+ if (!state) {
1143
+ state = { inFlight: null, triggerSerial: 0, firedSerial: 0, cooldownUntil: 0 };
1144
+ this.restartCoalescer.set(name, state);
1145
+ }
1146
+ return state;
1147
+ }
1148
+ /**
1149
+ * Start the post-online cooldown for an app. Called from the `app:online`
1150
+ * transition (both initial startApp and subsequent restartApp paths
1151
+ * funnel through this), so a freshly-online app cannot be torn down by
1152
+ * a rebuild signal that was already in flight when it finished booting.
1153
+ */
1154
+ startPostOnlineCooldown(name) {
1155
+ const state = this.getCoalescerState(name);
1156
+ state.cooldownUntil = Date.now() + OrchestratorService.POST_ONLINE_COOLDOWN_MS;
1157
+ }
229
1158
  /**
230
- * Resolve raw app name to its handle key.
231
- * In stack mode, handles are namespaced (e.g., 'omni/dev/main' for app 'main').
232
- * This finds the handle whose key ends with the raw app name.
1159
+ * Resolve a user-supplied name to a canonical handle key.
1160
+ *
1161
+ * Inputs accepted:
1162
+ * - exact canonical key (`omni/dev/payments`) → returned as-is
1163
+ * - bare short name (`payments`) → returns the canonical key whose
1164
+ * last `/`-segment matches, IF AND ONLY IF the match is unique.
1165
+ *
1166
+ * A qualified name that matches no handle exactly resolves to nothing: it
1167
+ * names a deployment this daemon does not have, and the nearest app with
1168
+ * the same last segment is a different one.
1169
+ *
1170
+ * Returns `undefined` for no match. Throws on ambiguity (multiple
1171
+ * stacks expose the same short name) so the caller can surface a
1172
+ * clear error to the operator instead of operating on the wrong
1173
+ * app.
233
1174
  */
234
1175
  resolveAppName(rawName) {
235
1176
  if (this.handles.has(rawName))
236
1177
  return rawName;
1178
+ // A qualified name is an ADDRESS, and the only handle that satisfies it
1179
+ // is the one with that address. Falling through to the short-name match
1180
+ // answered a question about one deployment with another's data:
1181
+ //
1182
+ // omnitron logs daos/deployed/main → daos/dev/main
1183
+ //
1184
+ // — the node's app asked for, this laptop's app printed, with nothing
1185
+ // said. `startApp` already refuses this for the same reason ("a
1186
+ // PROJECT-QUALIFIED name with no live handle is not this daemon's to
1187
+ // start"); the resolver every other caller goes through did not.
1188
+ if (rawName.includes('/'))
1189
+ return undefined;
1190
+ const effective = effectiveAppName(rawName);
1191
+ const matches = [];
237
1192
  for (const key of this.handles.keys()) {
238
- if (key.endsWith(`/${rawName}`))
239
- return key;
1193
+ if (effectiveAppName(key) === effective)
1194
+ matches.push(key);
240
1195
  }
241
- return undefined;
1196
+ if (matches.length === 0)
1197
+ return undefined;
1198
+ if (matches.length === 1)
1199
+ return matches[0];
1200
+ throw new Error(`App name '${rawName}' is ambiguous — multiple matches: ${matches.join(', ')}. Use the full canonical name.`);
242
1201
  }
243
1202
  /**
244
1203
  * Reload app with zero-downtime strategy:
@@ -251,9 +1210,11 @@ export class OrchestratorService extends EventEmitter {
251
1210
  * zero-downtime is not possible (single-process, no supervisor).
252
1211
  */
253
1212
  async reloadApp(name) {
254
- const handle = this.handles.get(name);
1213
+ const canonical = this.resolveAppName(name) ?? name;
1214
+ const handle = this.handles.get(canonical);
255
1215
  if (!handle)
256
1216
  throw new Error(`Unknown app: ${name}`);
1217
+ name = canonical;
257
1218
  // Classic mode or no supervisor — cannot do zero-downtime, fall back to restart
258
1219
  if (handle.mode !== 'bootstrap' || !handle.supervisor) {
259
1220
  return this.restartApp(name);
@@ -279,8 +1240,11 @@ export class OrchestratorService extends EventEmitter {
279
1240
  this.logger.info({ app: name, child: childName }, 'Child reloaded successfully');
280
1241
  }
281
1242
  catch (err) {
282
- this.logger.error({ app: name, child: childName, error: err.message }, 'Failed to reload child — aborting zero-downtime reload');
283
- throw new Error(`Zero-downtime reload failed for '${name}' at child '${childName}': ${err.message}`);
1243
+ // `error: err`, not `error: err.message`: the logger aliases `error`
1244
+ // onto pino's error serializer, so the object arrives with its stack
1245
+ // and a string arrives as a string with nothing behind it.
1246
+ this.logger.error({ app: name, child: childName, error: err }, 'Failed to reload child — aborting zero-downtime reload');
1247
+ throw new Error(`Zero-downtime reload failed for '${name}' at child '${childName}': ${err.message}`, { cause: err });
284
1248
  }
285
1249
  }
286
1250
  this.logger.info({ app: name }, 'Zero-downtime reload complete');
@@ -313,9 +1277,11 @@ export class OrchestratorService extends EventEmitter {
313
1277
  // Scaling — delegates to PM supervisor
314
1278
  // ============================================================================
315
1279
  async scaleApp(name, instances) {
316
- const handle = this.handles.get(name);
1280
+ const canonical = this.resolveAppName(name) ?? name;
1281
+ const handle = this.handles.get(canonical);
317
1282
  if (!handle)
318
1283
  throw new Error(`Unknown app: ${name}`);
1284
+ name = canonical;
319
1285
  if (!this.config)
320
1286
  throw new Error('Ecosystem config not loaded');
321
1287
  if (handle.mode !== 'bootstrap') {
@@ -345,11 +1311,18 @@ export class OrchestratorService extends EventEmitter {
345
1311
  return handles.map((h) => this.toProcessInfo(h));
346
1312
  }
347
1313
  getApp(name) {
348
- const h = this.handles.get(name);
1314
+ const canonical = this.resolveAppName(name) ?? name;
1315
+ const h = this.handles.get(canonical);
349
1316
  return h ? this.toProcessInfo(h) : null;
350
1317
  }
351
1318
  getHandle(name) {
352
- return this.handles.get(name);
1319
+ // Match getAppStatus / getLogs name-resolution semantics — accept either
1320
+ // a fully namespaced handle key ('project/stack/app') or the bare app
1321
+ // name and resolve to the right handle. Without this, CLI commands like
1322
+ // `omnitron env main` fail with "Unknown app: main" even though the
1323
+ // app exists as `omni/dev/main`.
1324
+ const resolved = this.resolveAppName(name);
1325
+ return this.handles.get(resolved ?? name);
353
1326
  }
354
1327
  /**
355
1328
  * List all handle names matching a prefix.
@@ -364,41 +1337,63 @@ export class OrchestratorService extends EventEmitter {
364
1337
  getWorkerHandle(processId) {
365
1338
  return this.pm.getWorkerHandle(processId);
366
1339
  }
1340
+ /**
1341
+ * Surface the underlying IProcessInfo for a titan-pm processId
1342
+ * so callers (e.g. the daemon's `inspect` RPC) can read uptime
1343
+ * + status without reaching into the process-manager directly.
1344
+ */
1345
+ getChildProcessInfo(processId) {
1346
+ return this.pm.getProcess(processId);
1347
+ }
367
1348
  // ============================================================================
368
1349
  // Monitoring — async, delegates to PM supervisor for bootstrap mode
369
1350
  // ============================================================================
1351
+ /**
1352
+ * Pull the live DI dependency graph from a running app's first child
1353
+ * worker. Used by the daemon RPC backing `omnitron inspect <app>
1354
+ * --graph`. Returns null if the app isn't running, has no children,
1355
+ * or the worker doesn't expose `getDependencyGraph` (older bootstraps).
1356
+ */
1357
+ async getDependencyGraph(name) {
1358
+ const handle = this.getHandle(name);
1359
+ if (!handle || handle.mode !== 'bootstrap' || !handle.supervisor)
1360
+ return null;
1361
+ const childNames = handle.supervisor.getChildNames();
1362
+ for (const childName of childNames) {
1363
+ const processId = handle.supervisor.getChildProcessId(childName);
1364
+ if (!processId)
1365
+ continue;
1366
+ const workerHandle = this.pm.getWorkerHandle(processId);
1367
+ const proxy = workerHandle?.proxy;
1368
+ if (!proxy?.getDependencyGraph)
1369
+ continue;
1370
+ try {
1371
+ const graph = await proxy.getDependencyGraph();
1372
+ return graph;
1373
+ }
1374
+ catch (err) {
1375
+ this.logger.warn({ app: name, child: childName, error: err.message }, 'getDependencyGraph: child threw — trying next worker');
1376
+ }
1377
+ }
1378
+ return null;
1379
+ }
1380
+ /**
1381
+ * `name` is resolved like every other per-app question (`main` finds
1382
+ * `daos/dev/main`); it was compared verbatim, so `omnitron metrics main`
1383
+ * printed «(no data)». An unknown name answers `{}`, and the caller says so.
1384
+ */
370
1385
  async getMetrics(name) {
1386
+ const wanted = name === undefined ? undefined : this.resolveAppName(name);
1387
+ if (name !== undefined && wanted === undefined)
1388
+ return {};
371
1389
  const result = {};
372
1390
  for (const [appName, handle] of this.handles) {
373
- if (name && appName !== name)
1391
+ if (wanted && appName !== wanted)
374
1392
  continue;
375
1393
  if (handle.mode === 'bootstrap' && handle.supervisor) {
376
- const childNames = handle.supervisor.getChildNames();
377
- let cpu = 0, memory = 0, requests = 0, errors = 0;
378
- for (const childName of childNames) {
379
- // Try PM-level RPC metrics first
380
- const m = await handle.supervisor.getChildMetrics(childName);
381
- if (m && (m.cpu > 0 || m.memory > 0)) {
382
- cpu += m.cpu;
383
- memory += m.memory;
384
- requests += m.requests ?? 0;
385
- errors += m.errors ?? 0;
386
- }
387
- else {
388
- // Fallback: OS-level metrics via ps for the child PID
389
- const processId = handle.supervisor.getChildProcessId(childName);
390
- if (processId) {
391
- const workerHandle = this.pm.getWorkerHandle(processId);
392
- const childPid = workerHandle?.pid;
393
- if (childPid) {
394
- const osMetrics = await this.sampleProcessMetrics(childPid);
395
- cpu += osMetrics.cpu;
396
- memory += osMetrics.memory;
397
- }
398
- }
399
- }
400
- }
401
- const aggregated = { cpu, memory, requests, errors };
1394
+ // Same sampler the poller uses, so `omnitron metrics` and `omnitron
1395
+ // list` cannot report different numbers for the same app.
1396
+ const aggregated = await this.sampleAppMetrics(handle);
402
1397
  handle.lastMetrics = aggregated;
403
1398
  result[appName] = aggregated;
404
1399
  }
@@ -409,21 +1404,36 @@ export class OrchestratorService extends EventEmitter {
409
1404
  }
410
1405
  return result;
411
1406
  }
1407
+ /**
1408
+ * Each app's health, asked of every process it runs — its supervisor
1409
+ * children and its worker pools — and combined (`app-health.ts`).
1410
+ *
1411
+ * `name` is resolved like every other per-app question: `main` finds
1412
+ * `daos/dev/main`. It was compared verbatim, so `omnitron health main`
1413
+ * matched nothing and the answer was empty. An unknown name answers `{}`,
1414
+ * and the caller says so.
1415
+ */
412
1416
  async getHealth(name) {
1417
+ const wanted = name === undefined ? undefined : this.resolveAppName(name);
1418
+ if (name !== undefined && wanted === undefined)
1419
+ return {};
413
1420
  const result = {};
414
1421
  for (const [appName, handle] of this.handles) {
415
- if (name && appName !== name)
1422
+ if (wanted && appName !== wanted)
1423
+ continue;
1424
+ if (handle.status !== 'online') {
1425
+ result[appName] = notRunningHealth(handle.status);
416
1426
  continue;
1427
+ }
417
1428
  if (handle.mode === 'bootstrap' && handle.supervisor) {
418
- const childNames = handle.supervisor.getChildNames();
419
- if (childNames.length > 0) {
420
- const health = await handle.supervisor.getChildHealth(childNames[0]);
421
- handle.lastHealth = health;
422
- result[appName] = health;
423
- }
424
- else {
425
- result[appName] = handle.lastHealth;
1429
+ const supervisor = handle.supervisor;
1430
+ const answers = await Promise.all(supervisor.getChildNames().map(async (child) => [child, await supervisor.getChildHealth(child)]));
1431
+ for (const [pool, instance] of handle.topologyPools) {
1432
+ answers.push([pool, poolHealth(instance.metrics)]);
426
1433
  }
1434
+ const health = answers.length > 0 ? combineProcessHealth(answers) : handle.lastHealth;
1435
+ handle.lastHealth = health;
1436
+ result[appName] = health;
427
1437
  }
428
1438
  else {
429
1439
  result[appName] = handle.lastHealth;
@@ -488,8 +1498,13 @@ export class OrchestratorService extends EventEmitter {
488
1498
  }
489
1499
  getLogs(name, lines = 100) {
490
1500
  const result = [];
1501
+ // Stack-mode handles are keyed as `<project>/<stack>/<app>` while CLI
1502
+ // users typically pass just the bare app name. Resolve via the same
1503
+ // matcher used by getAppStatus() so `omnitron logs main` works in
1504
+ // both single-app and stack modes.
1505
+ const resolved = name ? this.resolveAppName(name) ?? name : undefined;
491
1506
  for (const [appName, handle] of this.handles) {
492
- if (name && appName !== name)
1507
+ if (resolved && appName !== resolved)
493
1508
  continue;
494
1509
  result.push({ app: appName, lines: handle.getLogs(lines) });
495
1510
  }
@@ -521,7 +1536,7 @@ export class OrchestratorService extends EventEmitter {
521
1536
  else {
522
1537
  this.logger.error({ app: entry.name, code, signal }, 'App exited unexpectedly');
523
1538
  handle.markCrashed();
524
- this.handleClassicCrash(entry, handle, config);
1539
+ this.scheduleRecovery(entry, handle, config);
525
1540
  }
526
1541
  this.persistState();
527
1542
  });
@@ -535,30 +1550,57 @@ export class OrchestratorService extends EventEmitter {
535
1550
  */
536
1551
  async launchBootstrapMode(entry, handle, config) {
537
1552
  handle.markStarting();
538
- const bootstrapAbsPath = path.resolve(this.cwd, entry.bootstrap);
539
- // Load bootstrap definition to check for process topology and requirements
1553
+ const bootstrapAbsPath = path.resolve(entry.cwd ?? this.cwd, entry.bootstrap);
1554
+ // Load bootstrap definition to check for process topology and requirements.
1555
+ //
1556
+ // In dev, in a process of its own (`freshModuleGraph`): imported here, a
1557
+ // package the definition uses would be the one this daemon first saw, for
1558
+ // as long as it runs (bootstrap-loader.ts has the measurement). What comes
1559
+ // back is data, which is all the daemon reads of it — here, in the build
1560
+ // service and in the stack resolver; the app's own processes load the
1561
+ // definition themselves, functions and all.
540
1562
  let topology;
541
1563
  let definition;
542
1564
  try {
543
- definition = await loadBootstrapConfig(bootstrapAbsPath, { devMode: this.devMode });
1565
+ definition = await loadBootstrapConfig(bootstrapAbsPath, { devMode: this.devMode, freshModuleGraph: true });
544
1566
  topology = definition.processes;
545
1567
  }
546
1568
  catch (err) {
547
- this.logger.warn({ app: entry.name, error: err.message }, 'Could not load bootstrap config for topology — using single-process mode');
1569
+ // Refused, with the reason. This fell back to «single-process mode» at
1570
+ // warn — but every definition declares its topology (IAppDefinition:
1571
+ // `processes` is required), so the fallback started the app in a shape
1572
+ // it was never written for. Measured 2026-09-23 on the master: paysys
1573
+ // and messaging, whose definitions failed to load («Parameter
1574
+ // decorators only work when experimental decorators are enabled»), were
1575
+ // started that way and died on «Dependency 'AuthModule:JWTService' not
1576
+ // found» and a module path that single mode could not resolve — the
1577
+ // operator saw those, and the cause sat one line up at warn.
1578
+ throw new Error(`Could not load ${entry.name}'s definition from ${bootstrapAbsPath}: ${err.message}`, { cause: err });
548
1579
  }
549
- // Populate omnitronConfig from app's config/default.json if not set by bootstrap
550
- if (definition && !definition.omnitronConfig) {
551
- try {
552
- const srcDir = path.dirname(bootstrapAbsPath);
553
- const appRoot = path.resolve(srcDir, '..');
554
- const configPath = path.join(appRoot, 'config', 'default.json');
555
- const content = fs.readFileSync(configPath, 'utf-8');
556
- const json = JSON.parse(content);
557
- if (json.omnitron) {
558
- definition.omnitronConfig = json.omnitron;
1580
+ // The app's `config/default.json`, read on every start and put on a COPY
1581
+ // of the definition. This wrote the section onto the loader's cached
1582
+ // definition «if not set», so the first start's reading was every later
1583
+ // start's — and the logger level beside it, read in the same block, was
1584
+ // never read again at all (`project/declared-config.ts`).
1585
+ if (definition) {
1586
+ const declared = readDeclaredConfig(bootstrapAbsPath);
1587
+ if (declared.malformed) {
1588
+ this.logger.error({ app: entry.name, configPath: declared.configPath, error: declared.malformed }, 'config/default.json does not parse — its `omnitron` section is being ignored');
1589
+ }
1590
+ definition = withDeclaredConfig(definition, declared);
1591
+ // The only place the parent can learn the level the app chose; children
1592
+ // are spawned with it below. Validated rather than trusted: this is a
1593
+ // hand-edited JSON file, and a level pino does not know ("verbose", a
1594
+ // typo) would be handed straight to a child's logger constructor.
1595
+ const level = declared.loggerLevel;
1596
+ if (typeof level === 'string') {
1597
+ if (PINO_LEVELS.has(level)) {
1598
+ handle.appLogLevel = level;
1599
+ }
1600
+ else {
1601
+ this.logger.warn({ app: entry.name, configPath: declared.configPath, level }, 'config/default.json declares a logger level this runtime does not know — children keep the default');
559
1602
  }
560
1603
  }
561
- catch { /* config file missing — skip */ }
562
1604
  }
563
1605
  // Resolve infrastructure config and inject as env vars.
564
1606
  // Skip if already injected by project.service.ts (stack flow sets OMNITRON_STACK).
@@ -588,25 +1630,95 @@ export class OrchestratorService extends EventEmitter {
588
1630
  }
589
1631
  }
590
1632
  // Build all entry points via esbuild (dev mode bundles from source .ts)
1633
+ //
1634
+ // On a *restart* (as opposed to a first start), buildResults already
1635
+ // has a fresh entry — esbuild's watcher just produced the bundle
1636
+ // that triggered us. Reusing it skips a redundant cold rebuild
1637
+ // (which was costing ~500ms per restart and racing against the
1638
+ // watcher's own incremental output). The watch context is also
1639
+ // preserved (we never unwatched), so we don't re-arm it either.
591
1640
  if (definition && this.devMode) {
592
1641
  try {
593
1642
  if (!this.buildService) {
594
- this.buildService = new BuildService(/* isDev */ true);
1643
+ this.buildService = new BuildService(/* isDev */ true, {
1644
+ onEsbuildReset: (info) => this.metricsBridge?.recordResilientHandleReset(info),
1645
+ onBuildResult: ({ app, kind, durationMs, ok }) => this.metricsBridge?.recordEsbuildBuild(app, kind, durationMs, ok),
1646
+ });
1647
+ }
1648
+ const existing = this.buildResults.get(entry.name);
1649
+ const existingWatch = this.buildService.isWatching?.(entry.name);
1650
+ // Bundle on-disk presence check. A restart can race against
1651
+ // the user (or a script) removing `.omnitron-build/` between
1652
+ // builds — without this stat the orchestrator hands the
1653
+ // worker a path that no longer exists and the child dies
1654
+ // with `ERR_MODULE_NOT_FOUND` before the supervisor can
1655
+ // recover. Cheap stat, big DX win.
1656
+ const bundlePresent = existing
1657
+ ? fs.existsSync(existing.bootstrapPath)
1658
+ : false;
1659
+ if (existing && existingWatch && bundlePresent) {
1660
+ // Restart path — keep the already-built bundle and live watch.
1661
+ this.logger.debug({ app: entry.name, modules: existing.modulePaths.size }, 'Reusing existing esbuild bundle + watch (restart path)');
595
1662
  }
596
- const buildResult = await this.buildService.buildApp(entry.name, bootstrapAbsPath, definition);
597
- this.buildResults.set(entry.name, buildResult);
598
- this.logger.info({ app: entry.name, modules: buildResult.modulePaths.size }, 'Built entry points via esbuild');
599
- // Start esbuild watch for incremental rebuilds on source changes.
600
- // On rebuild → restart the app (stop + re-build + re-launch).
601
- await this.buildService.watchApp(entry.name, bootstrapAbsPath, definition, () => {
602
- this.logger.info({ app: entry.name }, 'esbuild rebuild detected — restarting');
603
- this.restartApp(entry.name).catch((err) => {
604
- this.logger.error({ app: entry.name, error: err.message }, 'Restart after esbuild rebuild failed');
1663
+ else {
1664
+ if (existing && existingWatch && !bundlePresent) {
1665
+ // Bundle was deleted out from under us. Clear the watch
1666
+ // and the cached BuildResult so the next call rebuilds
1667
+ // from cold and re-arms the watcher cleanly.
1668
+ this.logger.warn({ app: entry.name, bootstrapPath: existing.bootstrapPath }, 'Cached bundle missing on disk — forcing rebuild');
1669
+ try {
1670
+ await this.buildService.unwatchApp?.(entry.name);
1671
+ }
1672
+ catch (err) {
1673
+ this.logger.debug({ err, app: entry.name }, 'unwatchApp failed (non-fatal)');
1674
+ }
1675
+ this.buildResults.delete(entry.name);
1676
+ }
1677
+ // First-start path — build everything from cold and arm the watcher.
1678
+ const buildResult = await this.buildService.buildApp(entry.name, bootstrapAbsPath, definition);
1679
+ this.buildResults.set(entry.name, buildResult);
1680
+ this.logger.info({ app: entry.name, modules: buildResult.modulePaths.size }, 'Built entry points via esbuild');
1681
+ // Start esbuild watch for incremental rebuilds on source changes.
1682
+ // On rebuild → fire a restart trigger; the coalescer takes it
1683
+ // from there.
1684
+ await this.buildService.watchApp(entry.name, bootstrapAbsPath, definition, () => {
1685
+ // Gate: skip if the app is in a non-running state — repeated
1686
+ // restarts of an escalated/errored process just churn forever
1687
+ // without forward progress (Fix #248).
1688
+ // Skip restart when the app is in a terminal non-running
1689
+ // state. Includes 'crashed' (supervisor escalated after
1690
+ // exceeding its restart budget) and 'errored' (start
1691
+ // failed). 'stopped' means operator explicitly stopped it.
1692
+ // In all three cases, restarting just churns a known-broken
1693
+ // process; the operator must fix the underlying issue and
1694
+ // call `omnitron start <app>` to clear the state.
1695
+ const status = this.getAppStatus(entry.name);
1696
+ if (status === 'errored' || status === 'crashed' || status === 'stopped') {
1697
+ this.logger.warn({ app: entry.name, status }, 'esbuild rebuild detected but app is not running — skipping restart');
1698
+ return;
1699
+ }
1700
+ this.logger.info({ app: entry.name }, 'esbuild rebuild detected — restarting');
1701
+ this.metricsBridge?.recordEsbuildBuild(entry.name, 'rebuild', 0, true);
1702
+ this.restartApp(entry.name).catch((err) => {
1703
+ this.logger.error({ app: entry.name, error: err.message }, 'Restart after esbuild rebuild failed');
1704
+ });
605
1705
  });
606
- });
1706
+ }
607
1707
  }
608
1708
  catch (err) {
609
- this.logger.error({ app: entry.name, error: err.message }, 'esbuild failed — child processes will fall back to tsx');
1709
+ // The "fall back to tsx" wording was misleading — the orchestrator
1710
+ // does not inject a tsx loader into spawned children, so without
1711
+ // a build the bootstrap-process will try to `import` the source
1712
+ // .ts file (referenced as `./app.module.js` in the topology) and
1713
+ // fail with a cryptic "Cannot find module" downstream. Surface
1714
+ // the build error directly so the operator sees the actual
1715
+ // problem on `omnitron start/restart`.
1716
+ // esbuild's error carries an `errors[]` with file, line and column.
1717
+ // `.message` is the summary; the locations are what an operator needs
1718
+ // to fix the build, and they exist only on the original.
1719
+ this.logger.error({ app: entry.name, error: err }, 'esbuild build failed — app cannot start without bundled output');
1720
+ throw new Error(`Cannot launch '${entry.name}' in dev mode: esbuild build failed (${err.message}). ` +
1721
+ `Fix the build error and retry, or run with bundled output present in apps/${entry.name}/.omnitron-build/.`, { cause: err });
610
1722
  }
611
1723
  }
612
1724
  const buildResult = this.buildResults.get(entry.name);
@@ -634,10 +1746,11 @@ export class OrchestratorService extends EventEmitter {
634
1746
  */
635
1747
  async launchTopology(entry, handle, config, bootstrapAbsPath, topology, buildResult) {
636
1748
  const policy = buildRestartPolicy(entry, config);
637
- if (!this.daemonNetron || !this.daemonSocketPath) {
638
- throw new Error(`Cannot launch topology for '${entry.name}': daemon Netron not available. ` +
639
- 'Ensure setDaemonNetron() is called before starting apps.');
640
- }
1749
+ // Wait for daemon Netron to be wired up. Cheap no-op if already ready;
1750
+ // otherwise blocks (with timeout) instead of immediately failing — the
1751
+ // race between daemon startup and stack auto-resume is benign and just
1752
+ // needs a single rendezvous.
1753
+ await this.ensureDaemonNetronReady();
641
1754
  const serviceRouter = new ServiceRouter(this.daemonNetron, this.logger);
642
1755
  const daemonSocketUrl = `unix://${this.daemonSocketPath}`;
643
1756
  // Topology startup order: providers (topology.expose) before consumers (topology.access).
@@ -658,14 +1771,20 @@ export class OrchestratorService extends EventEmitter {
658
1771
  size: instances,
659
1772
  strategy: PoolStrategy.POWER_OF_TWO,
660
1773
  metrics: true,
661
- replaceUnhealthy: true,
662
- requestTimeout: 120_000,
1774
+ // `replaceUnhealthy` is deliberately not set: titan-pm declares it
1775
+ // and reads it nowhere. An unhealthy worker is replaced by the
1776
+ // health path regardless, so `true` was a no-op that happened to
1777
+ // match the behaviour — and `false` would have been a no-op that did
1778
+ // not, leaving an operator believing they had turned replacement off.
1779
+ // What actually governs it is `healthCheck.unhealthyThreshold`.
1780
+ requestTimeout: CHILD_REQUEST_TIMEOUT,
663
1781
  maxQueueSize: 200,
664
1782
  spawnOptions: {
665
1783
  name: `${entry.name}/${procEntry.name}`,
666
1784
  version: '1.0.0',
667
1785
  allMethodsPublic: true,
668
- startupTimeout: procEntry.startupTimeout ?? 60_000,
1786
+ ...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
1787
+ startupTimeout: procEntry.startupTimeout ?? entry.startupTimeout ?? DEFAULT_CHILD_STARTUP_TIMEOUT,
669
1788
  ...(entry.env && { env: entry.env }),
670
1789
  ...(entry.cwd && { cwd: entry.cwd }),
671
1790
  dependencies: {
@@ -688,48 +1807,86 @@ export class OrchestratorService extends EventEmitter {
688
1807
  enabled: procEntry.health?.enabled !== false,
689
1808
  interval: procEntry.health?.interval ?? config.monitoring.healthCheck.interval,
690
1809
  },
1810
+ // `scaling.strategy` is what opts a pool into autoscaling; anything
1811
+ // else — including the default of not declaring it — means the pool
1812
+ // stays at the declared `instances`, which is what 'fixed' says on
1813
+ // the tin.
1814
+ //
1815
+ // Both arms of this ternary used to set `enabled: true`, differing
1816
+ // only in where the numbers came from. The setting existed, the
1817
+ // mechanism existed, and nothing connected them: every pool in the
1818
+ // platform was autoscaled whether or not it asked to be. Live effect
1819
+ // on two pools declared `instances: 2` — grown to their cap of eight
1820
+ // and held there, sixteen processes where four were configured.
1821
+ //
1822
+ // `min` follows the declared instance count rather than 1: a pool
1823
+ // asked for two workers should not be scaled below two.
691
1824
  autoScale: procEntry.scaling?.strategy === 'auto'
692
1825
  ? {
693
1826
  enabled: true,
694
- min: 1,
1827
+ min: instances,
695
1828
  max: procEntry.scaling?.maxInstances ?? instances * 4,
696
1829
  targetCPU: procEntry.scaling?.targetCPU ?? 70,
697
1830
  targetMemory: procEntry.scaling?.targetMemory ?? 80,
698
1831
  queueThreshold: procEntry.scaling?.queueThreshold ?? 10,
699
1832
  cooldownPeriod: procEntry.scaling?.cooldownPeriod ?? 30_000,
700
1833
  }
701
- : {
702
- enabled: true,
703
- min: 1,
704
- max: instances * 4,
705
- targetCPU: 70,
706
- targetMemory: 80,
707
- queueThreshold: 10,
708
- cooldownPeriod: 30_000,
709
- },
1834
+ : { enabled: false, min: instances, max: instances },
710
1835
  };
711
1836
  this.logger.info({ app: entry.name, process: procEntry.name, instances }, 'Creating topology process pool');
712
1837
  // pm.pool() returns a Proxy wrapping ProcessPool — execute() dispatches to pool workers
713
1838
  const pool = await this.pm.pool(BOOTSTRAP_PROCESS_PATH, poolOptions);
714
1839
  poolNames.push(procEntry.name);
1840
+ // Track the pool on the AppHandle so `getInfo()` can report the
1841
+ // actual worker count for pool-managed topology entries. Without
1842
+ // this the orchestrator falls back to `supervisor.getChildProcessId()`
1843
+ // which returns null for pool workers (they aren't supervisor
1844
+ // children) and the entry is mis-reported as `status: 'stopped'`.
1845
+ handle.topologyPools.set(procEntry.name, pool);
1846
+ this.attachPoolLogCapture(entry.name, procEntry.name, handle, pool);
715
1847
  // Auto-discover @Service metadata from pool workers and expose via ServiceRouter.
716
1848
  // No manual `exports` needed — @Service decorator IS the source of truth.
717
1849
  // Requires explicit `topology: { expose: true }` — opt-in for security.
718
1850
  if (procEntry.topology?.expose === true) {
719
1851
  try {
1852
+ // A pool worker reports ready before its Application has started, and
1853
+ // `@Service` classes are exposed to Netron during start — so asking
1854
+ // once, or once more 500ms later, asks a worker that is still booting.
1855
+ // What it answers then is an empty list, or a service whose method
1856
+ // list has not been filled in yet, and both used to end the same way:
1857
+ // nothing registered on the daemon, the consumer process starting
1858
+ // seconds later, querying, and being told the service does not exist.
1859
+ // The consumer resolves its topology proxy exactly once, so that
1860
+ // answer disabled the feature for the life of the process — in the
1861
+ // case that surfaced this, every OHLCV candle aggregation.
1862
+ //
1863
+ // Poll instead, and treat "a service with no methods" as not ready
1864
+ // rather than as a service, because an empty method list produces a
1865
+ // proxy that can do nothing.
1866
+ const usable = (list) => Array.isArray(list) && list.length > 0 && list.some((svc) => svc.methods?.length > 0);
720
1867
  let services = await pool.execute('getExposedServices');
721
- // Defensive: if services list is empty, the worker's Application may still
722
- // be registering services. Retry once after a short delay.
723
- if (!services || services.length === 0) {
724
- await new Promise((r) => setTimeout(r, 500));
1868
+ const deadline = Date.now() + TOPOLOGY_EXPOSE_TIMEOUT_MS;
1869
+ while (!usable(services) && Date.now() < deadline) {
1870
+ await new Promise((r) => setTimeout(r, TOPOLOGY_EXPOSE_POLL_MS));
725
1871
  services = await pool.execute('getExposedServices');
726
1872
  }
727
- if (!services || services.length === 0) {
728
- this.logger.warn({ app: entry.name, process: procEntry.name }, 'No @Service found on pool worker — topology.expose has no effect');
1873
+ if (!usable(services)) {
1874
+ this.logger.warn({
1875
+ app: entry.name,
1876
+ process: procEntry.name,
1877
+ waitedMs: TOPOLOGY_EXPOSE_TIMEOUT_MS,
1878
+ reported: services?.map((svc) => `${svc.name}(${svc.methods?.length ?? 0})`) ?? [],
1879
+ }, 'No usable @Service found on pool worker — topology.expose has no effect, and any ' +
1880
+ 'process declaring topology.access on it will start without a proxy');
729
1881
  }
730
- for (const svc of services) {
731
- if (svc.methods.length === 0)
1882
+ for (const svc of services ?? []) {
1883
+ if (svc.methods.length === 0) {
1884
+ // Silently skipped before. A consumer then failed to find a
1885
+ // service its own declaration named, with nothing anywhere
1886
+ // connecting the two facts.
1887
+ this.logger.warn({ app: entry.name, process: procEntry.name, service: svc.name }, 'Pool service exposes no methods — not registered on the daemon');
732
1888
  continue;
1889
+ }
733
1890
  await serviceRouter.exposePoolService(procEntry.name, svc.name, svc.version ?? '1.0.0', pool, svc.methods);
734
1891
  this.logger.info({ app: entry.name, process: procEntry.name, service: svc.name, methods: svc.methods.length }, 'Auto-discovered and exposed pool service on daemon Netron');
735
1892
  }
@@ -748,7 +1905,10 @@ export class OrchestratorService extends EventEmitter {
748
1905
  name: `${entry.name}/${procEntry.name}`,
749
1906
  version: '1.0.0',
750
1907
  allMethodsPublic: true,
751
- startupTimeout: procEntry.startupTimeout ?? entry.startupTimeout ?? 30_000,
1908
+ // Without this the child's client takes netron's 5000 ms; the pool
1909
+ // path beside it has always stated one. See CHILD_REQUEST_TIMEOUT.
1910
+ requestTimeout: CHILD_REQUEST_TIMEOUT,
1911
+ startupTimeout: procEntry.startupTimeout ?? entry.startupTimeout ?? DEFAULT_CHILD_STARTUP_TIMEOUT,
752
1912
  ...(entry.env && { env: entry.env }),
753
1913
  ...(entry.cwd && { cwd: entry.cwd }),
754
1914
  dependencies: {
@@ -766,6 +1926,7 @@ export class OrchestratorService extends EventEmitter {
766
1926
  timeout: procEntry.health?.timeout ?? config.monitoring.healthCheck.timeout,
767
1927
  },
768
1928
  observability: { metrics: true },
1929
+ ...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
769
1930
  };
770
1931
  children.push({
771
1932
  name: `${entry.name}/${procEntry.name}`,
@@ -781,25 +1942,20 @@ export class OrchestratorService extends EventEmitter {
781
1942
  window: policy.window ?? config.supervision.window,
782
1943
  backoff: policy.backoff ?? config.supervision.backoff,
783
1944
  children,
784
- onChildCrash: async (_child, _error) => {
785
- // Always restart — supervisor's maxRestarts/window handles escalation.
786
- // Critical flag determines whether exceeding max restarts kills the app.
787
- return RestartDecision.RESTART;
788
- },
1945
+ // No custom onChildCrash — supervisor's built-in budget gate
1946
+ // (process-supervisor.ts:524-531) does what we want:
1947
+ // * within maxRestarts/window → RESTART
1948
+ // * over budget → ESCALATE (if child is critical) or IGNORE
1949
+ // Wiring an unconditional RESTART here masked the budget and
1950
+ // turned every crashing child into an infinite respawn storm.
789
1951
  };
790
1952
  // Create supervisor WITHOUT starting — wire events first to avoid race condition
791
1953
  const supervisor = this.pm.createSupervisor(supervisorConfig);
792
- supervisor.on('child:started', (childName) => {
793
- const processId = supervisor.getChildProcessId(childName);
794
- if (processId) {
795
- const workerHandle = this.pm.getWorkerHandle(processId);
796
- const info = this.pm.getProcess(processId);
797
- const childPid = workerHandle?.pid ?? info?.pid;
798
- handle.markOnline(childPid ?? process.pid);
799
- }
800
- this.persistState();
801
- this.emit('app:online', entry.name);
802
- });
1954
+ // The app's pid is its server's: the single process with a transport, the
1955
+ // one `handle.port` names. A topology with none has no pid to give.
1956
+ const server = singleEntries.find((p) => p.transports?.http) ?? singleEntries.find((p) => p.transports) ?? null;
1957
+ handle.declareChildren(children.map((child) => child.name), server ? `${entry.name}/${server.name}` : null);
1958
+ this.onChildStarted(entry, handle, supervisor);
803
1959
  // Set supervisor on handle BEFORE start so attachLogCapture can access it
804
1960
  handle.supervisor = supervisor;
805
1961
  handle.serviceRouter = serviceRouter;
@@ -808,6 +1964,73 @@ export class OrchestratorService extends EventEmitter {
808
1964
  this.wireSupervisorEvents(entry, handle, supervisor);
809
1965
  // Now start — events will be captured correctly
810
1966
  await supervisor.start();
1967
+ // Step 4: register the single processes that declare `topology.expose`.
1968
+ //
1969
+ // This was a pool-only feature, ignored in silence everywhere else: a
1970
+ // process declaring `expose: true` got no registration and no warning,
1971
+ // and every consumer naming it in `topology.access` started without a
1972
+ // proxy. Measured on priceverse, whose `collector` is a single process
1973
+ // holding the only objects that know whether the exchange WebSockets are
1974
+ // up — the server process could not ask, and answered `ready: down —
1975
+ // exchanges unavailable` while three were connected.
1976
+ //
1977
+ // After `supervisor.start()` because there is no child to ask before it.
1978
+ // A consumer among these children may therefore start before the service
1979
+ // is registered; consumers re-resolve their proxy on use for exactly that
1980
+ // reason (see priceverse's `OhlcvSchedulerService`).
1981
+ await this.exposeChildTopologyServices(entry, supervisor, serviceRouter, singleEntries);
1982
+ }
1983
+ /**
1984
+ * Register each single process's `@Service` classes on the daemon.
1985
+ *
1986
+ * Mirrors the pool path above, including its poll: a child reports ready
1987
+ * before its Application has started, and `@Service` classes are exposed to
1988
+ * Netron during start, so asking once asks a process that is still booting
1989
+ * and gets an empty list back.
1990
+ */
1991
+ async exposeChildTopologyServices(entry, supervisor, serviceRouter, singleEntries) {
1992
+ for (const procEntry of singleEntries) {
1993
+ if (procEntry.topology?.expose !== true)
1994
+ continue;
1995
+ const childName = `${entry.name}/${procEntry.name}`;
1996
+ const getProxy = () => supervisor.getChildProxy(childName);
1997
+ try {
1998
+ const usable = (list) => Array.isArray(list) && list.length > 0 && list.some((svc) => svc.methods?.length > 0);
1999
+ const ask = async () => {
2000
+ const child = getProxy();
2001
+ if (!child?.getExposedServices)
2002
+ return undefined;
2003
+ return (await child.getExposedServices());
2004
+ };
2005
+ let services = await ask();
2006
+ const deadline = Date.now() + TOPOLOGY_EXPOSE_TIMEOUT_MS;
2007
+ while (!usable(services) && Date.now() < deadline) {
2008
+ await new Promise((r) => setTimeout(r, TOPOLOGY_EXPOSE_POLL_MS));
2009
+ services = await ask();
2010
+ }
2011
+ if (!usable(services)) {
2012
+ this.logger.warn({
2013
+ app: entry.name,
2014
+ process: procEntry.name,
2015
+ waitedMs: TOPOLOGY_EXPOSE_TIMEOUT_MS,
2016
+ reported: services?.map((svc) => `${svc.name}(${svc.methods?.length ?? 0})`) ?? [],
2017
+ }, 'No usable @Service found on this process — topology.expose has no effect, and any ' +
2018
+ 'process declaring topology.access on it will start without a proxy');
2019
+ continue;
2020
+ }
2021
+ for (const svc of services ?? []) {
2022
+ if (svc.methods.length === 0) {
2023
+ this.logger.warn({ app: entry.name, process: procEntry.name, service: svc.name }, 'Process service exposes no methods — not registered on the daemon');
2024
+ continue;
2025
+ }
2026
+ await serviceRouter.exposeChildService(procEntry.name, svc.name, svc.version ?? '1.0.0', getProxy, svc.methods);
2027
+ this.logger.info({ app: entry.name, process: procEntry.name, service: svc.name, methods: svc.methods.length }, 'Auto-discovered and exposed process service on daemon Netron');
2028
+ }
2029
+ }
2030
+ catch (err) {
2031
+ this.logger.error({ app: entry.name, process: procEntry.name, error: err.message }, "Failed to auto-discover/expose this process's services");
2032
+ }
2033
+ }
811
2034
  }
812
2035
  /**
813
2036
  * Launch single-process bootstrap app (no topology — original behavior).
@@ -823,11 +2046,22 @@ export class OrchestratorService extends EventEmitter {
823
2046
  const spawnOpts = {
824
2047
  name: entry.name,
825
2048
  version: '1.0.0',
2049
+ // As above: a single child's client otherwise runs on netron's default.
2050
+ requestTimeout: CHILD_REQUEST_TIMEOUT,
826
2051
  ...(entry.env && { env: entry.env }),
827
2052
  dependencies: {
828
2053
  bootstrapPath: bootstrapAbsPath,
829
2054
  ...(bundledModulePath && { bundledModulePath }),
830
2055
  },
2056
+ // Omitted when unset, which delegates the budget to titan-pm —
2057
+ // `process-spawner.ts`: `options.startupTimeout ?? resources?.timeout ??
2058
+ // 30_000`. Worth naming, because it is otherwise invisible from here:
2059
+ // an APP that declares nothing gets 30s while its CHILD processes get
2060
+ // `DEFAULT_CHILD_STARTUP_TIMEOUT`, 60s. Backwards if anything, since the
2061
+ // app usually does the heavier work — and the downstream project's own
2062
+ // config carries a note about a 30s default that kept a whole stack from
2063
+ // starting. Left as delegation rather than changed here: raising it is a
2064
+ // decision about titan-pm's contract, not about this call site.
831
2065
  ...(entry.startupTimeout != null && { startupTimeout: entry.startupTimeout }),
832
2066
  health: {
833
2067
  enabled: true,
@@ -835,6 +2069,7 @@ export class OrchestratorService extends EventEmitter {
835
2069
  timeout: config.monitoring.healthCheck.timeout,
836
2070
  },
837
2071
  observability: { metrics: true },
2072
+ ...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
838
2073
  };
839
2074
  const childConfig = {
840
2075
  name: entry.name,
@@ -847,7 +2082,8 @@ export class OrchestratorService extends EventEmitter {
847
2082
  size: instances,
848
2083
  strategy: PoolStrategy.ROUND_ROBIN,
849
2084
  metrics: true,
850
- replaceUnhealthy: true,
2085
+ // See the note on the other pool: `replaceUnhealthy` is declared by
2086
+ // titan-pm and read by nothing.
851
2087
  healthCheck: {
852
2088
  enabled: true,
853
2089
  interval: config.monitoring.healthCheck.interval,
@@ -876,27 +2112,19 @@ export class OrchestratorService extends EventEmitter {
876
2112
  window: policy.window ?? config.supervision.window,
877
2113
  backoff: policy.backoff ?? config.supervision.backoff,
878
2114
  children: [childConfig],
879
- onChildCrash: async (_child, _error) => {
880
- // Always restart — supervisor's maxRestarts/window handles escalation.
881
- // Critical flag determines whether exceeding max restarts kills the app.
882
- return RestartDecision.RESTART;
883
- },
2115
+ // No custom onChildCrash — supervisor's built-in budget gate
2116
+ // (process-supervisor.ts:524-531) does what we want:
2117
+ // * within maxRestarts/window → RESTART
2118
+ // * over budget → ESCALATE (if child is critical) or IGNORE
2119
+ // Wiring an unconditional RESTART here masked the budget and
2120
+ // turned every crashing child into an infinite respawn storm.
884
2121
  };
885
2122
  // Create supervisor WITHOUT starting — wire events first to avoid race condition
886
2123
  const supervisor = this.pm.createSupervisor(supervisorConfig);
887
- // Wire supervisor events BEFORE start so we catch the initial child:started
888
- supervisor.on('child:started', (childName) => {
889
- const processId = supervisor.getChildProcessId(childName);
890
- if (processId) {
891
- // Prefer WorkerHandle.pid (actual child OS PID), fallback to IProcessInfo.pid
892
- const workerHandle = this.pm.getWorkerHandle(processId);
893
- const info = this.pm.getProcess(processId);
894
- const childPid = workerHandle?.pid ?? info?.pid;
895
- handle.markOnline(childPid ?? process.pid);
896
- }
897
- this.persistState();
898
- this.emit('app:online', entry.name);
899
- });
2124
+ // Wire supervisor events BEFORE start so we catch the initial child:started.
2125
+ // One child, and it is the whole app — so it is the server too.
2126
+ handle.declareChildren([childConfig.name], childConfig.name);
2127
+ this.onChildStarted(entry, handle, supervisor);
900
2128
  // Set supervisor on handle BEFORE start so attachLogCapture can access it
901
2129
  handle.supervisor = supervisor;
902
2130
  handle.instanceCount = instances;
@@ -904,14 +2132,50 @@ export class OrchestratorService extends EventEmitter {
904
2132
  // Now start — events will be captured correctly
905
2133
  await supervisor.start();
906
2134
  }
2135
+ /**
2136
+ * A child's start, for both launch paths: the app is online when the start
2137
+ * completes it, and not before (`AppHandle.childStarted`).
2138
+ *
2139
+ * Each path had its own copy of this handler, and both marked the app
2140
+ * online — with that child's pid — on every child's start. `app:online`
2141
+ * went out once per child with it: three STARTED events for one boot of a
2142
+ * three-process app, the first two about an app still coming up.
2143
+ */
2144
+ onChildStarted(entry, handle, supervisor) {
2145
+ supervisor.on('child:started', (childName) => {
2146
+ // A pool child's proxy answers every property with a function —
2147
+ // `__processId` included — so only a string is a process id. The old
2148
+ // handler took that function for one, found no worker behind it, and
2149
+ // gave such an app the DAEMON's pid.
2150
+ const processId = supervisor.getChildProcessId(childName);
2151
+ const pid = typeof processId === 'string'
2152
+ ? (this.pm.getWorkerHandle(processId)?.pid ?? this.pm.getProcess(processId)?.pid ?? null)
2153
+ : null;
2154
+ const whole = handle.childStarted(childName, pid);
2155
+ this.persistState();
2156
+ if (!whole)
2157
+ return;
2158
+ this.startPostOnlineCooldown(entry.name);
2159
+ this.emit('app:online', entry.name);
2160
+ });
2161
+ }
907
2162
  /**
908
2163
  * Wire common supervisor events (crash, restart, escalate) and log capture.
909
2164
  */
910
2165
  wireSupervisorEvents(entry, handle, supervisor) {
911
2166
  supervisor.on('child:started', (childName) => {
912
2167
  this.attachLogCapture(entry.name, childName, handle);
2168
+ this.attachExitCapture(entry.name, childName, handle);
2169
+ // The app re-entered a live state — drop the previous crash
2170
+ // context so a future `omnitron inspect` shows the *next*
2171
+ // crash, not the last one.
2172
+ handle.clearLastExit();
913
2173
  });
2174
+ // A child that is down leaves the app partial until it is back — a
2175
+ // sibling's restart must not bring the app `online` around it.
2176
+ supervisor.on('child:stopped', (childName) => handle.childGone(childName));
914
2177
  supervisor.on('child:crash', (childName, error) => {
2178
+ handle.childGone(childName);
915
2179
  // If app never reached 'online', mark as errored (not crashed — it never ran)
916
2180
  if (handle.status === 'starting') {
917
2181
  handle.markErrored();
@@ -919,10 +2183,94 @@ export class OrchestratorService extends EventEmitter {
919
2183
  else {
920
2184
  handle.markCrashed();
921
2185
  }
2186
+ // Stash the supervisor-provided error message onto whatever
2187
+ // exit context the workerHandle.onExit hook already wrote. If
2188
+ // the exit event hasn't fired yet (race) we still need to
2189
+ // persist *something* — record a synthetic entry that the
2190
+ // exit hook below will overwrite when it does fire.
2191
+ if (handle.lastExit) {
2192
+ handle.lastExit.message = error.message;
2193
+ }
2194
+ else {
2195
+ handle.recordExit({ code: null, signal: null, expected: false, message: error.message });
2196
+ }
922
2197
  this.logger.error({ app: entry.name, child: childName, error: error.message }, 'Process crashed');
923
2198
  this.persistState();
924
2199
  this.emit('app:crash', entry.name, error);
925
2200
  });
2201
+ // T#60 — startup failures (child throws/rejects in waitForReady
2202
+ // before ever reporting ready) come through this event, NOT
2203
+ // child:crash. Without a handler the orchestrator just sees
2204
+ // status='errored' with no clue what went wrong. titan-pm
2205
+ // already stuffs the captured stderr into `error.details.stderr`
2206
+ // so we surface it on `lastExit.stderrTail`.
2207
+ supervisor.on('child:start-failed', (childName, error) => {
2208
+ const details = error.details;
2209
+ const stderrTail = typeof details?.stderr === 'string'
2210
+ ? details.stderr.split('\n').filter((l) => l.length > 0).slice(-80)
2211
+ : [];
2212
+ for (const line of stderrTail)
2213
+ handle.appendStderr(line);
2214
+ // A child that never came up because WE stopped the app is not a
2215
+ // failure, and reporting it as one is worse than saying nothing: the
2216
+ // message titan-pm builds for an aborted start says the process "did not
2217
+ // choose to exit, so its own logs will not explain this" — true, and it
2218
+ // sends the reader hunting a fault in an application that was doing
2219
+ // nothing wrong. Every `stack stop` issued while anything was still
2220
+ // booting wrote one of these per child, at error level, into the app's
2221
+ // own log.
2222
+ //
2223
+ // `'stopping'` only — NOT `'stopped'`, which is also an AppHandle's
2224
+ // initial value, so accepting it would swallow the first genuine failure
2225
+ // of any app whose handle had not yet been marked starting.
2226
+ const expected = handle.status === 'stopping' ||
2227
+ error.stoppedDuringStartup === true;
2228
+ if (expected) {
2229
+ handle.recordExit({
2230
+ code: null,
2231
+ signal: null,
2232
+ expected: true,
2233
+ message: error.message,
2234
+ });
2235
+ this.logger.info({ app: entry.name, child: childName, err: error.message }, 'Process stopped before it finished starting');
2236
+ this.persistState();
2237
+ return;
2238
+ }
2239
+ // Put what the child printed into the app's LOG FILE, not only into the
2240
+ // in-memory ring `omnitron inspect` reads.
2241
+ //
2242
+ // A child that dies during startup never produces a WorkerHandle at all
2243
+ // — titan-pm registers one only after `waitForReady` resolves — so there
2244
+ // is no `onLog` to subscribe to here and nothing to replay. What does
2245
+ // exist is the output `waitForReady` captured while waiting, which it
2246
+ // attaches to the error. Feeding that to the log handlers is the only
2247
+ // way a failed boot reaches disk, and a failed boot is the case where
2248
+ // reading it matters most: an application that dies before its logger
2249
+ // exists has said everything it is ever going to say on these streams.
2250
+ // The failure itself first. A child can fail without printing a word —
2251
+ // it reports the error over the IPC channel and exits, which is what a
2252
+ // module that throws on import does — so the streams below are often
2253
+ // empty and the only account of what happened is on the error.
2254
+ this.persistChildFailure(entry.name, childName, error);
2255
+ this.persistChildOutput(entry.name, childName, details?.stdout, 'stdout');
2256
+ this.persistChildOutput(entry.name, childName, details?.stderr, 'stderr');
2257
+ handle.recordExit({
2258
+ code: null,
2259
+ signal: null,
2260
+ expected: false,
2261
+ message: error.message,
2262
+ });
2263
+ // The supervisor gives up on a non-critical child whose start failed (a
2264
+ // critical one throws, and `startApp` says so). That went unseen while
2265
+ // the first child's start made the app `online`; now the app would wait
2266
+ // for a child that is not coming, in `starting` — which `omnitron start`
2267
+ // takes for a start in progress and leaves alone. A declared process
2268
+ // that did not start is an app that did not start.
2269
+ if (handle.status === 'starting')
2270
+ handle.markErrored();
2271
+ this.logger.error({ app: entry.name, child: childName, err: error.message }, 'Process failed to start');
2272
+ this.persistState();
2273
+ });
926
2274
  supervisor.on('child:restart', (childName, attempt) => {
927
2275
  handle.restarts = attempt;
928
2276
  this.logger.info({ app: entry.name, child: childName, attempt }, 'Process restarting');
@@ -949,9 +2297,45 @@ export class OrchestratorService extends EventEmitter {
949
2297
  const workerHandle = this.pm.getWorkerHandle(processId);
950
2298
  if (!workerHandle?.onLog)
951
2299
  return;
952
- workerHandle.onLog((line, _stream) => {
2300
+ workerHandle.onLog(this.childLogForwarder(appName, handle));
2301
+ }
2302
+ /**
2303
+ * Attach log capture to the workers of a topology POOL.
2304
+ *
2305
+ * `attachLogCapture` above is driven by `child:started`, which only
2306
+ * supervisor children emit — a pool worker is not one, so nothing ever
2307
+ * subscribed to its output. Everything those workers print went nowhere:
2308
+ * the OHLCV aggregator and the image transformer have been mute since they
2309
+ * were written, and since titan-pm started holding pre-capture output it was
2310
+ * also a megabyte per worker retained for a reader that never arrived.
2311
+ *
2312
+ * Attached once, at pool creation. A worker replaced later by autoscaling or
2313
+ * a health restart is not covered; these pools are fixed-size, and the
2314
+ * alternative is an event this package does not expose yet.
2315
+ */
2316
+ attachPoolLogCapture(appName, processName, handle, pool) {
2317
+ const forward = this.childLogForwarder(appName, handle);
2318
+ let attached = 0;
2319
+ for (const workerId of pool.getWorkerIds()) {
2320
+ const workerHandle = pool.getWorkerHandle(workerId);
2321
+ if (!workerHandle?.onLog)
2322
+ continue;
2323
+ workerHandle.onLog(forward);
2324
+ attached++;
2325
+ }
2326
+ this.logger.debug({ app: appName, process: processName, workers: attached }, 'Pool log capture attached');
2327
+ }
2328
+ /** The one place a child's line becomes an app's line. */
2329
+ childLogForwarder(appName, handle) {
2330
+ return (line, stream) => {
953
2331
  // In-memory ring buffer (for live CLI tailing)
954
2332
  handle.appendLog(line);
2333
+ // Separate stderr ring — feeds `lastExit.stderrTail` on death.
2334
+ // Without this, the operator running `omnitron inspect` after
2335
+ // a crash sees no hint at WHY (the in-flight pipe is gone).
2336
+ if (stream === 'stderr') {
2337
+ handle.appendStderr(line);
2338
+ }
955
2339
  // Persist to disk via registered log handlers (LogManager)
956
2340
  for (const handler of this.appLogHandlers) {
957
2341
  try {
@@ -961,6 +2345,108 @@ export class OrchestratorService extends EventEmitter {
961
2345
  // Log handler failure must not break the app
962
2346
  }
963
2347
  }
2348
+ };
2349
+ }
2350
+ /**
2351
+ * Write the failure a child reported before it could start to the app's log
2352
+ * handlers, as one structured line.
2353
+ *
2354
+ * `error.childStack` is the stack from inside the child when it managed to
2355
+ * send one — for a module that throws on import that is the entire
2356
+ * diagnosis, and it exists nowhere else once the process is gone.
2357
+ */
2358
+ persistChildFailure(appName, childName, error) {
2359
+ const childStack = error.childStack;
2360
+ const entry = JSON.stringify({
2361
+ level: 50,
2362
+ time: new Date().toISOString(),
2363
+ processName: qualifyChildName(appName, childName),
2364
+ event: 'process.start_failed',
2365
+ err: { message: error.message, ...(childStack ? { stack: childStack } : {}) },
2366
+ msg: `Process failed to start: ${error.message}`,
2367
+ });
2368
+ for (const handler of this.appLogHandlers) {
2369
+ try {
2370
+ handler(appName, entry);
2371
+ }
2372
+ catch {
2373
+ // A log handler must not break failure reporting.
2374
+ }
2375
+ }
2376
+ }
2377
+ /**
2378
+ * Write a dead child's captured output to the app's log handlers.
2379
+ *
2380
+ * Emitted as pino-shaped JSON so it lands in the same file, at the same
2381
+ * level, as everything else that app ever logged — a reader tailing
2382
+ * `app.log` after a failed start finds the reason there rather than having
2383
+ * to know that this particular failure hides in a different place.
2384
+ */
2385
+ persistChildOutput(appName, childName, output, stream) {
2386
+ if (!output)
2387
+ return;
2388
+ const time = new Date().toISOString();
2389
+ for (const line of output.split('\n')) {
2390
+ if (!line.trim())
2391
+ continue;
2392
+ // Already-structured lines pass through untouched; the child may well
2393
+ // have got far enough to produce them.
2394
+ const entry = line.trimStart().startsWith('{')
2395
+ ? line
2396
+ : JSON.stringify({
2397
+ // stderr is not error. A boot script's own progress markers go
2398
+ // there, and a failing start emits a run of them — classifying
2399
+ // every one at 50 filled the log with errors that are nothing of
2400
+ // the sort, next to the single line that IS the failure. The
2401
+ // failure is reported at 50 by `persistChildFailure`; this is the
2402
+ // context around it, and `stream` keeps it greppable.
2403
+ level: stream === 'stderr' ? 40 : 30,
2404
+ time,
2405
+ processName: qualifyChildName(appName, childName),
2406
+ stream,
2407
+ msg: line,
2408
+ });
2409
+ for (const handler of this.appLogHandlers) {
2410
+ try {
2411
+ handler(appName, entry);
2412
+ }
2413
+ catch {
2414
+ // A log handler must not break failure reporting.
2415
+ }
2416
+ }
2417
+ }
2418
+ }
2419
+ /**
2420
+ * Subscribe to the WorkerHandle's `exit` event so the orchestrator
2421
+ * keeps the exit code / signal / "expected" flag past the
2422
+ * WorkerHandle's GC. Pre-fix the supervisor's `child:crash` event
2423
+ * fired with an Error but never surfaced the underlying
2424
+ * code/signal — operators had no way to tell SIGKILL from a
2425
+ * thrown TypeError. The captured info lands on
2426
+ * `handle.lastExit` and is surfaced by `omnitron inspect`.
2427
+ */
2428
+ attachExitCapture(appName, childName, handle) {
2429
+ const supervisor = handle.supervisor;
2430
+ if (!supervisor)
2431
+ return;
2432
+ const processId = supervisor.getChildProcessId(childName);
2433
+ if (!processId)
2434
+ return;
2435
+ const workerHandle = this.pm.getWorkerHandle(processId);
2436
+ if (!workerHandle?.onExit)
2437
+ return;
2438
+ workerHandle.onExit((info) => {
2439
+ // Preserve the supervisor's error message if `child:crash`
2440
+ // already fired before us. recordExit overwrites the synthetic
2441
+ // entry with the authoritative code/signal.
2442
+ const message = handle.lastExit?.message;
2443
+ handle.recordExit({
2444
+ code: info.code,
2445
+ signal: info.signal,
2446
+ expected: info.expected,
2447
+ ...(message !== undefined && { message }),
2448
+ });
2449
+ this.logger.debug({ app: appName, child: childName, code: info.code, signal: info.signal, expected: info.expected }, 'Captured worker exit context');
964
2450
  });
965
2451
  }
966
2452
  // ============================================================================
@@ -970,30 +2456,58 @@ export class OrchestratorService extends EventEmitter {
970
2456
  const child = handle.childProcess;
971
2457
  if (!child || !child.pid)
972
2458
  return;
2459
+ const pid = child.pid;
2460
+ // P0-D — Signal the whole process group, not just the direct
2461
+ // child. classic-launcher spawns with `detached: true` so the
2462
+ // child becomes group leader (pgid === pid). `process.kill(-pid)`
2463
+ // delivers the signal to every process in the group, reaping
2464
+ // grandchildren (ffmpeg, docker, worker pools …) that ignore the
2465
+ // direct-child kill. Falls back to `child.kill` when the negative-
2466
+ // pid call fails (test environments, restricted permissions, the
2467
+ // child wasn't started detached — bootstrap-mode currently isn't).
2468
+ const sendSignal = (sig) => {
2469
+ try {
2470
+ process.kill(-pid, sig);
2471
+ return;
2472
+ }
2473
+ catch (err) {
2474
+ const code = err.code;
2475
+ if (code !== 'ESRCH' && code !== 'EPERM') {
2476
+ // ESRCH/EPERM = group already gone or not a group leader.
2477
+ // Anything else (EINVAL etc.) → log via best-effort.
2478
+ this.logger.debug({ app: handle.name, pid, sig, code }, 'process.kill(-pid) fallback');
2479
+ }
2480
+ }
2481
+ try {
2482
+ child.kill(sig);
2483
+ }
2484
+ catch { /* best-effort */ }
2485
+ };
973
2486
  await new Promise((resolve) => {
974
2487
  const timer = setTimeout(() => {
975
- try {
976
- child.kill('SIGKILL');
977
- }
978
- catch {
979
- // SIGKILL send is best-effort
980
- }
2488
+ sendSignal('SIGKILL');
981
2489
  resolve();
982
2490
  }, timeout);
983
2491
  child.once('exit', () => {
984
2492
  clearTimeout(timer);
985
2493
  resolve();
986
2494
  });
987
- try {
988
- child.kill(force ? 'SIGKILL' : 'SIGTERM');
989
- }
990
- catch {
991
- clearTimeout(timer);
992
- resolve();
993
- }
2495
+ sendSignal(force ? 'SIGKILL' : 'SIGTERM');
994
2496
  });
995
2497
  }
996
- async handleClassicCrash(entry, handle, config) {
2498
+ /**
2499
+ * Cancel a pending crash cool-down for `name`, if one is armed.
2500
+ * An explicit stop means "this app is going down now, stay down",
2501
+ * which the cool-down must not undo ten minutes later.
2502
+ */
2503
+ clearCoolDown(name) {
2504
+ const timer = this.coolDownTimers.get(name);
2505
+ if (timer) {
2506
+ clearTimeout(timer);
2507
+ this.coolDownTimers.delete(name);
2508
+ }
2509
+ }
2510
+ async scheduleRecovery(entry, handle, config) {
997
2511
  const policy = buildRestartPolicy(entry, config);
998
2512
  if (!policy.enabled)
999
2513
  return;
@@ -1001,87 +2515,416 @@ export class OrchestratorService extends EventEmitter {
1001
2515
  handle.status = 'crashed';
1002
2516
  this.logger.error({ app: entry.name }, 'Max restarts exceeded');
1003
2517
  this.emit('app:escalated', entry.name);
2518
+ // P1-I — auto-recovery after a cool-down. pm2 has this as
2519
+ // `restart_delay` + a fresh attempt window; pre-fix omnitron
2520
+ // left the app permanently 'crashed' until human intervention.
2521
+ // We reset the counter + schedule one fresh attempt 10 minutes
2522
+ // out. If THAT attempt also crashes through maxRestarts we
2523
+ // land back here and pause another 10 minutes — bounded
2524
+ // self-healing without thrashing.
2525
+ //
2526
+ // The wait happens OUTSIDE the lifecycle queue, and that is the
2527
+ // whole point. It used to be a queued `crash-restart` whose
2528
+ // `run()` slept for the entire cool-down — but `enqueue` will
2529
+ // not preempt a RUNNING head, and with an empty queue the
2530
+ // sleeper always became the head immediately. So the `onPreempt`
2531
+ // hook meant to cancel it could never fire, and an operator's
2532
+ // `stop` or `restart` simply queued behind ten minutes of sleep:
2533
+ // the CLI's RPC timed out after 60s while the op stayed queued,
2534
+ // to fire whenever the cool-down finally elapsed. An app that
2535
+ // exhausted its restart budget was unreachable by every command
2536
+ // the CLI offers — `restart` hung, `stop` hung, `stack start`
2537
+ // skipped it — and the only recovery was restarting the daemon.
2538
+ const COOL_DOWN_MS = 10 * 60_000;
2539
+ this.clearCoolDown(entry.name);
2540
+ const timer = setTimeout(() => {
2541
+ this.coolDownTimers.delete(entry.name);
2542
+ // Re-read the handle: every start replaces it (see
2543
+ // `startAppInternal`), so the one captured above is stale the
2544
+ // moment anything restarts this app. Only a still-crashed app
2545
+ // wants this attempt — an operator who started it meanwhile
2546
+ // has already settled the question.
2547
+ const current = this.handles.get(entry.name);
2548
+ if (!current || current.status !== 'crashed')
2549
+ return;
2550
+ this.logger.info({ app: entry.name }, 'Cool-down elapsed — attempting fresh recovery');
2551
+ current.restarts = 0; // fresh attempt window
2552
+ current.status = 'stopped';
2553
+ void this.lifecycleQueue
2554
+ .enqueue(entry.name, 'crash-restart', () => this.startAppInternal(entry, config))
2555
+ .catch((err) => {
2556
+ if (err instanceof LifecyclePreempted)
2557
+ return;
2558
+ this.logger.error({ app: entry.name, err: err.message }, 'Cool-down recovery failed');
2559
+ });
2560
+ }, COOL_DOWN_MS);
2561
+ timer.unref?.();
2562
+ this.coolDownTimers.set(entry.name, timer);
1004
2563
  return;
1005
2564
  }
1006
2565
  const backoff = policy.backoff ?? config.supervision.backoff;
1007
2566
  const delay = this.calculateBackoff(handle.restarts, backoff);
1008
2567
  this.logger.info({ app: entry.name, delay, attempt: handle.restarts + 1 }, 'Scheduling restart');
1009
- const timer = setTimeout(async () => {
1010
- // Guard: don't restart if orchestrator was stopped during backoff
1011
- if (handle.status === 'stopping' || handle.status === 'stopped')
1012
- return;
1013
- try {
1014
- handle.status = 'stopped';
1015
- await this.startApp(entry, config);
1016
- }
1017
- catch (err) {
1018
- this.logger.error({ app: entry.name, error: err.message }, 'Restart failed');
2568
+ // P0-I — schedule the crash-restart through the lifecycle queue
2569
+ // so an explicit stop preempts the pending restart before it
2570
+ // fires. The setTimeout is still the timing source, but the
2571
+ // queue.enqueue with `onPreempt` guarantees the timer is cleared
2572
+ // (and the promise rejected with LifecyclePreempted) the moment a
2573
+ // higher-priority op for the same app arrives. Pre-fix the timer
2574
+ // could fire during stopApp's `await stopChild(...)` window and
2575
+ // spawn a zombie under the same handle.
2576
+ let timer = null;
2577
+ const restartPromise = this.lifecycleQueue.enqueue(entry.name, 'crash-restart', () => new Promise((resolve, reject) => {
2578
+ timer = setTimeout(async () => {
2579
+ timer = null;
2580
+ // Guard: don't restart if orchestrator was stopped during backoff
2581
+ if (handle.status === 'stopping' || handle.status === 'stopped') {
2582
+ resolve();
2583
+ return;
2584
+ }
2585
+ try {
2586
+ handle.status = 'stopped';
2587
+ // `startAppInternal`, not `startApp`.
2588
+ //
2589
+ // This task holds the lifecycle-queue slot for `entry.name`.
2590
+ // `startApp` enqueues 'start' on that same key, and the queue
2591
+ // cannot run it until this task returns — so the await never
2592
+ // settles and the restart silently never happens. Measured:
2593
+ // `startApp:enter` with no matching exit, and the app left in
2594
+ // whatever state the crash put it in.
2595
+ //
2596
+ // This is why the whole policy below — bounded attempts,
2597
+ // exponential backoff, the ten-minute cool-down described in its
2598
+ // own comment as "bounded self-healing" — had never restarted
2599
+ // anything. The queue was introduced (P0-I) to stop a restart
2600
+ // timer racing a stop; it also made the restart unreachable, and
2601
+ // nothing failed loudly enough to say so.
2602
+ await this.startAppInternal(entry, config);
2603
+ resolve();
2604
+ }
2605
+ catch (err) {
2606
+ const message = err.message;
2607
+ this.logger.error({ app: entry.name, error: message }, 'Restart failed');
2608
+ // Said separately and after, because reading the process table
2609
+ // takes a moment and the failure itself must not wait for it.
2610
+ void this.explainPortConflict(message).then((held) => {
2611
+ if (held)
2612
+ this.logger.error({ app: entry.name }, `Restart failed — ${held}`);
2613
+ });
2614
+ reject(err);
2615
+ }
2616
+ }, delay);
2617
+ }), () => {
2618
+ // onPreempt — explicit stop / restart arrived. Cancel the
2619
+ // pending timer so the restart never fires.
2620
+ if (timer) {
2621
+ clearTimeout(timer);
2622
+ timer = null;
1019
2623
  }
1020
- }, delay);
1021
- // Allow cleanup on explicit stop
2624
+ });
2625
+ // Swallow LifecyclePreempted — that's the explicit-stop happy
2626
+ // path, not an error. Anything else gets logged.
2627
+ restartPromise.catch((err) => {
2628
+ if (err instanceof LifecyclePreempted)
2629
+ return;
2630
+ this.logger.warn({ app: entry.name, err: err.message }, 'Crash restart task rejected');
2631
+ });
2632
+ // Legacy field — callers still poke handle.crashRestartTimer to
2633
+ // check "is a restart pending"; keep it in sync with the queued
2634
+ // timer so existing inspect / status code paths don't break.
1022
2635
  handle.crashRestartTimer = timer;
1023
2636
  }
1024
2637
  calculateBackoff(attempt, backoff) {
1025
- const initial = backoff.initial ?? 1000;
1026
- const max = backoff.max ?? 30_000;
1027
- const factor = backoff.factor ?? 2;
1028
- if (backoff.type === 'exponential')
1029
- return Math.min(initial * Math.pow(factor, attempt), max);
1030
- if (backoff.type === 'linear')
1031
- return Math.min(initial * (attempt + 1), max);
1032
- return initial;
1033
- }
1034
- /** Sample CPU/memory for a single OS process via `ps` (async — does not block event loop) */
2638
+ // Delegate to the shared backoff library in titan-pm. Pre-fix the
2639
+ // inline impl here defaulted `initial=1000` while supervisor's
2640
+ // copy defaulted to 300 — same config produced 3.3× longer
2641
+ // delays in classic-mode than bootstrap-mode. Now both routes
2642
+ // run the same math.
2643
+ return computeBackoff(attempt + 1, backoff);
2644
+ }
2645
+ /**
2646
+ * The live OS processes behind one topology entry.
2647
+ *
2648
+ * Single-instance entries run as supervisor children; entries with
2649
+ * `instances > 1` run in a pool the supervisor knows nothing about. Both
2650
+ * shapes are resolved here, in one place, because metrics sampling and
2651
+ * status reporting each used to resolve them separately — and disagreed.
2652
+ * The pool branch asserted `online` from `pool.size > 0` before looking at
2653
+ * a pid at all, so a worker whose process had exited still read as online
2654
+ * (observed live: two pool workers reporting `online` on pids that `kill
2655
+ * -0` said were gone).
2656
+ */
2657
+ resolveTopologyPids(handle, topo) {
2658
+ return this.resolveTopologyMembers(handle, topo).members.map((m) => m.pid);
2659
+ }
2660
+ /**
2661
+ * The live processes behind one topology entry, keyed by the PM process id
2662
+ * as well as the OS pid.
2663
+ *
2664
+ * The pid alone answers "is it alive"; the process id is what any question
2665
+ * ABOUT that process has to be asked with — `pm.getProcess(id)` carries the
2666
+ * spawn time, and the supervisor's restart counter is keyed by child name.
2667
+ * `resolveTopologyPids` used to discard both on the way out, which is why
2668
+ * the `omnitron list` DTO had nothing to report but the parent's numbers.
2669
+ */
2670
+ resolveTopologyMembers(handle, topo) {
2671
+ const isPoolEntry = (topo.instances ?? 1) > 1;
2672
+ if (isPoolEntry) {
2673
+ const pool = handle.topologyPools.get(topo.name);
2674
+ if (!pool || pool.size === 0)
2675
+ return { members: [], childName: null };
2676
+ const members = pool
2677
+ .getWorkerIds()
2678
+ .map((processId) => ({ processId, pid: this.pm.getWorkerHandle(processId)?.pid }))
2679
+ .filter((m) => typeof m.pid === 'number');
2680
+ return { members, childName: null };
2681
+ }
2682
+ const childNames = handle.supervisor?.getChildNames() ?? [];
2683
+ const childName = childNames.find((cn) => cn.includes(topo.name)) ?? topo.name;
2684
+ const processId = handle.supervisor?.getChildProcessId(childName);
2685
+ if (!processId)
2686
+ return { members: [], childName };
2687
+ const pid = this.pm.getWorkerHandle(processId)?.pid;
2688
+ return { members: typeof pid === 'number' ? [{ processId, pid }] : [], childName };
2689
+ }
2690
+ /**
2691
+ * Sample every process an app owns, and record the per-entry readings on
2692
+ * the handle on the way through.
2693
+ *
2694
+ * A bootstrap app is several OS processes, so "the memory this app uses"
2695
+ * is their sum. The periodic poller used to sample the single pid stored
2696
+ * on the handle — which for a multi-process app is whichever child was
2697
+ * registered last — and overwrite `lastMetrics` with it every five
2698
+ * seconds, while `getMetrics()` right beside it aggregated correctly and
2699
+ * wrote to the same field. The poller won on frequency, so the aggregate
2700
+ * was never what anyone saw: `acme/dev/main` reported 122 MB against 437 MB
2701
+ * actually resident across its three processes.
2702
+ *
2703
+ * ## Why `ps` and not the child's own report
2704
+ *
2705
+ * The aggregating path preferred the PM-level metrics a child reports over
2706
+ * its RPC channel, falling back to `ps`. Those two are not the same
2707
+ * quantities:
2708
+ *
2709
+ * - `cpu` from the child is `(cpuUsage().user + .system) / 1e6` — CPU
2710
+ * SECONDS CONSUMED SINCE START, a monotonically growing counter. `ps`
2711
+ * `%cpu` is a percentage. The dashboard column is labelled `CPU %`,
2712
+ * so an app running for a day would have read `86400`. Observed
2713
+ * climbing live: 12.845 to 13.297 across twelve seconds of idling.
2714
+ * - `memory` from the child is `memoryUsage().heapUsed` — the V8 heap.
2715
+ * `ps` reports RSS, which is what "memory this app uses" means to
2716
+ * whoever is sizing a host. Observed: 66 MB heap against 149 MB
2717
+ * resident for the same process.
2718
+ *
2719
+ * Mixing them per-child, depending on which source answered first, gave a
2720
+ * column that was neither quantity. `ps` is asked for both figures here so
2721
+ * that every number under one label means one thing.
2722
+ */
2723
+ async sampleAppMetrics(handle) {
2724
+ const topology = handle.topologyProcesses ?? [];
2725
+ const childNames = handle.supervisor?.getChildNames() ?? [];
2726
+ // Which pids belong to which topology entry. Everything is sampled in a
2727
+ // single `ps` call: sampling one child at a time would fork a process per
2728
+ // child per polling interval, and a monitor that costs more than what it
2729
+ // monitors is its own kind of defect.
2730
+ const pidsByEntry = new Map();
2731
+ for (const topo of topology) {
2732
+ pidsByEntry.set(topo.name, this.resolveTopologyPids(handle, topo));
2733
+ }
2734
+ if (topology.length === 0) {
2735
+ // No declared topology: fall back to the supervisor's children, and
2736
+ // failing that to the handle's own pid.
2737
+ for (const childName of childNames) {
2738
+ const processId = handle.supervisor?.getChildProcessId(childName);
2739
+ const pid = processId ? this.pm.getWorkerHandle(processId)?.pid : undefined;
2740
+ if (typeof pid === 'number')
2741
+ pidsByEntry.set(childName, [pid]);
2742
+ }
2743
+ if (pidsByEntry.size === 0 && handle.pid) {
2744
+ pidsByEntry.set(handle.name, [handle.pid]);
2745
+ }
2746
+ }
2747
+ const batch = await this.sampleProcessMetricsBatch([...pidsByEntry.values()].flat());
2748
+ let cpu = 0;
2749
+ let memory = 0;
2750
+ const sampled = new Map();
2751
+ for (const [name, pids] of pidsByEntry) {
2752
+ const sample = { cpu: 0, memory: 0 };
2753
+ for (const pid of pids) {
2754
+ const os = batch.get(pid);
2755
+ if (!os)
2756
+ continue;
2757
+ sample.cpu += os.cpu;
2758
+ sample.memory += os.memory;
2759
+ }
2760
+ sampled.set(name, sample);
2761
+ cpu += sample.cpu;
2762
+ memory += sample.memory;
2763
+ }
2764
+ // Traffic can only come from inside the child; `ps` knows nothing about
2765
+ // it. Its CPU and memory figures are deliberately NOT used — see the note
2766
+ // on this method. Only what a child reports as TRAFFIC is summed: the
2767
+ // `requests`/`errors` a child on an older runtime reports are the call
2768
+ // counters of its wrapper — the supervisor's own health and metrics
2769
+ // calls — and adding them up counted this daemon's questions as the
2770
+ // app's requests (`combineProcessTraffic`).
2771
+ const reports = [];
2772
+ for (const childName of childNames) {
2773
+ const reported = await handle.supervisor?.getChildMetrics(childName);
2774
+ reports.push(reported?.traffic);
2775
+ }
2776
+ const traffic = combineProcessTraffic(reports);
2777
+ // A sample that produced nothing for an app that HAS live processes is a
2778
+ // failed sample, not a measurement of zero — `ps` can time out under load,
2779
+ // and it did: an app was observed reporting 0 MB one second and 143 MB the
2780
+ // next. Overwriting the last known reading with zeros would put the exact
2781
+ // placeholder this method exists to remove back on the dashboard, so the
2782
+ // previous reading is kept instead.
2783
+ const sampledNothing = batch.size === 0 && [...pidsByEntry.values()].some((pids) => pids.length > 0);
2784
+ if (sampledNothing && handle.lastMetrics) {
2785
+ return handle.lastMetrics;
2786
+ }
2787
+ handle.childMetrics = sampled;
2788
+ return traffic
2789
+ ? {
2790
+ cpu,
2791
+ memory,
2792
+ requests: traffic.requests,
2793
+ errors: traffic.serverErrors,
2794
+ traffic,
2795
+ ...(traffic.latency
2796
+ ? {
2797
+ latency: {
2798
+ p50: traffic.latency.p50,
2799
+ p75: traffic.latency.p75,
2800
+ p90: traffic.latency.p90,
2801
+ p95: traffic.latency.p95,
2802
+ p99: traffic.latency.p99,
2803
+ mean: traffic.latency.mean,
2804
+ },
2805
+ }
2806
+ : {}),
2807
+ }
2808
+ : { cpu, memory };
2809
+ }
2810
+ /**
2811
+ * Sample several processes in one `ps` call.
2812
+ *
2813
+ * Pids the OS no longer knows are simply absent from the result — an
2814
+ * omission the caller can see, unlike a zeroed reading it cannot tell
2815
+ * apart from an idle process.
2816
+ */
2817
+ async sampleProcessMetricsBatch(pids) {
2818
+ const out = new Map();
2819
+ const unique = [...new Set(pids)];
2820
+ if (unique.length === 0)
2821
+ return out;
2822
+ try {
2823
+ const psOutput = await new Promise((resolve, reject) => {
2824
+ execFile('ps', ['-p', unique.join(','), '-o', 'pid=,rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
2825
+ // `ps` exits non-zero when NONE of the pids exist, but still
2826
+ // prints the ones that do — so stdout is worth reading either way.
2827
+ if (err && !stdout)
2828
+ reject(err);
2829
+ else
2830
+ resolve(stdout.trim());
2831
+ });
2832
+ });
2833
+ for (const [pid, sample] of parsePsBatch(psOutput))
2834
+ out.set(pid, sample);
2835
+ }
2836
+ catch {
2837
+ // Leave the map empty: no reading is honest, a zero reading is not.
2838
+ }
2839
+ return out;
2840
+ }
2841
+ /**
2842
+ * Sample CPU/memory for a single OS process via `ps`.
2843
+ *
2844
+ * `null` when it could not be read. This used to return
2845
+ * `{ cpu: 0, memory: 0 }` on any failure, and the caller wrote that
2846
+ * straight over `handle.lastMetrics` — so a `ps` that timed out under load
2847
+ * made a running classic-mode app read as idle, indistinguishable from one
2848
+ * that genuinely was. The batch path above already treated absence as
2849
+ * absence; these two now agree.
2850
+ */
1035
2851
  async sampleProcessMetrics(pid) {
1036
2852
  try {
1037
- const { execFile } = await import('node:child_process');
1038
2853
  const psOutput = await new Promise((resolve, reject) => {
1039
- execFile('ps', ['-p', String(pid), '-o', 'rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
1040
- if (err)
2854
+ execFile('ps', ['-p', String(pid), '-o', 'pid=,rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
2855
+ if (err && !stdout)
1041
2856
  reject(err);
1042
2857
  else
1043
2858
  resolve(stdout.trim());
1044
2859
  });
1045
2860
  });
1046
- const [rssKb, cpuPercent] = psOutput.split(/\s+/);
1047
- return {
1048
- cpu: parseFloat(cpuPercent ?? '0'),
1049
- memory: parseInt(rssKb ?? '0', 10) * 1024,
1050
- };
2861
+ return parsePsLine(psOutput)?.sample ?? null;
1051
2862
  }
1052
2863
  catch {
1053
- return { cpu: 0, memory: 0 };
2864
+ return null;
1054
2865
  }
1055
2866
  }
1056
- /** Periodic metrics polling for classic mode processes */
2867
+ /**
2868
+ * Periodic CPU/memory sampling for every managed process.
2869
+ *
2870
+ * Idempotent: re-arming replaces the previous timer rather than stacking a
2871
+ * second one, so `setConfig` can call it on every config reload.
2872
+ */
1057
2873
  startMetricsPolling(interval) {
2874
+ if (this.metricsTimer) {
2875
+ clearInterval(this.metricsTimer);
2876
+ this.metricsTimer = null;
2877
+ }
1058
2878
  this.metricsTimer = setInterval(async () => {
1059
2879
  for (const handle of this.handles.values()) {
1060
2880
  if (handle.status !== 'online')
1061
2881
  continue;
1062
2882
  if (handle.mode === 'classic' && handle.childProcess?.pid) {
1063
2883
  const m = await this.sampleProcessMetrics(handle.childProcess.pid);
1064
- handle.lastMetrics = { ...m, requests: 0, errors: 0 };
2884
+ // A failed read keeps the previous numbers rather than replacing
2885
+ // them with zeros — the same rule the bootstrap path below uses.
2886
+ if (m) {
2887
+ handle.lastMetrics = {
2888
+ ...m,
2889
+ requests: handle.lastMetrics?.requests ?? 0,
2890
+ errors: handle.lastMetrics?.errors ?? 0,
2891
+ };
2892
+ }
1065
2893
  }
1066
- else if (handle.mode === 'bootstrap' && handle.pid) {
1067
- const m = await this.sampleProcessMetrics(handle.pid);
1068
- handle.lastMetrics = {
1069
- ...m,
1070
- requests: handle.lastMetrics?.requests ?? 0,
1071
- errors: handle.lastMetrics?.errors ?? 0,
1072
- };
2894
+ else if (handle.mode === 'bootstrap') {
2895
+ // Every process the app owns, not just the one on the handle.
2896
+ // As sampled. The counts used to fall back to the previous reading
2897
+ // whenever a sample said 0 — which kept a stale figure alive
2898
+ // across a process restart (traffic counts restart from zero with
2899
+ // the process) and, while `requests` was the wrapper's call count,
2900
+ // mixed the two meanings in one field.
2901
+ handle.lastMetrics = await this.sampleAppMetrics(handle);
1073
2902
  }
1074
2903
  }
1075
2904
  }, interval);
2905
+ this.metricsTimer.unref();
1076
2906
  }
1077
2907
  // ============================================================================
1078
2908
  // Private — Utilities
1079
2909
  // ============================================================================
1080
2910
  toProcessInfo(handle) {
2911
+ // Ghost-online guard. The in-memory `handle.status` reflects the
2912
+ // last event the supervisor observed. If a child died between
2913
+ // the last sweep and now (kill -9 by an operator, OOM kill,
2914
+ // container restart) the handle still claims 'online' until the
2915
+ // next event lands. A cheap signal-0 verification at DTO-mapping
2916
+ // time keeps `omnitron list` honest: when the OS reports the pid
2917
+ // is gone, surface 'crashed' instead of lying to the operator.
2918
+ // `isAlive` covers EPERM as alive (correct — process exists, we
2919
+ // just can't signal it).
2920
+ let status = handle.status;
2921
+ if (status === 'online' && handle.pid != null && !isAlive(handle.pid)) {
2922
+ status = 'crashed';
2923
+ }
1081
2924
  const info = {
1082
2925
  name: handle.name,
1083
2926
  pid: handle.pid,
1084
- status: handle.status,
2927
+ status,
1085
2928
  cpu: handle.cpu,
1086
2929
  memory: handle.memory,
1087
2930
  uptime: handle.uptime,
@@ -1095,28 +2938,49 @@ export class OrchestratorService extends EventEmitter {
1095
2938
  if (handle.topologyProcesses && handle.topologyProcesses.length > 0 && handle.supervisor) {
1096
2939
  info.processes = [];
1097
2940
  for (const topo of handle.topologyProcesses) {
1098
- const childNames = handle.supervisor.getChildNames();
1099
- // Match topology entry to supervisor child by name
1100
- const childName = childNames.find((cn) => cn.includes(topo.name)) ?? topo.name;
1101
- const processId = handle.supervisor.getChildProcessId(childName);
1102
- let childPid = null;
1103
- let childStatus = 'stopped';
1104
- if (processId) {
1105
- const wh = this.pm.getWorkerHandle(processId);
1106
- childPid = wh?.pid ?? null;
1107
- childStatus = childPid ? 'online' : 'stopped';
1108
- }
2941
+ const isPoolEntry = (topo.instances ?? 1) > 1;
2942
+ // The same ghost-online guard the parent gets a few lines above.
2943
+ // Without it the pool branch reported `online` from the pool's own
2944
+ // bookkeeping, never checking whether the pid it printed alongside
2945
+ // still existed.
2946
+ const { members, childName } = this.resolveTopologyMembers(handle, topo);
2947
+ const alive = members.filter((m) => isAlive(m.pid));
2948
+ const declaredInstances = topo.instances ?? 1;
2949
+ const liveInstances = isPoolEntry
2950
+ ? (handle.topologyPools.get(topo.name)?.size ?? 0)
2951
+ : alive.length;
2952
+ const chosen = alive[0] ?? members[0];
2953
+ const childPid = chosen?.pid ?? null;
2954
+ const childStatus = alive.length > 0 ? 'online' : members.length > 0 ? 'crashed' : 'stopped';
2955
+ // The row prints one pid, so the row's clock is that pid's clock.
2956
+ // `startTime` is written when the process id is registered and a
2957
+ // restart registers a new one, so it moves when the process does.
2958
+ // (`IProcessInfo.restartCount` does NOT: it is assigned 0 there and
2959
+ // incremented nowhere. The supervisor keeps the real count, by child
2960
+ // name — reading the field instead would have replaced a wrong
2961
+ // constant with a wrong lookup.)
2962
+ const chosenInfo = chosen ? this.pm.getProcess(chosen.processId) : undefined;
2963
+ const childUptime = typeof chosenInfo?.startTime === 'number' && chosenInfo.startTime > 0
2964
+ ? Math.max(0, Date.now() - chosenInfo.startTime)
2965
+ : 0;
2966
+ // A pool is not a supervisor child and nothing counts its restarts.
2967
+ // `null` says so; `0` would be absence written as a value from the
2968
+ // domain, leaving "never restarted" and "not tracked" identical.
2969
+ const childRestarts = isPoolEntry || !childName ? null : (handle.supervisor?.getRestartCount(childName) ?? null);
1109
2970
  // Derive process type from declarations for backward-compatible DTO
1110
- const derivedType = topo.transports ? 'server' : (topo.instances ?? 1) > 1 ? 'worker' : 'custom';
2971
+ const derivedType = topo.transports ? 'server' : isPoolEntry ? 'worker' : 'custom';
2972
+ const sample = handle.childMetrics.get(topo.name);
1111
2973
  info.processes.push({
1112
2974
  name: topo.name,
1113
2975
  type: derivedType,
1114
2976
  pid: childPid,
1115
2977
  status: childStatus,
1116
- cpu: 0,
1117
- memory: 0,
1118
- uptime: handle.uptime,
1119
- restarts: 0,
2978
+ cpu: sample?.cpu ?? 0,
2979
+ memory: sample?.memory ?? 0,
2980
+ uptime: childUptime,
2981
+ restarts: childRestarts,
2982
+ instances: liveInstances,
2983
+ declaredInstances,
1120
2984
  });
1121
2985
  }
1122
2986
  }