@omnitron-dev/omnitron 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1247) hide show
  1. package/dist/BUILD.json +5 -0
  2. package/dist/cli/omnitron.js +351 -29
  3. package/dist/cli/omnitron.js.map +1 -1
  4. package/dist/cluster/cluster.rpc-service.d.ts +40 -0
  5. package/dist/cluster/cluster.rpc-service.d.ts.map +1 -1
  6. package/dist/cluster/cluster.rpc-service.js +49 -3
  7. package/dist/cluster/cluster.rpc-service.js.map +1 -1
  8. package/dist/cluster/config-sync.d.ts.map +1 -1
  9. package/dist/cluster/config-sync.js.map +1 -1
  10. package/dist/cluster/leader-election.d.ts +90 -2
  11. package/dist/cluster/leader-election.d.ts.map +1 -1
  12. package/dist/cluster/leader-election.js +176 -22
  13. package/dist/cluster/leader-election.js.map +1 -1
  14. package/dist/cluster/mesh-link.d.ts +136 -0
  15. package/dist/cluster/mesh-link.d.ts.map +1 -0
  16. package/dist/cluster/mesh-link.js +212 -0
  17. package/dist/cluster/mesh-link.js.map +1 -0
  18. package/dist/cluster/mesh.d.ts +59 -0
  19. package/dist/cluster/mesh.d.ts.map +1 -0
  20. package/dist/cluster/mesh.js +117 -0
  21. package/dist/cluster/mesh.js.map +1 -0
  22. package/dist/cluster/slave-connector.d.ts +193 -9
  23. package/dist/cluster/slave-connector.d.ts.map +1 -1
  24. package/dist/cluster/slave-connector.js +409 -37
  25. package/dist/cluster/slave-connector.js.map +1 -1
  26. package/dist/commands/audit.d.ts +65 -0
  27. package/dist/commands/audit.d.ts.map +1 -0
  28. package/dist/commands/audit.js +233 -0
  29. package/dist/commands/audit.js.map +1 -0
  30. package/dist/commands/backup.d.ts +4 -0
  31. package/dist/commands/backup.d.ts.map +1 -1
  32. package/dist/commands/backup.js +255 -33
  33. package/dist/commands/backup.js.map +1 -1
  34. package/dist/commands/cluster.d.ts.map +1 -1
  35. package/dist/commands/cluster.js +78 -47
  36. package/dist/commands/cluster.js.map +1 -1
  37. package/dist/commands/daemon-cmd.d.ts +7 -1
  38. package/dist/commands/daemon-cmd.d.ts.map +1 -1
  39. package/dist/commands/daemon-cmd.js +87 -21
  40. package/dist/commands/daemon-cmd.js.map +1 -1
  41. package/dist/commands/daemon-required.d.ts +34 -0
  42. package/dist/commands/daemon-required.d.ts.map +1 -0
  43. package/dist/commands/daemon-required.js +92 -0
  44. package/dist/commands/daemon-required.js.map +1 -0
  45. package/dist/commands/deploy.d.ts +70 -8
  46. package/dist/commands/deploy.d.ts.map +1 -1
  47. package/dist/commands/deploy.js +99 -64
  48. package/dist/commands/deploy.js.map +1 -1
  49. package/dist/commands/discover.d.ts +25 -4
  50. package/dist/commands/discover.d.ts.map +1 -1
  51. package/dist/commands/discover.js +155 -102
  52. package/dist/commands/discover.js.map +1 -1
  53. package/dist/commands/doctor.d.ts +293 -0
  54. package/dist/commands/doctor.d.ts.map +1 -0
  55. package/dist/commands/doctor.js +1706 -0
  56. package/dist/commands/doctor.js.map +1 -0
  57. package/dist/commands/env.d.ts +8 -1
  58. package/dist/commands/env.d.ts.map +1 -1
  59. package/dist/commands/env.js +13 -8
  60. package/dist/commands/env.js.map +1 -1
  61. package/dist/commands/exec.d.ts.map +1 -1
  62. package/dist/commands/exec.js +3 -2
  63. package/dist/commands/exec.js.map +1 -1
  64. package/dist/commands/fleet-asking.d.ts +87 -0
  65. package/dist/commands/fleet-asking.d.ts.map +1 -0
  66. package/dist/commands/fleet-asking.js +176 -0
  67. package/dist/commands/fleet-asking.js.map +1 -0
  68. package/dist/commands/fleet.d.ts +18 -0
  69. package/dist/commands/fleet.d.ts.map +1 -1
  70. package/dist/commands/fleet.js +314 -74
  71. package/dist/commands/fleet.js.map +1 -1
  72. package/dist/commands/health-check.d.ts +8 -6
  73. package/dist/commands/health-check.d.ts.map +1 -1
  74. package/dist/commands/health-check.js +26 -23
  75. package/dist/commands/health-check.js.map +1 -1
  76. package/dist/commands/health.d.ts.map +1 -1
  77. package/dist/commands/health.js +22 -8
  78. package/dist/commands/health.js.map +1 -1
  79. package/dist/commands/infra.d.ts +42 -1
  80. package/dist/commands/infra.d.ts.map +1 -1
  81. package/dist/commands/infra.js +418 -49
  82. package/dist/commands/infra.js.map +1 -1
  83. package/dist/commands/init.d.ts.map +1 -1
  84. package/dist/commands/init.js +21 -36
  85. package/dist/commands/init.js.map +1 -1
  86. package/dist/commands/inspect-graph.d.ts.map +1 -1
  87. package/dist/commands/inspect-graph.js +2 -6
  88. package/dist/commands/inspect-graph.js.map +1 -1
  89. package/dist/commands/inspect.d.ts.map +1 -1
  90. package/dist/commands/inspect.js +110 -14
  91. package/dist/commands/inspect.js.map +1 -1
  92. package/dist/commands/k8s.d.ts.map +1 -1
  93. package/dist/commands/k8s.js +23 -18
  94. package/dist/commands/k8s.js.map +1 -1
  95. package/dist/commands/kb.d.ts.map +1 -1
  96. package/dist/commands/kb.js +88 -29
  97. package/dist/commands/kb.js.map +1 -1
  98. package/dist/commands/list.d.ts.map +1 -1
  99. package/dist/commands/list.js +21 -9
  100. package/dist/commands/list.js.map +1 -1
  101. package/dist/commands/log-health.d.ts +120 -0
  102. package/dist/commands/log-health.d.ts.map +1 -0
  103. package/dist/commands/log-health.js +212 -0
  104. package/dist/commands/log-health.js.map +1 -0
  105. package/dist/commands/logs.d.ts +96 -2
  106. package/dist/commands/logs.d.ts.map +1 -1
  107. package/dist/commands/logs.js +522 -196
  108. package/dist/commands/logs.js.map +1 -1
  109. package/dist/commands/metrics.d.ts.map +1 -1
  110. package/dist/commands/metrics.js +35 -17
  111. package/dist/commands/metrics.js.map +1 -1
  112. package/dist/commands/monit.d.ts.map +1 -1
  113. package/dist/commands/monit.js +3 -2
  114. package/dist/commands/monit.js.map +1 -1
  115. package/dist/commands/node.d.ts +68 -0
  116. package/dist/commands/node.d.ts.map +1 -1
  117. package/dist/commands/node.js +285 -68
  118. package/dist/commands/node.js.map +1 -1
  119. package/dist/commands/output.d.ts +10 -0
  120. package/dist/commands/output.d.ts.map +1 -1
  121. package/dist/commands/output.js +41 -1
  122. package/dist/commands/output.js.map +1 -1
  123. package/dist/commands/pipeline.d.ts.map +1 -1
  124. package/dist/commands/pipeline.js +26 -19
  125. package/dist/commands/pipeline.js.map +1 -1
  126. package/dist/commands/project.d.ts +19 -0
  127. package/dist/commands/project.d.ts.map +1 -1
  128. package/dist/commands/project.js +125 -27
  129. package/dist/commands/project.js.map +1 -1
  130. package/dist/commands/release-store.d.ts +28 -0
  131. package/dist/commands/release-store.d.ts.map +1 -0
  132. package/dist/commands/release-store.js +158 -0
  133. package/dist/commands/release-store.js.map +1 -0
  134. package/dist/commands/release.d.ts +61 -0
  135. package/dist/commands/release.d.ts.map +1 -0
  136. package/dist/commands/release.js +555 -0
  137. package/dist/commands/release.js.map +1 -0
  138. package/dist/commands/reload.d.ts +16 -0
  139. package/dist/commands/reload.d.ts.map +1 -1
  140. package/dist/commands/reload.js +21 -5
  141. package/dist/commands/reload.js.map +1 -1
  142. package/dist/commands/remote.d.ts +40 -0
  143. package/dist/commands/remote.d.ts.map +1 -1
  144. package/dist/commands/remote.js +146 -35
  145. package/dist/commands/remote.js.map +1 -1
  146. package/dist/commands/restart.d.ts +16 -0
  147. package/dist/commands/restart.d.ts.map +1 -1
  148. package/dist/commands/restart.js +21 -5
  149. package/dist/commands/restart.js.map +1 -1
  150. package/dist/commands/scale.d.ts.map +1 -1
  151. package/dist/commands/scale.js +4 -3
  152. package/dist/commands/scale.js.map +1 -1
  153. package/dist/commands/secret.d.ts +54 -0
  154. package/dist/commands/secret.d.ts.map +1 -1
  155. package/dist/commands/secret.js +236 -34
  156. package/dist/commands/secret.js.map +1 -1
  157. package/dist/commands/service-units.d.ts +181 -0
  158. package/dist/commands/service-units.d.ts.map +1 -0
  159. package/dist/commands/service-units.js +291 -0
  160. package/dist/commands/service-units.js.map +1 -0
  161. package/dist/commands/service.d.ts +39 -0
  162. package/dist/commands/service.d.ts.map +1 -0
  163. package/dist/commands/service.js +370 -0
  164. package/dist/commands/service.js.map +1 -0
  165. package/dist/commands/spinner.d.ts +29 -0
  166. package/dist/commands/spinner.d.ts.map +1 -0
  167. package/dist/commands/spinner.js +44 -0
  168. package/dist/commands/spinner.js.map +1 -0
  169. package/dist/commands/stack.d.ts +49 -1
  170. package/dist/commands/stack.d.ts.map +1 -1
  171. package/dist/commands/stack.js +325 -12
  172. package/dist/commands/stack.js.map +1 -1
  173. package/dist/commands/start.d.ts +16 -0
  174. package/dist/commands/start.d.ts.map +1 -1
  175. package/dist/commands/start.js +20 -3
  176. package/dist/commands/start.js.map +1 -1
  177. package/dist/commands/status.d.ts +9 -0
  178. package/dist/commands/status.d.ts.map +1 -1
  179. package/dist/commands/status.js +45 -29
  180. package/dist/commands/status.js.map +1 -1
  181. package/dist/commands/stop.d.ts +16 -0
  182. package/dist/commands/stop.d.ts.map +1 -1
  183. package/dist/commands/stop.js +34 -6
  184. package/dist/commands/stop.js.map +1 -1
  185. package/dist/commands/tor.d.ts +14 -7
  186. package/dist/commands/tor.d.ts.map +1 -1
  187. package/dist/commands/tor.js +113 -33
  188. package/dist/commands/tor.js.map +1 -1
  189. package/dist/commands/up.d.ts +85 -1
  190. package/dist/commands/up.d.ts.map +1 -1
  191. package/dist/commands/up.js +180 -32
  192. package/dist/commands/up.js.map +1 -1
  193. package/dist/commands/webapp.d.ts.map +1 -1
  194. package/dist/commands/webapp.js +2 -1
  195. package/dist/commands/webapp.js.map +1 -1
  196. package/dist/config/defaults.d.ts +16 -0
  197. package/dist/config/defaults.d.ts.map +1 -1
  198. package/dist/config/defaults.js +26 -3
  199. package/dist/config/defaults.js.map +1 -1
  200. package/dist/config/define-ecosystem.d.ts.map +1 -1
  201. package/dist/config/define-ecosystem.js +2 -0
  202. package/dist/config/define-ecosystem.js.map +1 -1
  203. package/dist/config/index.d.ts +1 -1
  204. package/dist/config/index.d.ts.map +1 -1
  205. package/dist/config/index.js +1 -1
  206. package/dist/config/index.js.map +1 -1
  207. package/dist/config/loader.d.ts +45 -1
  208. package/dist/config/loader.d.ts.map +1 -1
  209. package/dist/config/loader.js +64 -6
  210. package/dist/config/loader.js.map +1 -1
  211. package/dist/config/omnitron-config.schema.d.ts.map +1 -1
  212. package/dist/config/omnitron-config.schema.js +16 -0
  213. package/dist/config/omnitron-config.schema.js.map +1 -1
  214. package/dist/config/types.d.ts +411 -42
  215. package/dist/config/types.d.ts.map +1 -1
  216. package/dist/daemon/config-reload.d.ts +27 -0
  217. package/dist/daemon/config-reload.d.ts.map +1 -0
  218. package/dist/daemon/config-reload.js +42 -0
  219. package/dist/daemon/config-reload.js.map +1 -0
  220. package/dist/daemon/daemon-client.d.ts +136 -2
  221. package/dist/daemon/daemon-client.d.ts.map +1 -1
  222. package/dist/daemon/daemon-client.js +307 -21
  223. package/dist/daemon/daemon-client.js.map +1 -1
  224. package/dist/daemon/daemon-entry.js +90 -30
  225. package/dist/daemon/daemon-entry.js.map +1 -1
  226. package/dist/daemon/daemon-scheduler.d.ts +4 -0
  227. package/dist/daemon/daemon-scheduler.d.ts.map +1 -1
  228. package/dist/daemon/daemon-scheduler.js +137 -7
  229. package/dist/daemon/daemon-scheduler.js.map +1 -1
  230. package/dist/daemon/daemon-state-store.service.d.ts +263 -0
  231. package/dist/daemon/daemon-state-store.service.d.ts.map +1 -0
  232. package/dist/daemon/daemon-state-store.service.js +446 -0
  233. package/dist/daemon/daemon-state-store.service.js.map +1 -0
  234. package/dist/daemon/daemon.d.ts +205 -1
  235. package/dist/daemon/daemon.d.ts.map +1 -1
  236. package/dist/daemon/daemon.js +1139 -139
  237. package/dist/daemon/daemon.js.map +1 -1
  238. package/dist/daemon/daemon.module.d.ts.map +1 -1
  239. package/dist/daemon/daemon.module.js +163 -83
  240. package/dist/daemon/daemon.module.js.map +1 -1
  241. package/dist/daemon/daemon.rpc-service.d.ts +90 -1
  242. package/dist/daemon/daemon.rpc-service.d.ts.map +1 -1
  243. package/dist/daemon/daemon.rpc-service.js +438 -96
  244. package/dist/daemon/daemon.rpc-service.js.map +1 -1
  245. package/dist/daemon/pid-manager.d.ts +16 -3
  246. package/dist/daemon/pid-manager.d.ts.map +1 -1
  247. package/dist/daemon/pid-manager.js +31 -4
  248. package/dist/daemon/pid-manager.js.map +1 -1
  249. package/dist/daemon/spawn-daemon.d.ts +73 -0
  250. package/dist/daemon/spawn-daemon.d.ts.map +1 -0
  251. package/dist/daemon/spawn-daemon.js +101 -0
  252. package/dist/daemon/spawn-daemon.js.map +1 -0
  253. package/dist/daemon/state-store.d.ts +65 -3
  254. package/dist/daemon/state-store.d.ts.map +1 -1
  255. package/dist/daemon/state-store.js +100 -58
  256. package/dist/daemon/state-store.js.map +1 -1
  257. package/dist/database/connection.d.ts +71 -0
  258. package/dist/database/connection.d.ts.map +1 -0
  259. package/dist/database/connection.js +129 -0
  260. package/dist/database/connection.js.map +1 -0
  261. package/dist/database/dialect.d.ts +60 -0
  262. package/dist/database/dialect.d.ts.map +1 -0
  263. package/dist/database/dialect.js +80 -0
  264. package/dist/database/dialect.js.map +1 -0
  265. package/dist/database/migrate.d.ts +5 -2
  266. package/dist/database/migrate.d.ts.map +1 -1
  267. package/dist/database/migrate.js +23 -33
  268. package/dist/database/migrate.js.map +1 -1
  269. package/dist/database/migration-runner.d.ts +50 -0
  270. package/dist/database/migration-runner.d.ts.map +1 -0
  271. package/dist/database/migration-runner.js +143 -0
  272. package/dist/database/migration-runner.js.map +1 -0
  273. package/dist/database/migrations/006_login_throttle.d.ts +17 -0
  274. package/dist/database/migrations/006_login_throttle.d.ts.map +1 -0
  275. package/dist/database/migrations/006_login_throttle.js +33 -0
  276. package/dist/database/migrations/006_login_throttle.js.map +1 -0
  277. package/dist/database/migrations/007_sync_ingested.d.ts +21 -0
  278. package/dist/database/migrations/007_sync_ingested.d.ts.map +1 -0
  279. package/dist/database/migrations/007_sync_ingested.js +39 -0
  280. package/dist/database/migrations/007_sync_ingested.js.map +1 -0
  281. package/dist/database/migrations/008_node_health_version_text.d.ts +24 -0
  282. package/dist/database/migrations/008_node_health_version_text.d.ts.map +1 -0
  283. package/dist/database/migrations/008_node_health_version_text.js +35 -0
  284. package/dist/database/migrations/008_node_health_version_text.js.map +1 -0
  285. package/dist/database/migrations/009_node_health_unmeasured.d.ts +40 -0
  286. package/dist/database/migrations/009_node_health_unmeasured.d.ts.map +1 -0
  287. package/dist/database/migrations/009_node_health_unmeasured.js +63 -0
  288. package/dist/database/migrations/009_node_health_unmeasured.js.map +1 -0
  289. package/dist/database/migrations/010_metrics_raw_had_no_reader.d.ts +22 -0
  290. package/dist/database/migrations/010_metrics_raw_had_no_reader.d.ts.map +1 -0
  291. package/dist/database/migrations/010_metrics_raw_had_no_reader.js +26 -0
  292. package/dist/database/migrations/010_metrics_raw_had_no_reader.js.map +1 -0
  293. package/dist/database/migrations/011_alerts_nobody_had_to_write.d.ts +56 -0
  294. package/dist/database/migrations/011_alerts_nobody_had_to_write.d.ts.map +1 -0
  295. package/dist/database/migrations/011_alerts_nobody_had_to_write.js +79 -0
  296. package/dist/database/migrations/011_alerts_nobody_had_to_write.js.map +1 -0
  297. package/dist/database/migrations/012_a_disk_nobody_watched.d.ts +26 -0
  298. package/dist/database/migrations/012_a_disk_nobody_watched.d.ts.map +1 -0
  299. package/dist/database/migrations/012_a_disk_nobody_watched.js +40 -0
  300. package/dist/database/migrations/012_a_disk_nobody_watched.js.map +1 -0
  301. package/dist/database/migrations/013_an_alarm_delivered.d.ts +14 -0
  302. package/dist/database/migrations/013_an_alarm_delivered.d.ts.map +1 -0
  303. package/dist/database/migrations/013_an_alarm_delivered.js +28 -0
  304. package/dist/database/migrations/013_an_alarm_delivered.js.map +1 -0
  305. package/dist/database/migrations/index.d.ts +21 -0
  306. package/dist/database/migrations/index.d.ts.map +1 -0
  307. package/dist/database/migrations/index.js +47 -0
  308. package/dist/database/migrations/index.js.map +1 -0
  309. package/dist/database/schema.d.ts +27 -11
  310. package/dist/database/schema.d.ts.map +1 -1
  311. package/dist/database/sqlite-date-binding.d.ts +71 -0
  312. package/dist/database/sqlite-date-binding.d.ts.map +1 -0
  313. package/dist/database/sqlite-date-binding.js +174 -0
  314. package/dist/database/sqlite-date-binding.js.map +1 -0
  315. package/dist/execution/data-channel.d.ts +48 -0
  316. package/dist/execution/data-channel.d.ts.map +1 -0
  317. package/dist/execution/data-channel.js +55 -0
  318. package/dist/execution/data-channel.js.map +1 -0
  319. package/dist/execution/execution.service.d.ts +139 -0
  320. package/dist/execution/execution.service.d.ts.map +1 -1
  321. package/dist/execution/execution.service.js +273 -10
  322. package/dist/execution/execution.service.js.map +1 -1
  323. package/dist/index.d.ts +1 -1
  324. package/dist/index.d.ts.map +1 -1
  325. package/dist/infrastructure/bare-metal-plan.d.ts +208 -0
  326. package/dist/infrastructure/bare-metal-plan.d.ts.map +1 -0
  327. package/dist/infrastructure/bare-metal-plan.js +363 -0
  328. package/dist/infrastructure/bare-metal-plan.js.map +1 -0
  329. package/dist/infrastructure/bare-metal-runner.d.ts +65 -0
  330. package/dist/infrastructure/bare-metal-runner.d.ts.map +1 -0
  331. package/dist/infrastructure/bare-metal-runner.js +354 -0
  332. package/dist/infrastructure/bare-metal-runner.js.map +1 -0
  333. package/dist/infrastructure/config-payload.d.ts +149 -0
  334. package/dist/infrastructure/config-payload.d.ts.map +1 -0
  335. package/dist/infrastructure/config-payload.js +235 -0
  336. package/dist/infrastructure/config-payload.js.map +1 -0
  337. package/dist/infrastructure/container-runtime.d.ts +169 -1
  338. package/dist/infrastructure/container-runtime.d.ts.map +1 -1
  339. package/dist/infrastructure/container-runtime.js +525 -33
  340. package/dist/infrastructure/container-runtime.js.map +1 -1
  341. package/dist/infrastructure/gateway-reachability.d.ts +85 -0
  342. package/dist/infrastructure/gateway-reachability.d.ts.map +1 -0
  343. package/dist/infrastructure/gateway-reachability.js +116 -0
  344. package/dist/infrastructure/gateway-reachability.js.map +1 -0
  345. package/dist/infrastructure/gateway-redis-db.d.ts +29 -0
  346. package/dist/infrastructure/gateway-redis-db.d.ts.map +1 -0
  347. package/dist/infrastructure/gateway-redis-db.js +32 -0
  348. package/dist/infrastructure/gateway-redis-db.js.map +1 -0
  349. package/dist/infrastructure/host-inspection.d.ts +223 -0
  350. package/dist/infrastructure/host-inspection.d.ts.map +1 -0
  351. package/dist/infrastructure/host-inspection.js +501 -0
  352. package/dist/infrastructure/host-inspection.js.map +1 -0
  353. package/dist/infrastructure/infrastructure-gate.d.ts.map +1 -1
  354. package/dist/infrastructure/infrastructure.service.d.ts +206 -6
  355. package/dist/infrastructure/infrastructure.service.d.ts.map +1 -1
  356. package/dist/infrastructure/infrastructure.service.js +415 -89
  357. package/dist/infrastructure/infrastructure.service.js.map +1 -1
  358. package/dist/infrastructure/known-machines.d.ts +76 -0
  359. package/dist/infrastructure/known-machines.d.ts.map +1 -0
  360. package/dist/infrastructure/known-machines.js +94 -0
  361. package/dist/infrastructure/known-machines.js.map +1 -0
  362. package/dist/infrastructure/node-credentials.d.ts +45 -0
  363. package/dist/infrastructure/node-credentials.d.ts.map +1 -0
  364. package/dist/infrastructure/node-credentials.js +80 -0
  365. package/dist/infrastructure/node-credentials.js.map +1 -0
  366. package/dist/infrastructure/orphan-containers.d.ts +99 -0
  367. package/dist/infrastructure/orphan-containers.d.ts.map +1 -0
  368. package/dist/infrastructure/orphan-containers.js +106 -0
  369. package/dist/infrastructure/orphan-containers.js.map +1 -0
  370. package/dist/infrastructure/phantom-endpoint-janitor.d.ts +20 -0
  371. package/dist/infrastructure/phantom-endpoint-janitor.d.ts.map +1 -1
  372. package/dist/infrastructure/phantom-endpoint-janitor.js +56 -6
  373. package/dist/infrastructure/phantom-endpoint-janitor.js.map +1 -1
  374. package/dist/infrastructure/presets/gateway-openresty.d.ts.map +1 -1
  375. package/dist/infrastructure/presets/gateway-openresty.js +29 -3
  376. package/dist/infrastructure/presets/gateway-openresty.js.map +1 -1
  377. package/dist/infrastructure/presets/minio.d.ts.map +1 -1
  378. package/dist/infrastructure/presets/minio.js +36 -3
  379. package/dist/infrastructure/presets/minio.js.map +1 -1
  380. package/dist/infrastructure/presets/postgres.d.ts.map +1 -1
  381. package/dist/infrastructure/presets/postgres.js +69 -2
  382. package/dist/infrastructure/presets/postgres.js.map +1 -1
  383. package/dist/infrastructure/presets/redis.d.ts.map +1 -1
  384. package/dist/infrastructure/presets/redis.js +13 -0
  385. package/dist/infrastructure/presets/redis.js.map +1 -1
  386. package/dist/infrastructure/presets/registry.d.ts.map +1 -1
  387. package/dist/infrastructure/presets/registry.js +81 -12
  388. package/dist/infrastructure/presets/registry.js.map +1 -1
  389. package/dist/infrastructure/presets/tor.d.ts +38 -0
  390. package/dist/infrastructure/presets/tor.d.ts.map +1 -1
  391. package/dist/infrastructure/presets/tor.js +78 -69
  392. package/dist/infrastructure/presets/tor.js.map +1 -1
  393. package/dist/infrastructure/presets/torrc.d.ts +54 -0
  394. package/dist/infrastructure/presets/torrc.d.ts.map +1 -0
  395. package/dist/infrastructure/presets/torrc.js +153 -0
  396. package/dist/infrastructure/presets/torrc.js.map +1 -0
  397. package/dist/infrastructure/presets/types.d.ts +17 -0
  398. package/dist/infrastructure/presets/types.d.ts.map +1 -1
  399. package/dist/infrastructure/provisioning-outcome.d.ts +85 -0
  400. package/dist/infrastructure/provisioning-outcome.d.ts.map +1 -0
  401. package/dist/infrastructure/provisioning-outcome.js +122 -0
  402. package/dist/infrastructure/provisioning-outcome.js.map +1 -0
  403. package/dist/infrastructure/server-registry.d.ts +27 -0
  404. package/dist/infrastructure/server-registry.d.ts.map +1 -1
  405. package/dist/infrastructure/server-registry.js +60 -5
  406. package/dist/infrastructure/server-registry.js.map +1 -1
  407. package/dist/infrastructure/service-binding.d.ts +60 -0
  408. package/dist/infrastructure/service-binding.d.ts.map +1 -0
  409. package/dist/infrastructure/service-binding.js +90 -0
  410. package/dist/infrastructure/service-binding.js.map +1 -0
  411. package/dist/infrastructure/service-credentials.d.ts +105 -0
  412. package/dist/infrastructure/service-credentials.d.ts.map +1 -0
  413. package/dist/infrastructure/service-credentials.js +141 -0
  414. package/dist/infrastructure/service-credentials.js.map +1 -0
  415. package/dist/infrastructure/service-resolver.d.ts +160 -12
  416. package/dist/infrastructure/service-resolver.d.ts.map +1 -1
  417. package/dist/infrastructure/service-resolver.js +463 -53
  418. package/dist/infrastructure/service-resolver.js.map +1 -1
  419. package/dist/infrastructure/shipped-config.d.ts +43 -0
  420. package/dist/infrastructure/shipped-config.d.ts.map +1 -0
  421. package/dist/infrastructure/shipped-config.js +159 -0
  422. package/dist/infrastructure/shipped-config.js.map +1 -0
  423. package/dist/infrastructure/stack-infra-manager.d.ts +6 -0
  424. package/dist/infrastructure/stack-infra-manager.d.ts.map +1 -1
  425. package/dist/infrastructure/stack-infra-manager.js +31 -13
  426. package/dist/infrastructure/stack-infra-manager.js.map +1 -1
  427. package/dist/infrastructure/types.d.ts +261 -6
  428. package/dist/infrastructure/types.d.ts.map +1 -1
  429. package/dist/mcp/mcp-bridge.d.ts.map +1 -1
  430. package/dist/mcp/mcp-bridge.js.map +1 -1
  431. package/dist/mcp/tool-groups/apps.tools.d.ts +2 -1
  432. package/dist/mcp/tool-groups/apps.tools.d.ts.map +1 -1
  433. package/dist/mcp/tool-groups/apps.tools.js +2 -2
  434. package/dist/mcp/tool-groups/apps.tools.js.map +1 -1
  435. package/dist/mcp/tool-groups/infra.tools.d.ts +16 -2
  436. package/dist/mcp/tool-groups/infra.tools.d.ts.map +1 -1
  437. package/dist/mcp/tool-groups/infra.tools.js +51 -41
  438. package/dist/mcp/tool-groups/infra.tools.js.map +1 -1
  439. package/dist/mcp/tool-groups/kb.tools.d.ts.map +1 -1
  440. package/dist/mcp/tool-groups/kb.tools.js +4 -6
  441. package/dist/mcp/tool-groups/kb.tools.js.map +1 -1
  442. package/dist/mcp/tool-groups/management.tools.d.ts +20 -3
  443. package/dist/mcp/tool-groups/management.tools.d.ts.map +1 -1
  444. package/dist/mcp/tool-groups/management.tools.js +158 -125
  445. package/dist/mcp/tool-groups/management.tools.js.map +1 -1
  446. package/dist/mcp/tool-groups/monitoring.tools.d.ts +10 -1
  447. package/dist/mcp/tool-groups/monitoring.tools.d.ts.map +1 -1
  448. package/dist/mcp/tool-groups/monitoring.tools.js +65 -18
  449. package/dist/mcp/tool-groups/monitoring.tools.js.map +1 -1
  450. package/dist/mcp/unavailable-tools.d.ts +35 -0
  451. package/dist/mcp/unavailable-tools.d.ts.map +1 -0
  452. package/dist/mcp/unavailable-tools.js +59 -0
  453. package/dist/mcp/unavailable-tools.js.map +1 -0
  454. package/dist/monitoring/app-health.indicator.d.ts.map +1 -1
  455. package/dist/monitoring/app-health.indicator.js +12 -0
  456. package/dist/monitoring/app-health.indicator.js.map +1 -1
  457. package/dist/monitoring/desktop-notifier.d.ts +26 -0
  458. package/dist/monitoring/desktop-notifier.d.ts.map +1 -0
  459. package/dist/monitoring/desktop-notifier.js +49 -0
  460. package/dist/monitoring/desktop-notifier.js.map +1 -0
  461. package/dist/monitoring/docker-health.indicator.d.ts +17 -1
  462. package/dist/monitoring/docker-health.indicator.d.ts.map +1 -1
  463. package/dist/monitoring/docker-health.indicator.js +38 -8
  464. package/dist/monitoring/docker-health.indicator.js.map +1 -1
  465. package/dist/monitoring/event-loop-stall.indicator.d.ts +26 -0
  466. package/dist/monitoring/event-loop-stall.indicator.d.ts.map +1 -0
  467. package/dist/monitoring/event-loop-stall.indicator.js +47 -0
  468. package/dist/monitoring/event-loop-stall.indicator.js.map +1 -0
  469. package/dist/monitoring/event-loop-watch.d.ts +124 -0
  470. package/dist/monitoring/event-loop-watch.d.ts.map +1 -0
  471. package/dist/monitoring/event-loop-watch.js +179 -0
  472. package/dist/monitoring/event-loop-watch.js.map +1 -0
  473. package/dist/monitoring/host-disk.d.ts +6 -0
  474. package/dist/monitoring/host-disk.d.ts.map +1 -0
  475. package/dist/monitoring/host-disk.js +26 -0
  476. package/dist/monitoring/host-disk.js.map +1 -0
  477. package/dist/monitoring/log-line.d.ts +188 -0
  478. package/dist/monitoring/log-line.d.ts.map +1 -0
  479. package/dist/monitoring/log-line.js +458 -0
  480. package/dist/monitoring/log-line.js.map +1 -0
  481. package/dist/monitoring/log-manager.d.ts +89 -8
  482. package/dist/monitoring/log-manager.d.ts.map +1 -1
  483. package/dist/monitoring/log-manager.js +246 -77
  484. package/dist/monitoring/log-manager.js.map +1 -1
  485. package/dist/observability/metrics-bridge.d.ts.map +1 -1
  486. package/dist/observability/metrics-server.d.ts.map +1 -1
  487. package/dist/orchestrator/app-handle.d.ts +157 -2
  488. package/dist/orchestrator/app-handle.d.ts.map +1 -1
  489. package/dist/orchestrator/app-handle.js +165 -3
  490. package/dist/orchestrator/app-handle.js.map +1 -1
  491. package/dist/orchestrator/app-health.d.ts +33 -0
  492. package/dist/orchestrator/app-health.d.ts.map +1 -0
  493. package/dist/orchestrator/app-health.js +74 -0
  494. package/dist/orchestrator/app-health.js.map +1 -0
  495. package/dist/orchestrator/app-traffic.d.ts +14 -0
  496. package/dist/orchestrator/app-traffic.d.ts.map +1 -0
  497. package/dist/orchestrator/app-traffic.js +33 -0
  498. package/dist/orchestrator/app-traffic.js.map +1 -0
  499. package/dist/orchestrator/bootstrap-loader-child.d.ts +19 -0
  500. package/dist/orchestrator/bootstrap-loader-child.d.ts.map +1 -0
  501. package/dist/orchestrator/bootstrap-loader-child.js +43 -0
  502. package/dist/orchestrator/bootstrap-loader-child.js.map +1 -0
  503. package/dist/orchestrator/bootstrap-loader.d.ts +72 -0
  504. package/dist/orchestrator/bootstrap-loader.d.ts.map +1 -1
  505. package/dist/orchestrator/bootstrap-loader.js +350 -5
  506. package/dist/orchestrator/bootstrap-loader.js.map +1 -1
  507. package/dist/orchestrator/bootstrap-process.d.ts +99 -4
  508. package/dist/orchestrator/bootstrap-process.d.ts.map +1 -1
  509. package/dist/orchestrator/bootstrap-process.js +330 -19
  510. package/dist/orchestrator/bootstrap-process.js.map +1 -1
  511. package/dist/orchestrator/build-service.d.ts.map +1 -1
  512. package/dist/orchestrator/build-service.js.map +1 -1
  513. package/dist/orchestrator/call-exposed-service.d.ts +7 -0
  514. package/dist/orchestrator/call-exposed-service.d.ts.map +1 -0
  515. package/dist/orchestrator/call-exposed-service.js +50 -0
  516. package/dist/orchestrator/call-exposed-service.js.map +1 -0
  517. package/dist/orchestrator/classic-launcher.d.ts +1 -0
  518. package/dist/orchestrator/classic-launcher.d.ts.map +1 -1
  519. package/dist/orchestrator/classic-launcher.js +34 -3
  520. package/dist/orchestrator/classic-launcher.js.map +1 -1
  521. package/dist/orchestrator/file-watcher.d.ts +78 -1
  522. package/dist/orchestrator/file-watcher.d.ts.map +1 -1
  523. package/dist/orchestrator/file-watcher.js +181 -6
  524. package/dist/orchestrator/file-watcher.js.map +1 -1
  525. package/dist/orchestrator/module-worker-process.d.ts.map +1 -1
  526. package/dist/orchestrator/module-worker-process.js +10 -0
  527. package/dist/orchestrator/module-worker-process.js.map +1 -1
  528. package/dist/orchestrator/orchestrator.service.d.ts +371 -11
  529. package/dist/orchestrator/orchestrator.service.d.ts.map +1 -1
  530. package/dist/orchestrator/orchestrator.service.js +1602 -269
  531. package/dist/orchestrator/orchestrator.service.js.map +1 -1
  532. package/dist/orchestrator/owned-pids.d.ts +83 -0
  533. package/dist/orchestrator/owned-pids.d.ts.map +1 -0
  534. package/dist/orchestrator/owned-pids.js +93 -0
  535. package/dist/orchestrator/owned-pids.js.map +1 -0
  536. package/dist/orchestrator/port-conflict.d.ts +102 -0
  537. package/dist/orchestrator/port-conflict.d.ts.map +1 -0
  538. package/dist/orchestrator/port-conflict.js +131 -0
  539. package/dist/orchestrator/port-conflict.js.map +1 -0
  540. package/dist/orchestrator/process-janitor.d.ts +78 -4
  541. package/dist/orchestrator/process-janitor.d.ts.map +1 -1
  542. package/dist/orchestrator/process-janitor.js +246 -30
  543. package/dist/orchestrator/process-janitor.js.map +1 -1
  544. package/dist/orchestrator/ps-metrics.d.ts +37 -0
  545. package/dist/orchestrator/ps-metrics.d.ts.map +1 -0
  546. package/dist/orchestrator/ps-metrics.js +60 -0
  547. package/dist/orchestrator/ps-metrics.js.map +1 -0
  548. package/dist/orchestrator/service-router.d.ts +58 -2
  549. package/dist/orchestrator/service-router.d.ts.map +1 -1
  550. package/dist/orchestrator/service-router.js +157 -25
  551. package/dist/orchestrator/service-router.js.map +1 -1
  552. package/dist/orchestrator/topology-query.d.ts +49 -0
  553. package/dist/orchestrator/topology-query.d.ts.map +1 -0
  554. package/dist/orchestrator/topology-query.js +54 -0
  555. package/dist/orchestrator/topology-query.js.map +1 -0
  556. package/dist/project/artifact-builder.d.ts +174 -3
  557. package/dist/project/artifact-builder.d.ts.map +1 -1
  558. package/dist/project/artifact-builder.js +454 -42
  559. package/dist/project/artifact-builder.js.map +1 -1
  560. package/dist/project/build-decision.d.ts +58 -0
  561. package/dist/project/build-decision.d.ts.map +1 -0
  562. package/dist/project/build-decision.js +59 -0
  563. package/dist/project/build-decision.js.map +1 -0
  564. package/dist/project/config-resolver.d.ts +6 -4
  565. package/dist/project/config-resolver.d.ts.map +1 -1
  566. package/dist/project/config-resolver.js +128 -74
  567. package/dist/project/config-resolver.js.map +1 -1
  568. package/dist/project/declared-config.d.ts +42 -0
  569. package/dist/project/declared-config.d.ts.map +1 -0
  570. package/dist/project/declared-config.js +60 -0
  571. package/dist/project/declared-config.js.map +1 -0
  572. package/dist/project/dependency-analyzer.d.ts.map +1 -1
  573. package/dist/project/dependency-analyzer.js.map +1 -1
  574. package/dist/project/deploy-phases.d.ts +35 -0
  575. package/dist/project/deploy-phases.d.ts.map +1 -0
  576. package/dist/project/deploy-phases.js +75 -0
  577. package/dist/project/deploy-phases.js.map +1 -0
  578. package/dist/project/exec-in-group.d.ts +49 -0
  579. package/dist/project/exec-in-group.d.ts.map +1 -0
  580. package/dist/project/exec-in-group.js +100 -0
  581. package/dist/project/exec-in-group.js.map +1 -0
  582. package/dist/project/node-app-config.d.ts +101 -0
  583. package/dist/project/node-app-config.d.ts.map +1 -0
  584. package/dist/project/node-app-config.js +134 -0
  585. package/dist/project/node-app-config.js.map +1 -0
  586. package/dist/project/node-app-health.d.ts +76 -0
  587. package/dist/project/node-app-health.d.ts.map +1 -0
  588. package/dist/project/node-app-health.js +119 -0
  589. package/dist/project/node-app-health.js.map +1 -0
  590. package/dist/project/operator-account.d.ts +287 -0
  591. package/dist/project/operator-account.d.ts.map +1 -0
  592. package/dist/project/operator-account.js +373 -0
  593. package/dist/project/operator-account.js.map +1 -0
  594. package/dist/project/registry.d.ts +45 -8
  595. package/dist/project/registry.d.ts.map +1 -1
  596. package/dist/project/registry.js +217 -49
  597. package/dist/project/registry.js.map +1 -1
  598. package/dist/project/requirements-scanner.d.ts +1 -0
  599. package/dist/project/requirements-scanner.d.ts.map +1 -1
  600. package/dist/project/requirements-scanner.js +28 -3
  601. package/dist/project/requirements-scanner.js.map +1 -1
  602. package/dist/project/stack-env.d.ts +37 -0
  603. package/dist/project/stack-env.d.ts.map +1 -0
  604. package/dist/project/stack-env.js +85 -0
  605. package/dist/project/stack-env.js.map +1 -0
  606. package/dist/project/stand-containers.d.ts +4 -0
  607. package/dist/project/stand-containers.d.ts.map +1 -0
  608. package/dist/project/stand-containers.js +27 -0
  609. package/dist/project/stand-containers.js.map +1 -0
  610. package/dist/project/working-tree.d.ts +62 -0
  611. package/dist/project/working-tree.d.ts.map +1 -0
  612. package/dist/project/working-tree.js +101 -0
  613. package/dist/project/working-tree.js.map +1 -0
  614. package/dist/release/artifact-store.d.ts +360 -0
  615. package/dist/release/artifact-store.d.ts.map +1 -0
  616. package/dist/release/artifact-store.js +1040 -0
  617. package/dist/release/artifact-store.js.map +1 -0
  618. package/dist/release/attest-on-node.d.ts +92 -0
  619. package/dist/release/attest-on-node.d.ts.map +1 -0
  620. package/dist/release/attest-on-node.js +207 -0
  621. package/dist/release/attest-on-node.js.map +1 -0
  622. package/dist/release/attest.d.ts +135 -0
  623. package/dist/release/attest.d.ts.map +1 -0
  624. package/dist/release/attest.js +199 -0
  625. package/dist/release/attest.js.map +1 -0
  626. package/dist/release/build-run.d.ts +151 -0
  627. package/dist/release/build-run.d.ts.map +1 -0
  628. package/dist/release/build-run.js +609 -0
  629. package/dist/release/build-run.js.map +1 -0
  630. package/dist/release/builder.d.ts +57 -0
  631. package/dist/release/builder.d.ts.map +1 -0
  632. package/dist/release/builder.js +120 -0
  633. package/dist/release/builder.js.map +1 -0
  634. package/dist/release/daemon-omni.d.ts +41 -0
  635. package/dist/release/daemon-omni.d.ts.map +1 -0
  636. package/dist/release/daemon-omni.js +59 -0
  637. package/dist/release/daemon-omni.js.map +1 -0
  638. package/dist/release/definition-inputs.d.ts +34 -0
  639. package/dist/release/definition-inputs.d.ts.map +1 -0
  640. package/dist/release/definition-inputs.js +90 -0
  641. package/dist/release/definition-inputs.js.map +1 -0
  642. package/dist/release/delivered.d.ts +53 -0
  643. package/dist/release/delivered.d.ts.map +1 -0
  644. package/dist/release/delivered.js +81 -0
  645. package/dist/release/delivered.js.map +1 -0
  646. package/dist/release/layout.d.ts +61 -0
  647. package/dist/release/layout.d.ts.map +1 -0
  648. package/dist/release/layout.js +114 -0
  649. package/dist/release/layout.js.map +1 -0
  650. package/dist/release/load.d.ts +44 -0
  651. package/dist/release/load.d.ts.map +1 -0
  652. package/dist/release/load.js +84 -0
  653. package/dist/release/load.js.map +1 -0
  654. package/dist/release/manifest.d.ts +299 -0
  655. package/dist/release/manifest.d.ts.map +1 -0
  656. package/dist/release/manifest.js +176 -0
  657. package/dist/release/manifest.js.map +1 -0
  658. package/dist/release/migrations.d.ts +32 -0
  659. package/dist/release/migrations.d.ts.map +1 -0
  660. package/dist/release/migrations.js +51 -0
  661. package/dist/release/migrations.js.map +1 -0
  662. package/dist/release/publish.d.ts +71 -0
  663. package/dist/release/publish.d.ts.map +1 -0
  664. package/dist/release/publish.js +102 -0
  665. package/dist/release/publish.js.map +1 -0
  666. package/dist/release/store.d.ts +174 -0
  667. package/dist/release/store.d.ts.map +1 -0
  668. package/dist/release/store.js +257 -0
  669. package/dist/release/store.js.map +1 -0
  670. package/dist/services/alert.rpc-service.d.ts +11 -15
  671. package/dist/services/alert.rpc-service.d.ts.map +1 -1
  672. package/dist/services/alert.rpc-service.js +29 -14
  673. package/dist/services/alert.rpc-service.js.map +1 -1
  674. package/dist/services/alert.service.d.ts +96 -59
  675. package/dist/services/alert.service.d.ts.map +1 -1
  676. package/dist/services/alert.service.js +455 -70
  677. package/dist/services/alert.service.js.map +1 -1
  678. package/dist/services/anonymous-input.d.ts +26 -0
  679. package/dist/services/anonymous-input.d.ts.map +1 -0
  680. package/dist/services/anonymous-input.js +44 -0
  681. package/dist/services/anonymous-input.js.map +1 -0
  682. package/dist/services/audit.rpc-service.d.ts +27 -0
  683. package/dist/services/audit.rpc-service.d.ts.map +1 -0
  684. package/dist/services/audit.rpc-service.js +57 -0
  685. package/dist/services/audit.rpc-service.js.map +1 -0
  686. package/dist/services/audit.service.d.ts +165 -0
  687. package/dist/services/audit.service.d.ts.map +1 -0
  688. package/dist/services/audit.service.js +242 -0
  689. package/dist/services/audit.service.js.map +1 -0
  690. package/dist/services/auth-context.d.ts +21 -1
  691. package/dist/services/auth-context.d.ts.map +1 -1
  692. package/dist/services/auth-context.js +51 -5
  693. package/dist/services/auth-context.js.map +1 -1
  694. package/dist/services/auth.rpc-service.d.ts +0 -1
  695. package/dist/services/auth.rpc-service.d.ts.map +1 -1
  696. package/dist/services/auth.rpc-service.js +23 -9
  697. package/dist/services/auth.rpc-service.js.map +1 -1
  698. package/dist/services/auth.service.d.ts +47 -33
  699. package/dist/services/auth.service.d.ts.map +1 -1
  700. package/dist/services/auth.service.js +155 -23
  701. package/dist/services/auth.service.js.map +1 -1
  702. package/dist/services/backup-pipeline.d.ts +97 -0
  703. package/dist/services/backup-pipeline.d.ts.map +1 -0
  704. package/dist/services/backup-pipeline.js +286 -0
  705. package/dist/services/backup-pipeline.js.map +1 -0
  706. package/dist/services/backup-schedule.d.ts +69 -0
  707. package/dist/services/backup-schedule.d.ts.map +1 -0
  708. package/dist/services/backup-schedule.js +132 -0
  709. package/dist/services/backup-schedule.js.map +1 -0
  710. package/dist/services/backup.rpc-service.d.ts +31 -2
  711. package/dist/services/backup.rpc-service.d.ts.map +1 -1
  712. package/dist/services/backup.rpc-service.js +53 -0
  713. package/dist/services/backup.rpc-service.js.map +1 -1
  714. package/dist/services/backup.service.d.ts +458 -14
  715. package/dist/services/backup.service.d.ts.map +1 -1
  716. package/dist/services/backup.service.js +1368 -170
  717. package/dist/services/backup.service.js.map +1 -1
  718. package/dist/services/bundle-build-worker.d.ts +19 -0
  719. package/dist/services/bundle-build-worker.d.ts.map +1 -0
  720. package/dist/services/bundle-build-worker.js +40 -0
  721. package/dist/services/bundle-build-worker.js.map +1 -0
  722. package/dist/services/bundle-builder.d.ts +487 -0
  723. package/dist/services/bundle-builder.d.ts.map +1 -0
  724. package/dist/services/bundle-builder.js +1083 -0
  725. package/dist/services/bundle-builder.js.map +1 -0
  726. package/dist/services/bundle-worker-protocol.d.ts +19 -0
  727. package/dist/services/bundle-worker-protocol.d.ts.map +1 -0
  728. package/dist/services/bundle-worker-protocol.js +32 -0
  729. package/dist/services/bundle-worker-protocol.js.map +1 -0
  730. package/dist/services/deploy.rpc-service.d.ts +4 -1
  731. package/dist/services/deploy.rpc-service.d.ts.map +1 -1
  732. package/dist/services/deploy.rpc-service.js +10 -0
  733. package/dist/services/deploy.rpc-service.js.map +1 -1
  734. package/dist/services/deploy.service.d.ts +14 -24
  735. package/dist/services/deploy.service.d.ts.map +1 -1
  736. package/dist/services/deploy.service.js +46 -5
  737. package/dist/services/deploy.service.js.map +1 -1
  738. package/dist/services/discovery.rpc-service.d.ts +2 -1
  739. package/dist/services/discovery.rpc-service.d.ts.map +1 -1
  740. package/dist/services/discovery.rpc-service.js.map +1 -1
  741. package/dist/services/discovery.service.d.ts +32 -28
  742. package/dist/services/discovery.service.d.ts.map +1 -1
  743. package/dist/services/discovery.service.js +108 -133
  744. package/dist/services/discovery.service.js.map +1 -1
  745. package/dist/services/event-broadcaster.rpc-service.d.ts +36 -23
  746. package/dist/services/event-broadcaster.rpc-service.d.ts.map +1 -1
  747. package/dist/services/event-broadcaster.rpc-service.js +70 -32
  748. package/dist/services/event-broadcaster.rpc-service.js.map +1 -1
  749. package/dist/services/event-broadcaster.service.d.ts.map +1 -1
  750. package/dist/services/event-broadcaster.service.js.map +1 -1
  751. package/dist/services/fleet.rpc-service.d.ts +2 -1
  752. package/dist/services/fleet.rpc-service.d.ts.map +1 -1
  753. package/dist/services/fleet.rpc-service.js +3 -1
  754. package/dist/services/fleet.rpc-service.js.map +1 -1
  755. package/dist/services/fleet.service.d.ts +32 -34
  756. package/dist/services/fleet.service.d.ts.map +1 -1
  757. package/dist/services/fleet.service.js +66 -10
  758. package/dist/services/fleet.service.js.map +1 -1
  759. package/dist/services/health-check.rpc-service.d.ts +2 -1
  760. package/dist/services/health-check.rpc-service.d.ts.map +1 -1
  761. package/dist/services/health-check.rpc-service.js.map +1 -1
  762. package/dist/services/health-check.service.d.ts +50 -36
  763. package/dist/services/health-check.service.d.ts.map +1 -1
  764. package/dist/services/health-check.service.js +179 -182
  765. package/dist/services/health-check.service.js.map +1 -1
  766. package/dist/services/infrastructure.rpc-service.d.ts +297 -2
  767. package/dist/services/infrastructure.rpc-service.d.ts.map +1 -1
  768. package/dist/services/infrastructure.rpc-service.js +523 -11
  769. package/dist/services/infrastructure.rpc-service.js.map +1 -1
  770. package/dist/services/kubernetes.rpc-service.d.ts +2 -1
  771. package/dist/services/kubernetes.rpc-service.d.ts.map +1 -1
  772. package/dist/services/kubernetes.rpc-service.js.map +1 -1
  773. package/dist/services/kubernetes.service.d.ts +5 -39
  774. package/dist/services/kubernetes.service.d.ts.map +1 -1
  775. package/dist/services/kubernetes.service.js +29 -5
  776. package/dist/services/kubernetes.service.js.map +1 -1
  777. package/dist/services/local-bundle.d.ts +151 -0
  778. package/dist/services/local-bundle.d.ts.map +1 -0
  779. package/dist/services/local-bundle.js +233 -0
  780. package/dist/services/local-bundle.js.map +1 -0
  781. package/dist/services/lockout.d.ts +28 -0
  782. package/dist/services/lockout.d.ts.map +1 -0
  783. package/dist/services/lockout.js +41 -0
  784. package/dist/services/lockout.js.map +1 -0
  785. package/dist/services/log-collector.rpc-service.d.ts +16 -0
  786. package/dist/services/log-collector.rpc-service.d.ts.map +1 -1
  787. package/dist/services/log-collector.rpc-service.js +16 -0
  788. package/dist/services/log-collector.rpc-service.js.map +1 -1
  789. package/dist/services/log-collector.service.d.ts +172 -54
  790. package/dist/services/log-collector.service.d.ts.map +1 -1
  791. package/dist/services/log-collector.service.js +492 -102
  792. package/dist/services/log-collector.service.js.map +1 -1
  793. package/dist/services/log-retention.d.ts +50 -0
  794. package/dist/services/log-retention.d.ts.map +1 -0
  795. package/dist/services/log-retention.js +52 -0
  796. package/dist/services/log-retention.js.map +1 -0
  797. package/dist/services/maintenance-lock.d.ts +162 -0
  798. package/dist/services/maintenance-lock.d.ts.map +1 -0
  799. package/dist/services/maintenance-lock.js +316 -0
  800. package/dist/services/maintenance-lock.js.map +1 -0
  801. package/dist/services/master-address.d.ts +107 -0
  802. package/dist/services/master-address.d.ts.map +1 -0
  803. package/dist/services/master-address.js +154 -0
  804. package/dist/services/master-address.js.map +1 -0
  805. package/dist/services/migration-output.d.ts +16 -0
  806. package/dist/services/migration-output.d.ts.map +1 -0
  807. package/dist/services/migration-output.js +22 -0
  808. package/dist/services/migration-output.js.map +1 -0
  809. package/dist/services/netron-telemetry-transport.d.ts.map +1 -1
  810. package/dist/services/netron-telemetry-transport.js.map +1 -1
  811. package/dist/services/node-deploy-lease.d.ts +154 -0
  812. package/dist/services/node-deploy-lease.d.ts.map +1 -0
  813. package/dist/services/node-deploy-lease.js +386 -0
  814. package/dist/services/node-deploy-lease.js.map +1 -0
  815. package/dist/services/node-health.repository.d.ts +63 -4
  816. package/dist/services/node-health.repository.d.ts.map +1 -1
  817. package/dist/services/node-health.repository.js +108 -67
  818. package/dist/services/node-health.repository.js.map +1 -1
  819. package/dist/services/node-manager.rpc-service.d.ts +355 -10
  820. package/dist/services/node-manager.rpc-service.d.ts.map +1 -1
  821. package/dist/services/node-manager.rpc-service.js +730 -34
  822. package/dist/services/node-manager.rpc-service.js.map +1 -1
  823. package/dist/services/node-manager.service.d.ts +267 -95
  824. package/dist/services/node-manager.service.d.ts.map +1 -1
  825. package/dist/services/node-manager.service.js +805 -103
  826. package/dist/services/node-manager.service.js.map +1 -1
  827. package/dist/services/node-upgrade.d.ts +129 -0
  828. package/dist/services/node-upgrade.d.ts.map +1 -0
  829. package/dist/services/node-upgrade.js +161 -0
  830. package/dist/services/node-upgrade.js.map +1 -0
  831. package/dist/services/node-upgrade.service.d.ts +232 -0
  832. package/dist/services/node-upgrade.service.d.ts.map +1 -0
  833. package/dist/services/node-upgrade.service.js +468 -0
  834. package/dist/services/node-upgrade.service.js.map +1 -0
  835. package/dist/services/pipeline.rpc-service.d.ts +2 -1
  836. package/dist/services/pipeline.rpc-service.d.ts.map +1 -1
  837. package/dist/services/pipeline.rpc-service.js.map +1 -1
  838. package/dist/services/pipeline.service.d.ts +15 -46
  839. package/dist/services/pipeline.service.d.ts.map +1 -1
  840. package/dist/services/pipeline.service.js +72 -9
  841. package/dist/services/pipeline.service.js.map +1 -1
  842. package/dist/services/project.rpc-service.d.ts +133 -5
  843. package/dist/services/project.rpc-service.d.ts.map +1 -1
  844. package/dist/services/project.rpc-service.js +249 -22
  845. package/dist/services/project.rpc-service.js.map +1 -1
  846. package/dist/services/project.service.d.ts +677 -15
  847. package/dist/services/project.service.d.ts.map +1 -1
  848. package/dist/services/project.service.js +2798 -304
  849. package/dist/services/project.service.js.map +1 -1
  850. package/dist/services/redeploy-decision.d.ts +176 -0
  851. package/dist/services/redeploy-decision.d.ts.map +1 -0
  852. package/dist/services/redeploy-decision.js +142 -0
  853. package/dist/services/redeploy-decision.js.map +1 -0
  854. package/dist/services/release.rpc-service.d.ts +159 -0
  855. package/dist/services/release.rpc-service.d.ts.map +1 -0
  856. package/dist/services/release.rpc-service.js +395 -0
  857. package/dist/services/release.rpc-service.js.map +1 -0
  858. package/dist/services/release.service.d.ts +114 -0
  859. package/dist/services/release.service.d.ts.map +1 -0
  860. package/dist/services/release.service.js +208 -0
  861. package/dist/services/release.service.js.map +1 -0
  862. package/dist/services/remote-deployer.service.d.ts +569 -8
  863. package/dist/services/remote-deployer.service.d.ts.map +1 -1
  864. package/dist/services/remote-deployer.service.js +1618 -169
  865. package/dist/services/remote-deployer.service.js.map +1 -1
  866. package/dist/services/remote-ops.service.d.ts +51 -9
  867. package/dist/services/remote-ops.service.d.ts.map +1 -1
  868. package/dist/services/remote-ops.service.js +235 -39
  869. package/dist/services/remote-ops.service.js.map +1 -1
  870. package/dist/services/remote-provisioner.d.ts +122 -0
  871. package/dist/services/remote-provisioner.d.ts.map +1 -0
  872. package/dist/services/remote-provisioner.js +258 -0
  873. package/dist/services/remote-provisioner.js.map +1 -0
  874. package/dist/services/secrets.rpc-service.d.ts +51 -2
  875. package/dist/services/secrets.rpc-service.d.ts.map +1 -1
  876. package/dist/services/secrets.rpc-service.js +81 -2
  877. package/dist/services/secrets.rpc-service.js.map +1 -1
  878. package/dist/services/secrets.service.d.ts +108 -9
  879. package/dist/services/secrets.service.d.ts.map +1 -1
  880. package/dist/services/secrets.service.js +313 -48
  881. package/dist/services/secrets.service.js.map +1 -1
  882. package/dist/services/slave-storage.service.d.ts +107 -0
  883. package/dist/services/slave-storage.service.d.ts.map +1 -1
  884. package/dist/services/slave-storage.service.js +171 -2
  885. package/dist/services/slave-storage.service.js.map +1 -1
  886. package/dist/services/static-carry-forward.d.ts +85 -0
  887. package/dist/services/static-carry-forward.d.ts.map +1 -0
  888. package/dist/services/static-carry-forward.js +159 -0
  889. package/dist/services/static-carry-forward.js.map +1 -0
  890. package/dist/services/sync-policy.d.ts +167 -0
  891. package/dist/services/sync-policy.d.ts.map +1 -0
  892. package/dist/services/sync-policy.js +161 -0
  893. package/dist/services/sync-policy.js.map +1 -0
  894. package/dist/services/sync-summary.d.ts +39 -0
  895. package/dist/services/sync-summary.d.ts.map +1 -0
  896. package/dist/services/sync-summary.js +62 -0
  897. package/dist/services/sync-summary.js.map +1 -0
  898. package/dist/services/sync.rpc-service.d.ts +19 -7
  899. package/dist/services/sync.rpc-service.d.ts.map +1 -1
  900. package/dist/services/sync.rpc-service.js +36 -5
  901. package/dist/services/sync.rpc-service.js.map +1 -1
  902. package/dist/services/sync.service.d.ts +350 -18
  903. package/dist/services/sync.service.d.ts.map +1 -1
  904. package/dist/services/sync.service.js +864 -116
  905. package/dist/services/sync.service.js.map +1 -1
  906. package/dist/services/system-info.rpc-service.d.ts +2 -1
  907. package/dist/services/system-info.rpc-service.d.ts.map +1 -1
  908. package/dist/services/system-info.rpc-service.js.map +1 -1
  909. package/dist/services/system-info.service.d.ts +56 -68
  910. package/dist/services/system-info.service.d.ts.map +1 -1
  911. package/dist/services/system-info.service.js +240 -127
  912. package/dist/services/system-info.service.js.map +1 -1
  913. package/dist/services/telemetry.rpc-service.d.ts.map +1 -1
  914. package/dist/services/telemetry.rpc-service.js +9 -3
  915. package/dist/services/telemetry.rpc-service.js.map +1 -1
  916. package/dist/services/telemetry.service.d.ts +4 -2
  917. package/dist/services/telemetry.service.d.ts.map +1 -1
  918. package/dist/services/telemetry.service.js +4 -2
  919. package/dist/services/telemetry.service.js.map +1 -1
  920. package/dist/services/trace-collector.rpc-service.d.ts +2 -1
  921. package/dist/services/trace-collector.rpc-service.d.ts.map +1 -1
  922. package/dist/services/trace-collector.rpc-service.js.map +1 -1
  923. package/dist/services/trace-collector.service.d.ts +2 -41
  924. package/dist/services/trace-collector.service.d.ts.map +1 -1
  925. package/dist/services/trace-collector.service.js +27 -7
  926. package/dist/services/trace-collector.service.js.map +1 -1
  927. package/dist/services/wait-for-postgres.d.ts +28 -0
  928. package/dist/services/wait-for-postgres.d.ts.map +1 -0
  929. package/dist/services/wait-for-postgres.js +77 -0
  930. package/dist/services/wait-for-postgres.js.map +1 -0
  931. package/dist/shared/alert-expression.d.ts +60 -0
  932. package/dist/shared/alert-expression.d.ts.map +1 -0
  933. package/dist/shared/alert-expression.js +124 -0
  934. package/dist/shared/alert-expression.js.map +1 -0
  935. package/dist/shared/audit-outcome.d.ts +39 -0
  936. package/dist/shared/audit-outcome.d.ts.map +1 -0
  937. package/dist/shared/audit-outcome.js +45 -0
  938. package/dist/shared/audit-outcome.js.map +1 -0
  939. package/dist/shared/build-freshness.d.ts +59 -0
  940. package/dist/shared/build-freshness.d.ts.map +1 -0
  941. package/dist/shared/build-freshness.js +113 -0
  942. package/dist/shared/build-freshness.js.map +1 -0
  943. package/dist/shared/build-stamp.d.ts +26 -0
  944. package/dist/shared/build-stamp.d.ts.map +1 -0
  945. package/dist/shared/build-stamp.js +36 -0
  946. package/dist/shared/build-stamp.js.map +1 -0
  947. package/dist/shared/describe-error.d.ts +22 -0
  948. package/dist/shared/describe-error.d.ts.map +1 -0
  949. package/dist/shared/describe-error.js +39 -0
  950. package/dist/shared/describe-error.js.map +1 -0
  951. package/dist/shared/dto/alerts.d.ts +101 -0
  952. package/dist/shared/dto/alerts.d.ts.map +1 -0
  953. package/dist/shared/dto/alerts.js +9 -0
  954. package/dist/shared/dto/alerts.js.map +1 -0
  955. package/dist/shared/dto/auth.d.ts +49 -0
  956. package/dist/shared/dto/auth.d.ts.map +1 -0
  957. package/dist/shared/dto/auth.js +14 -0
  958. package/dist/shared/dto/auth.js.map +1 -0
  959. package/dist/shared/dto/backups.d.ts +16 -0
  960. package/dist/shared/dto/backups.d.ts.map +1 -0
  961. package/dist/shared/dto/backups.js +9 -0
  962. package/dist/shared/dto/backups.js.map +1 -0
  963. package/dist/shared/dto/deploy.d.ts +28 -0
  964. package/dist/shared/dto/deploy.d.ts.map +1 -0
  965. package/dist/shared/dto/deploy.js +8 -0
  966. package/dist/shared/dto/deploy.js.map +1 -0
  967. package/dist/shared/dto/discovery.d.ts +24 -0
  968. package/dist/shared/dto/discovery.d.ts.map +1 -0
  969. package/dist/shared/dto/discovery.js +9 -0
  970. package/dist/shared/dto/discovery.js.map +1 -0
  971. package/dist/shared/dto/fleet.d.ts +35 -0
  972. package/dist/shared/dto/fleet.d.ts.map +1 -0
  973. package/dist/shared/dto/fleet.js +8 -0
  974. package/dist/shared/dto/fleet.js.map +1 -0
  975. package/dist/shared/dto/health.d.ts +26 -0
  976. package/dist/shared/dto/health.d.ts.map +1 -0
  977. package/dist/shared/dto/health.js +9 -0
  978. package/dist/shared/dto/health.js.map +1 -0
  979. package/dist/shared/dto/kubernetes.d.ts +44 -0
  980. package/dist/shared/dto/kubernetes.d.ts.map +1 -0
  981. package/dist/shared/dto/kubernetes.js +9 -0
  982. package/dist/shared/dto/kubernetes.js.map +1 -0
  983. package/dist/shared/dto/logs.d.ts +77 -0
  984. package/dist/shared/dto/logs.d.ts.map +1 -0
  985. package/dist/shared/dto/logs.js +10 -0
  986. package/dist/shared/dto/logs.js.map +1 -0
  987. package/dist/shared/dto/metrics.d.ts +65 -0
  988. package/dist/shared/dto/metrics.d.ts.map +1 -0
  989. package/dist/shared/dto/metrics.js +17 -0
  990. package/dist/shared/dto/metrics.js.map +1 -0
  991. package/dist/shared/dto/nodes.d.ts +347 -0
  992. package/dist/shared/dto/nodes.d.ts.map +1 -0
  993. package/dist/shared/dto/nodes.js +8 -0
  994. package/dist/shared/dto/nodes.js.map +1 -0
  995. package/dist/shared/dto/pipelines.d.ts +53 -0
  996. package/dist/shared/dto/pipelines.d.ts.map +1 -0
  997. package/dist/shared/dto/pipelines.js +9 -0
  998. package/dist/shared/dto/pipelines.js.map +1 -0
  999. package/dist/shared/dto/project.d.ts +142 -8
  1000. package/dist/shared/dto/project.d.ts.map +1 -1
  1001. package/dist/shared/dto/services.d.ts +831 -7
  1002. package/dist/shared/dto/services.d.ts.map +1 -1
  1003. package/dist/shared/dto/system-info.d.ts +93 -0
  1004. package/dist/shared/dto/system-info.d.ts.map +1 -0
  1005. package/dist/shared/dto/system-info.js +9 -0
  1006. package/dist/shared/dto/system-info.js.map +1 -0
  1007. package/dist/shared/dto/traces.d.ts +49 -0
  1008. package/dist/shared/dto/traces.d.ts.map +1 -0
  1009. package/dist/shared/dto/traces.js +9 -0
  1010. package/dist/shared/dto/traces.js.map +1 -0
  1011. package/dist/shared/env-config.d.ts +74 -0
  1012. package/dist/shared/env-config.d.ts.map +1 -0
  1013. package/dist/shared/env-config.js +99 -0
  1014. package/dist/shared/env-config.js.map +1 -0
  1015. package/dist/shared/events.d.ts +45 -39
  1016. package/dist/shared/events.d.ts.map +1 -1
  1017. package/dist/shared/events.js +6 -0
  1018. package/dist/shared/events.js.map +1 -1
  1019. package/dist/shared/format.d.ts +1 -1
  1020. package/dist/shared/format.d.ts.map +1 -1
  1021. package/dist/shared/format.js +4 -0
  1022. package/dist/shared/format.js.map +1 -1
  1023. package/dist/shared/local-infra-host.d.ts +28 -0
  1024. package/dist/shared/local-infra-host.d.ts.map +1 -0
  1025. package/dist/shared/local-infra-host.js +28 -0
  1026. package/dist/shared/local-infra-host.js.map +1 -0
  1027. package/dist/shared/node-check.d.ts +37 -0
  1028. package/dist/shared/node-check.d.ts.map +1 -0
  1029. package/dist/shared/node-check.js +41 -0
  1030. package/dist/shared/node-check.js.map +1 -0
  1031. package/dist/shared/paths.d.ts +29 -0
  1032. package/dist/shared/paths.d.ts.map +1 -0
  1033. package/dist/shared/paths.js +54 -0
  1034. package/dist/shared/paths.js.map +1 -0
  1035. package/dist/shared/pnpm.d.ts +29 -0
  1036. package/dist/shared/pnpm.d.ts.map +1 -0
  1037. package/dist/shared/pnpm.js +57 -0
  1038. package/dist/shared/pnpm.js.map +1 -0
  1039. package/dist/shared/private-files.d.ts +29 -0
  1040. package/dist/shared/private-files.d.ts.map +1 -0
  1041. package/dist/shared/private-files.js +109 -0
  1042. package/dist/shared/private-files.js.map +1 -0
  1043. package/dist/shared/redact-env.d.ts +24 -0
  1044. package/dist/shared/redact-env.d.ts.map +1 -0
  1045. package/dist/shared/redact-env.js +36 -0
  1046. package/dist/shared/redact-env.js.map +1 -0
  1047. package/dist/shared/release-reading.d.ts +85 -0
  1048. package/dist/shared/release-reading.d.ts.map +1 -0
  1049. package/dist/shared/release-reading.js +81 -0
  1050. package/dist/shared/release-reading.js.map +1 -0
  1051. package/dist/shared/roles.d.ts +58 -3
  1052. package/dist/shared/roles.d.ts.map +1 -1
  1053. package/dist/shared/roles.js +55 -0
  1054. package/dist/shared/roles.js.map +1 -1
  1055. package/dist/shared/rpcauth.d.ts +13 -0
  1056. package/dist/shared/rpcauth.d.ts.map +1 -0
  1057. package/dist/shared/rpcauth.js +23 -0
  1058. package/dist/shared/rpcauth.js.map +1 -0
  1059. package/dist/shared/shell-escape.d.ts +9 -0
  1060. package/dist/shared/shell-escape.d.ts.map +1 -0
  1061. package/dist/shared/shell-escape.js +11 -0
  1062. package/dist/shared/shell-escape.js.map +1 -0
  1063. package/dist/shared/sync-reading.d.ts +45 -0
  1064. package/dist/shared/sync-reading.d.ts.map +1 -0
  1065. package/dist/shared/sync-reading.js +42 -0
  1066. package/dist/shared/sync-reading.js.map +1 -0
  1067. package/dist/shared/tokens.d.ts +19 -0
  1068. package/dist/shared/tokens.d.ts.map +1 -1
  1069. package/dist/shared/tokens.js +28 -0
  1070. package/dist/shared/tokens.js.map +1 -1
  1071. package/dist/webapp/webapp.service.d.ts +33 -2
  1072. package/dist/webapp/webapp.service.d.ts.map +1 -1
  1073. package/dist/webapp/webapp.service.js +143 -31
  1074. package/dist/webapp/webapp.service.js.map +1 -1
  1075. package/dist/workers/health-monitor-process.d.ts.map +1 -1
  1076. package/dist/workers/health-monitor-process.js +10 -0
  1077. package/dist/workers/health-monitor-process.js.map +1 -1
  1078. package/dist/workers/health-monitor.service.d.ts +45 -3
  1079. package/dist/workers/health-monitor.service.d.ts.map +1 -1
  1080. package/dist/workers/health-monitor.service.js +152 -25
  1081. package/dist/workers/health-monitor.service.js.map +1 -1
  1082. package/dist/workers/system-worker-manager.d.ts +43 -0
  1083. package/dist/workers/system-worker-manager.d.ts.map +1 -1
  1084. package/dist/workers/system-worker-manager.js +150 -3
  1085. package/dist/workers/system-worker-manager.js.map +1 -1
  1086. package/dist/workers/types.d.ts +8 -2
  1087. package/dist/workers/types.d.ts.map +1 -1
  1088. package/package.json +64 -30
  1089. package/webapp/dist/BUILD.json +5 -0
  1090. package/webapp/dist/assets/FormControlLabel-BrH2diyl.js +1 -0
  1091. package/webapp/dist/assets/Grid-BZ7xhoHQ.js +1 -0
  1092. package/webapp/dist/assets/Switch-DGSAxQPM.js +1 -0
  1093. package/webapp/dist/assets/alerts-sXqHJbsa.js +1 -0
  1094. package/webapp/dist/assets/apps-B_Ah8Flh.js +1 -0
  1095. package/webapp/dist/assets/audit-YXihwyui.js +1 -0
  1096. package/webapp/dist/assets/client-CGbBvy9k.js +1 -0
  1097. package/webapp/dist/assets/constants-veKGFkC6.js +1 -0
  1098. package/webapp/dist/assets/containers-Di4NOTLf.js +2 -0
  1099. package/webapp/dist/assets/daemon-memory-CCPZ4cV_.js +1 -0
  1100. package/webapp/dist/assets/dashboard-Cfuf95N3.js +1 -0
  1101. package/webapp/dist/assets/dashboard-builder-CowaYb-M.js +1 -0
  1102. package/webapp/dist/assets/deploy-release-dialog-D6C-XcKY.js +1 -0
  1103. package/webapp/dist/assets/deployments-ChTq5kCw.js +1 -0
  1104. package/webapp/dist/assets/detail-BsGWv7Dj.js +1 -0
  1105. package/webapp/dist/assets/detail-CyVf_cw5.js +2 -0
  1106. package/webapp/dist/assets/detail-D027j6aO.js +8 -0
  1107. package/webapp/dist/assets/formatters-BqjL73N3.js +1 -0
  1108. package/webapp/dist/assets/index-B8AHIPc-.css +1 -0
  1109. package/webapp/dist/assets/index-DiZbrwxT.js +414 -0
  1110. package/webapp/dist/assets/logs-L2b8Oqmo.js +12 -0
  1111. package/webapp/dist/assets/maplibre-gl-MlmG2kbn.js +801 -0
  1112. package/webapp/dist/assets/metrics-CQgjz2Lz.js +2 -0
  1113. package/webapp/dist/assets/nodes-Nwia2Ty6.js +7 -0
  1114. package/webapp/dist/assets/pipelines-DCu_rXOr.js +4 -0
  1115. package/webapp/dist/assets/projects-Bb4aV76h.js +1 -0
  1116. package/webapp/dist/assets/react-CwJFpaho.js +1 -0
  1117. package/webapp/dist/assets/react-apexcharts.esm-Dy0Fez42.js +1956 -0
  1118. package/webapp/dist/assets/release-bits-Dj4EOhhb.js +1 -0
  1119. package/webapp/dist/assets/release-build-panel-D6OCVlhT.js +1 -0
  1120. package/webapp/dist/assets/releases-CnAt3yO6.js +2 -0
  1121. package/webapp/dist/assets/rolldown-runtime-hePW80VL.js +1 -0
  1122. package/webapp/dist/assets/settings-DU-mjiwb.js +1 -0
  1123. package/webapp/dist/assets/settled-pair-C7SjZuns.js +1 -0
  1124. package/webapp/dist/assets/sign-in-Bcvjrr4C.js +1 -0
  1125. package/webapp/dist/assets/stacks-B-e8p50I.js +1 -0
  1126. package/webapp/dist/assets/sync-reading-C3PnJ7AX.js +1 -0
  1127. package/webapp/dist/assets/system-info-BtE3dgfw.js +1 -0
  1128. package/webapp/dist/assets/topology-Cwch-uF1.js +12 -0
  1129. package/webapp/dist/assets/topology-DLioOiRN.css +1 -0
  1130. package/webapp/dist/assets/traces-BbShR-4P.js +1 -0
  1131. package/webapp/dist/assets/use-stack-context-CZUgPq88.js +1 -0
  1132. package/webapp/dist/index.html +6 -6
  1133. package/dist/commands/dev.d.ts +0 -23
  1134. package/dist/commands/dev.d.ts.map +0 -1
  1135. package/dist/commands/dev.js +0 -76
  1136. package/dist/commands/dev.js.map +0 -1
  1137. package/dist/config/define-app.d.ts +0 -6
  1138. package/dist/config/define-app.d.ts.map +0 -1
  1139. package/dist/config/define-app.js +0 -38
  1140. package/dist/config/define-app.js.map +0 -1
  1141. package/dist/infrastructure/gateway-generator.d.ts +0 -63
  1142. package/dist/infrastructure/gateway-generator.d.ts.map +0 -1
  1143. package/dist/infrastructure/gateway-generator.js +0 -253
  1144. package/dist/infrastructure/gateway-generator.js.map +0 -1
  1145. package/dist/monitoring/health.service.d.ts +0 -14
  1146. package/dist/monitoring/health.service.d.ts.map +0 -1
  1147. package/dist/monitoring/health.service.js +0 -50
  1148. package/dist/monitoring/health.service.js.map +0 -1
  1149. package/dist/monitoring/metrics.service.d.ts +0 -14
  1150. package/dist/monitoring/metrics.service.d.ts.map +0 -1
  1151. package/dist/monitoring/metrics.service.js +0 -40
  1152. package/dist/monitoring/metrics.service.js.map +0 -1
  1153. package/dist/orchestrator/bootstrap-runner.d.ts +0 -15
  1154. package/dist/orchestrator/bootstrap-runner.d.ts.map +0 -1
  1155. package/dist/orchestrator/bootstrap-runner.js +0 -152
  1156. package/dist/orchestrator/bootstrap-runner.js.map +0 -1
  1157. package/dist/orchestrator/topology-proxy.d.ts +0 -65
  1158. package/dist/orchestrator/topology-proxy.d.ts.map +0 -1
  1159. package/dist/orchestrator/topology-proxy.js +0 -209
  1160. package/dist/orchestrator/topology-proxy.js.map +0 -1
  1161. package/dist/orchestrator/topology-relay.d.ts +0 -111
  1162. package/dist/orchestrator/topology-relay.d.ts.map +0 -1
  1163. package/dist/orchestrator/topology-relay.js +0 -255
  1164. package/dist/orchestrator/topology-relay.js.map +0 -1
  1165. package/dist/services/auth-middleware.d.ts +0 -47
  1166. package/dist/services/auth-middleware.d.ts.map +0 -1
  1167. package/dist/services/auth-middleware.js +0 -158
  1168. package/dist/services/auth-middleware.js.map +0 -1
  1169. package/dist/services/environment.rpc-service.d.ts +0 -22
  1170. package/dist/services/environment.rpc-service.d.ts.map +0 -1
  1171. package/dist/services/environment.rpc-service.js +0 -65
  1172. package/dist/services/environment.rpc-service.js.map +0 -1
  1173. package/dist/services/environment.service.d.ts +0 -87
  1174. package/dist/services/environment.service.d.ts.map +0 -1
  1175. package/dist/services/environment.service.js +0 -220
  1176. package/dist/services/environment.service.js.map +0 -1
  1177. package/dist/services/metrics-collector.service.d.ts +0 -103
  1178. package/dist/services/metrics-collector.service.d.ts.map +0 -1
  1179. package/dist/services/metrics-collector.service.js +0 -235
  1180. package/dist/services/metrics-collector.service.js.map +0 -1
  1181. package/dist/services/metrics.rpc-service.d.ts +0 -41
  1182. package/dist/services/metrics.rpc-service.d.ts.map +0 -1
  1183. package/dist/services/metrics.rpc-service.js +0 -90
  1184. package/dist/services/metrics.rpc-service.js.map +0 -1
  1185. package/dist/services/metrics.service.d.ts +0 -72
  1186. package/dist/services/metrics.service.d.ts.map +0 -1
  1187. package/dist/services/metrics.service.js +0 -185
  1188. package/dist/services/metrics.service.js.map +0 -1
  1189. package/dist/supervisor/omnitron-supervisor.d.ts +0 -21
  1190. package/dist/supervisor/omnitron-supervisor.d.ts.map +0 -1
  1191. package/dist/supervisor/omnitron-supervisor.js +0 -43
  1192. package/dist/supervisor/omnitron-supervisor.js.map +0 -1
  1193. package/dist/webapp/static-server.d.ts +0 -24
  1194. package/dist/webapp/static-server.d.ts.map +0 -1
  1195. package/dist/webapp/static-server.js +0 -152
  1196. package/dist/webapp/static-server.js.map +0 -1
  1197. package/webapp/dist/assets/Badge-D2ZBP5iK.js +0 -1
  1198. package/webapp/dist/assets/Card-vO88g06B.js +0 -1
  1199. package/webapp/dist/assets/CardActions-C9h7Q2wW.js +0 -1
  1200. package/webapp/dist/assets/CardContent-BA_Xmmhk.js +0 -1
  1201. package/webapp/dist/assets/CardHeader-ceIs__6x.js +0 -1
  1202. package/webapp/dist/assets/Chip-Ckk0xUhK.js +0 -1
  1203. package/webapp/dist/assets/DialogActions-BhHM7hov.js +0 -1
  1204. package/webapp/dist/assets/DialogContentText-BT2JQRmK.js +0 -1
  1205. package/webapp/dist/assets/FormControlLabel-CcQByVS8.js +0 -1
  1206. package/webapp/dist/assets/Grid-14PETRQW.js +0 -1
  1207. package/webapp/dist/assets/InputAdornment-B8aHIsQ7.js +0 -1
  1208. package/webapp/dist/assets/Skeleton-B74-Ds6O.js +0 -37
  1209. package/webapp/dist/assets/Switch-DvWD2wh4.js +0 -1
  1210. package/webapp/dist/assets/SwitchBase-CDU4nY8O.js +0 -1
  1211. package/webapp/dist/assets/Tab-DcCOYYp5.js +0 -1
  1212. package/webapp/dist/assets/TableContainer-CzTh4mu-.js +0 -1
  1213. package/webapp/dist/assets/TableRow-BySEreZR.js +0 -2
  1214. package/webapp/dist/assets/TextField-BLf-BHz2.js +0 -1
  1215. package/webapp/dist/assets/alerts-DnRbT41C.js +0 -1
  1216. package/webapp/dist/assets/apps-CdR0VH2d.js +0 -1
  1217. package/webapp/dist/assets/chunk-DtcAfS_W.js +0 -1
  1218. package/webapp/dist/assets/client-DU8I_lDc.js +0 -2
  1219. package/webapp/dist/assets/constants-BgnT-Xdy.js +0 -1
  1220. package/webapp/dist/assets/containers-Dw-rRXV1.js +0 -1
  1221. package/webapp/dist/assets/dashboard-CzgB0uZz.js +0 -1
  1222. package/webapp/dist/assets/dashboard-builder-DsA0yg33.js +0 -1
  1223. package/webapp/dist/assets/deployments-DtQAoIcQ.js +0 -1
  1224. package/webapp/dist/assets/detail-BJR-G7JG.js +0 -1
  1225. package/webapp/dist/assets/detail-nlCeSltl.js +0 -8
  1226. package/webapp/dist/assets/formatters-Do41SRmE.js +0 -1
  1227. package/webapp/dist/assets/index-ByQDwp2r.js +0 -566
  1228. package/webapp/dist/assets/index-CHVXOsTS.css +0 -1
  1229. package/webapp/dist/assets/isMuiElement-DwF2naaF.js +0 -1
  1230. package/webapp/dist/assets/logs-Kn8o4H7k.js +0 -11
  1231. package/webapp/dist/assets/metrics-DN8Zu5wT.js +0 -2
  1232. package/webapp/dist/assets/nodes-Bgch7AtN.js +0 -7
  1233. package/webapp/dist/assets/pipelines-Cry3qm-S.js +0 -4
  1234. package/webapp/dist/assets/popper-BJFSj6rc.js +0 -1
  1235. package/webapp/dist/assets/preload-helper-DSXbuxSR.js +0 -1
  1236. package/webapp/dist/assets/projects-Dym2GI1U.js +0 -1
  1237. package/webapp/dist/assets/react-apexcharts.esm-CSGK5Cxn.js +0 -958
  1238. package/webapp/dist/assets/react-apexcharts.esm-DEAWOwx5.js +0 -1003
  1239. package/webapp/dist/assets/settings-va3zTcow.js +0 -1
  1240. package/webapp/dist/assets/sign-in-ik35umbp.js +0 -1
  1241. package/webapp/dist/assets/stacks-CVFOnxbF.js +0 -1
  1242. package/webapp/dist/assets/system-info-npIXAjX2.js +0 -1
  1243. package/webapp/dist/assets/topology-CHpVij2M.css +0 -1
  1244. package/webapp/dist/assets/topology-bDe70vLs.js +0 -12
  1245. package/webapp/dist/assets/traces-DIIcT8DF.js +0 -1
  1246. package/webapp/dist/assets/use-stack-context-DHLwZv1r.js +0 -1
  1247. /package/webapp/dist/assets/{fs-CgZLtHiQ.js → fs-Ccl3sAcp.js} +0 -0
@@ -11,7 +11,13 @@
11
11
  *
12
12
  * Classic mode:
13
13
  * - child_process.fork() with manual exit handling (legacy compat)
14
- * - Real CPU/memory metrics via pidusage
14
+ *
15
+ * Metrics for both modes come from one `ps -o pid=,rss=,%cpu=` call covering
16
+ * every managed pid — supervisor children and pool workers alike. This line
17
+ * used to say "real CPU/memory metrics via pidusage"; `pidusage` is not a
18
+ * dependency of this package and appears nowhere else in the source. The
19
+ * sentence outlived the code it described, in the one area where a stale
20
+ * claim is hardest to catch: metrics that are wrong still look like metrics.
15
21
  *
16
22
  * Architecture:
17
23
  * pm.supervisor({ children: [{ process: bootstrap-process.js, ... }] })
@@ -21,18 +27,59 @@
21
27
  * → Two Netron instances: PM management plane + app HTTP data plane
22
28
  */
23
29
  import fs from 'node:fs';
30
+ import { readDeclaredConfig, withDeclaredConfig } from '../project/declared-config.js';
24
31
  import path from 'node:path';
25
32
  import { fileURLToPath } from 'node:url';
26
- import { PoolStrategy, SupervisionStrategy, } from '@omnitron-dev/titan-pm';
33
+ import { PoolStrategy, SupervisionStrategy, ProcessLifecycleQueue, LifecyclePreempted, computeBackoff, isAlive, discoverManagedProcesses, } from '@omnitron-dev/titan-pm';
27
34
  import { EventEmitter } from '@omnitron-dev/eventemitter';
28
35
  import { AppHandle } from './app-handle.js';
36
+ import { parsePsLine, parsePsBatch } from './ps-metrics.js';
29
37
  import { launchClassic } from './classic-launcher.js';
30
38
  import { resolveStartupOrder, resolveShutdownOrder } from './dependency-resolver.js';
31
39
  import { buildRestartPolicy } from '../supervisor/restart-policy.js';
40
+ /**
41
+ * How long to wait for a pool worker to finish exposing its `@Service`
42
+ * classes before giving up on `topology.expose`. A worker reports ready
43
+ * before its Application starts, and exposure happens during start, so the
44
+ * gap is a real one — measured at ~7s for a worker that opens database
45
+ * connections first. The consumer that needs the service starts right after
46
+ * this returns, so the wait is paid once at boot and buys a feature that
47
+ * otherwise stays off until the next restart.
48
+ */
49
+ /**
50
+ * `childName` is sometimes bare ('http') and sometimes already carries the app
51
+ * ('acme/dev/pricing/http'), depending on which supervisor path raised the
52
+ * event. Prefixing unconditionally produced
53
+ * `acme/dev/pricing/acme/dev/pricing/http` in the log file.
54
+ */
55
+ function qualifyChildName(appName, childName) {
56
+ return childName.startsWith(`${appName}/`) || childName === appName
57
+ ? childName
58
+ : `${appName}/${childName}`;
59
+ }
60
+ const TOPOLOGY_EXPOSE_TIMEOUT_MS = 30_000;
61
+ const TOPOLOGY_EXPOSE_POLL_MS = 250;
32
62
  import { ServiceRouter } from './service-router.js';
33
63
  import { loadBootstrapConfig, clearCacheFor } from './bootstrap-loader.js';
64
+ /** Levels a pino logger accepts. A child is spawned with nothing else. */
65
+ const PINO_LEVELS = new Set(['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent']);
66
+ /**
67
+ * Process-manager states in which a record no longer describes a live process.
68
+ *
69
+ * Read by `managerOwnedPids`, where the distinction decides whether a pid is
70
+ * protected from the janitor. Written as strings rather than imported from
71
+ * `ProcessStatus` because that export is a const object whose values are
72
+ * these same literals; the set has to hold values, not keys, and a typo here
73
+ * would silently protect a dead pid — so it is pinned by a test.
74
+ */
75
+ const TERMINAL_PROCESS_STATUSES = new Set(['stopped', 'failed', 'crashed']);
34
76
  import { BuildService } from './build-service.js';
77
+ import { execFile } from 'node:child_process';
35
78
  import { ProcessJanitor } from './process-janitor.js';
79
+ import { addressInUse, holderCommand, parseHolders, explainConflict } from './port-conflict.js';
80
+ import { collectOwnedPids } from './owned-pids.js';
81
+ import { combineProcessHealth, notRunningHealth, poolHealth } from './app-health.js';
82
+ import { combineProcessTraffic } from './app-traffic.js';
36
83
  import { CLI_VERSION } from '../config/defaults.js';
37
84
  const __dirname = path.dirname(fileURLToPath(import.meta.url));
38
85
  // Resolve bootstrap-process path — .ts in dev (tsx), .js when compiled
@@ -63,15 +110,77 @@ function mapStrategy(strategy) {
63
110
  * stack mode + bare-name RPC paths can be deduplicated.
64
111
  *
65
112
  * Examples:
66
- * effectiveAppName('omni/dev/paysys') === 'paysys'
67
- * effectiveAppName('paysys') === 'paysys'
68
- * effectiveAppName('a/b/c/paysys') === 'paysys' (last segment)
113
+ * effectiveAppName('omni/dev/payments') === 'payments'
114
+ * effectiveAppName('payments') === 'payments'
115
+ * effectiveAppName('a/b/c/payments') === 'payments' (last segment)
69
116
  */
70
- function effectiveAppName(key) {
117
+ export function effectiveAppName(key) {
71
118
  const idx = key.lastIndexOf('/');
72
119
  return idx >= 0 ? key.slice(idx + 1) : key;
73
120
  }
121
+ /**
122
+ * Auto-promote a bare `entry.name` to canonical `${project}/${stack}/${name}`
123
+ * form when the env carries OMNITRON_PROJECT + OMNITRON_STACK. The stack-mode
124
+ * boot path (ProjectService.startStack) already prefixes the name explicitly;
125
+ * this helper closes every other entry point that could otherwise register a
126
+ * bare-name handle which then becomes invisible to the stack-prefix-based
127
+ * UI lookup (ProjectService.toStackInfo filters on `${project}/${stack}/`).
128
+ *
129
+ * Idempotent: if the entry already looks namespaced (or env lacks the keys),
130
+ * the input is returned unchanged. An entry qualified under a DIFFERENT
131
+ * project/stack is preserved too — that's operator intent (cross-project
132
+ * administrative run), not a bug.
133
+ */
134
+ export function ensureNamespacedEntry(entry) {
135
+ const project = entry.env?.['OMNITRON_PROJECT'];
136
+ const stack = entry.env?.['OMNITRON_STACK'];
137
+ if (!project || !stack)
138
+ return entry;
139
+ const canonicalPrefix = `${project}/${stack}/`;
140
+ if (entry.name.startsWith(canonicalPrefix))
141
+ return entry;
142
+ if (entry.name.includes('/'))
143
+ return entry;
144
+ return { ...entry, name: `${canonicalPrefix}${entry.name}` };
145
+ }
74
146
  const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
147
+ /**
148
+ * Startup budget for a child process that declares none of its own.
149
+ *
150
+ * There were TWO of these and they disagreed — `?? 60_000` on the pooled path
151
+ * and `?? 30_000` on the single-process path — and, worse, the pooled one did
152
+ * not consult `entry.startupTimeout` at all. So an operator who set
153
+ * `startupTimeout: 120_000` on an app had it honoured for the app and
154
+ * SILENTLY IGNORED for its child processes, which kept 60 seconds and could
155
+ * not be given more by any setting.
156
+ *
157
+ * Measured on this host: an app whose own successful boot takes 120.6 seconds
158
+ * carried a 120 s budget, and its `deposit-worker` child — a deposit scanner
159
+ * — stayed `stopped` while the app itself came up. Two copies of one decision,
160
+ * one of which discarded the only knob an operator has.
161
+ *
162
+ * 60 s rather than 30 s, for the reason the project's own config already
163
+ * writes down: an over-long ceiling costs a slower failure report, a short one
164
+ * costs the whole stack.
165
+ */
166
+ const DEFAULT_CHILD_STARTUP_TIMEOUT = 60_000;
167
+ /**
168
+ * Deadline for an RPC the daemon makes to a child it supervises.
169
+ *
170
+ * A call across this connection is WORK — aggregate five minutes of candles,
171
+ * answer a health probe that touches a pool, hand back metrics — not a wire
172
+ * request, and netron's default (5000 ms) is sized for the latter. The pool
173
+ * path stated this number and the single-child path stated nothing, so a
174
+ * client built for a single child fell back to that default: measured, 94
175
+ * timeouts «after 5000ms» in the daemon log, every one on
176
+ * `callExposedService`, which lives on exactly those single children.
177
+ *
178
+ * It belongs here rather than in titan-pm because the orchestrator is what
179
+ * knows the nature of the work; the library's own fallback is 30 s
180
+ * (`process-pool.ts:212`) and applies when a caller states nothing — which,
181
+ * after this, no caller here does.
182
+ */
183
+ const CHILD_REQUEST_TIMEOUT = 120_000;
75
184
  export class OrchestratorService extends EventEmitter {
76
185
  logger;
77
186
  pm;
@@ -88,6 +197,30 @@ export class OrchestratorService extends EventEmitter {
88
197
  * for the same name isn't permanently locked out.
89
198
  */
90
199
  startingApps = new Map();
200
+ /**
201
+ * Per-app async serial-execution queue covering start / stop / restart
202
+ * / crash-restart-timer (P0-E + P0-I + P2-B in the audit). Pre-this-
203
+ * queue each path had its own coalescer:
204
+ * - startingApps: same-name start coalescing only — `stopApp` did
205
+ * NOT consult it, so stop racing in-flight start left an orphan
206
+ * supervisor when stop arrived before start wrote childProcess.
207
+ * - restartCoalescer: same-name restart coalescing; independent of
208
+ * stop+start.
209
+ * - handleClassicCrash setTimeout: a raw timer that could fire
210
+ * during the `await stopChild(...)` window and spawn a zombie
211
+ * against an app the operator just stopped.
212
+ * The queue collapses all four into one primitive ordered by
213
+ * priority (stop > restart > start > crash-restart). The same-key
214
+ * mutex means stop ALWAYS observes a completed start, and an
215
+ * explicit stop ALWAYS preempts a pending crash-restart timer.
216
+ */
217
+ lifecycleQueue = new ProcessLifecycleQueue();
218
+ /**
219
+ * Pending post-escalation cool-down timers, keyed by app name.
220
+ * Held here rather than inside a lifecycle-queue slot so that a
221
+ * crashed app still answers operator commands while it waits.
222
+ */
223
+ coolDownTimers = new Map();
91
224
  config = null;
92
225
  cwd;
93
226
  metricsTimer = null;
@@ -129,10 +262,23 @@ export class OrchestratorService extends EventEmitter {
129
262
  */
130
263
  static POST_ONLINE_COOLDOWN_MS = 3_000;
131
264
  /**
132
- * Orphan-process reaper. Lazily created on first `startAll` so unit
133
- * tests that hit `startApp` directly don't get a periodic timer.
265
+ * Orphan-process reaper. Lazily created on first reconcile so unit
266
+ * tests that hit `startApp` directly don't get a periodic timer
267
+ * until the first real reconcile runs (which they bypass by
268
+ * stubbing `bootReconciled = true`).
134
269
  */
135
270
  janitor = null;
271
+ /**
272
+ * One-shot guard for cold-start reconciliation. T#63 — every code
273
+ * path that lands an app on this orchestrator goes through
274
+ * `ensureBootReconciled()`, regardless of whether the entry-point
275
+ * is `startAll` (daemon-config) or `startApp` (project/stack
276
+ * mode). Pre-T#63 the cold-start sweep only fired inside
277
+ * `startAll`, so project-mode boots silently inherited orphans
278
+ * from the previous daemon generation.
279
+ */
280
+ bootReconciled = false;
281
+ bootReconcilePromise = null;
136
282
  /** Daemon's Netron instance for native service routing */
137
283
  daemonNetron = null;
138
284
  /** Daemon Unix socket path for child-to-daemon Netron connections */
@@ -220,13 +366,117 @@ export class OrchestratorService extends EventEmitter {
220
366
  // ============================================================================
221
367
  // Lifecycle
222
368
  // ============================================================================
369
+ /**
370
+ * Refresh the orchestrator's view of the ecosystem config. Called
371
+ * from `daemon.reloadConfig` so a subsequent `restartApp` reads the
372
+ * new entry definition (mode, env, dependsOn, restart policy)
373
+ * instead of the stale copy captured at first `startAll`. Pre-fix
374
+ * the two refs diverged silently after every reload — operators
375
+ * editing ecosystem.config.ts and running `omnitron reload`
376
+ * believed the daemon picked up the changes; in fact only daemon's
377
+ * own RPC service did, and individual app restarts kept running
378
+ * against the pre-reload definition until the daemon itself
379
+ * restarted. P0-G in the audit.
380
+ */
381
+ setConfig(config) {
382
+ this.config = config;
383
+ // Metrics polling is armed here rather than in `startAll` alone.
384
+ //
385
+ // It used to run only from `startAll`, which is the `omnitron start`
386
+ // path. Apps brought up any other way — `stack start`, a single
387
+ // `omnitron start <app>`, or the daemon resuming a stack at boot — were
388
+ // never sampled, so `omnitron list`, `status` and the console's app
389
+ // cards reported cpu 0 / memory 0 for processes that were serving
390
+ // traffic. Observed live: six apps online, every one showing 0 while
391
+ // `ps` reported ~105 MB RSS apiece.
392
+ //
393
+ // `setConfig` is the one point every path passes through, and the call
394
+ // is idempotent — an existing timer is cleared first.
395
+ this.ensureMetricsPolling();
396
+ }
397
+ /**
398
+ * Start metrics sampling if it is not already running.
399
+ *
400
+ * Called from every path that can produce a managed process, so no app can
401
+ * end up unsampled because of how it was started.
402
+ */
403
+ ensureMetricsPolling() {
404
+ if (this.metricsTimer)
405
+ return;
406
+ this.startMetricsPolling(this.config?.monitoring?.metrics?.interval ?? 5_000);
407
+ }
408
+ /**
409
+ * One-shot cold-start reconciliation: discover pre-existing managed
410
+ * processes, reap fork-workers left over from a previous daemon
411
+ * generation, and arm the periodic janitor sweep. Idempotent —
412
+ * concurrent callers join the in-flight promise; subsequent calls
413
+ * after success are no-ops.
414
+ *
415
+ * Called from BOTH `startAll` (top-level daemon-config boot) and
416
+ * `startApp` (project/stack mode boot), so every code path that
417
+ * lands an app on this orchestrator runs the cleanup. Pre-T#63 the
418
+ * cold-start sweep was gated on `startAll` and never ran in
419
+ * project mode, so an orphan from a previous daemon survived
420
+ * silently and held the listen port hostage from its successor.
421
+ */
422
+ async ensureBootReconciled(config) {
423
+ if (this.bootReconciled)
424
+ return;
425
+ if (this.bootReconcilePromise) {
426
+ await this.bootReconcilePromise;
427
+ return;
428
+ }
429
+ this.bootReconcilePromise = (async () => {
430
+ // Discovery: surface OMNITRON_MANAGED-tagged processes that match
431
+ // a configured app so the operator can see what's about to be
432
+ // reaped (or — once the ChildContract handshake lands — adopted).
433
+ // P2-A in the audit.
434
+ try {
435
+ const discovered = discoverManagedProcesses();
436
+ const adopted = [];
437
+ const knownApps = new Set(config?.apps.map((a) => a.name) ?? []);
438
+ for (const proc of discovered) {
439
+ if (proc.ppid === process.pid)
440
+ continue; // our own freshly-spawned child
441
+ const fqn = proc.fullyQualifiedName;
442
+ const bare = proc.appName;
443
+ if (knownApps.has(fqn) || knownApps.has(bare)) {
444
+ adopted.push(`${fqn}#${proc.pid}`);
445
+ }
446
+ }
447
+ if (adopted.length > 0) {
448
+ this.logger.warn({ discovered: adopted.length, apps: adopted }, 'Found pre-existing managed processes — they will be reaped by the janitor in 60s unless adopted');
449
+ }
450
+ }
451
+ catch (err) {
452
+ this.logger.debug({ err: err.message }, 'Process discovery failed — proceeding without reconcile');
453
+ }
454
+ // Reap any fork-workers left over from a previous daemon BEFORE
455
+ // launching apps. Without this, a hard-killed daemon's children
456
+ // are still alive (now reparented to init); they hold DB
457
+ // connections and listen ports that the new apps will fight
458
+ // for. The reap is bounded by `gracefulMs` (default 3s).
459
+ await this.ensureJanitor().coldStartSweep();
460
+ // Arm the periodic sweep so runtime leaks from partially-failed
461
+ // restart cycles (esbuild-rebuild → spawn-fail → orphan) get
462
+ // collected on every interval.
463
+ this.ensureJanitor().start();
464
+ })();
465
+ try {
466
+ await this.bootReconcilePromise;
467
+ this.bootReconciled = true;
468
+ }
469
+ finally {
470
+ this.bootReconcilePromise = null;
471
+ }
472
+ }
223
473
  async startAll(config) {
224
474
  this.config = config;
225
- // Reap any fork-workers left over from a previous daemon BEFORE
226
- // launching apps. Without this, a hard-killed daemon's children
227
- // are still alive (now adopted by init); they hold DB connections
228
- // and sockets that the new apps will fight for.
229
- await this.ensureJanitor().coldStartSweep();
475
+ // Cold-start orphan reap + periodic janitor arming. Idempotent —
476
+ // safe to call on every boot path. Project-mode flows that call
477
+ // `startApp` directly trigger the same one-shot reconcile, so
478
+ // every boot path lands in the same post-condition. (T#63)
479
+ await this.ensureBootReconciled(config);
230
480
  // Dependency-aware skip: if app A fails to start, every app that
231
481
  // declares A in `dependsOn` (transitively) is skipped instead of
232
482
  // attempted. Without this, a downstream app would launch, fail to
@@ -255,10 +505,8 @@ export class OrchestratorService extends EventEmitter {
255
505
  if (failed.size > 0 || blocked.size > 0) {
256
506
  this.logger.warn({ failed: [...failed], blocked: [...blocked] }, 'Startup completed with failures — some apps did not launch');
257
507
  }
258
- // Periodic sweep handles runtime leaks from partially-failed
259
- // restart cycles (esbuild-rebuild → spawn-fail → orphan).
260
- this.ensureJanitor().start();
261
- this.startMetricsPolling(config.monitoring.metrics.interval);
508
+ // Periodic sweep was armed by `ensureBootReconciled` above, and metrics
509
+ // polling by `setConfig` — no second call needed for either.
262
510
  this.persistState();
263
511
  }
264
512
  /**
@@ -276,44 +524,143 @@ export class OrchestratorService extends EventEmitter {
276
524
  return this.janitor;
277
525
  }
278
526
  /**
279
- * Walk every live handle's supervisor children and collect the OS
280
- * pids the orchestrator currently considers ours. Cheap — called
281
- * every janitor sweep.
527
+ * Every OS pid the orchestrator currently considers ours. Cheap — called
528
+ * on every janitor sweep, and the janitor kills whatever this set omits.
529
+ *
530
+ * The rule itself lives in `owned-pids.ts`, where it can be tested: the
531
+ * janitor's own tests supply this set as a fixture, so they could never
532
+ * catch it being incomplete — and it was, by a whole class of process.
533
+ */
534
+ /**
535
+ * Who is holding the port an app could not bind, in one sentence.
536
+ *
537
+ * Called where a start or restart failed, because that is the moment the
538
+ * answer is worth having and the moment nobody has it: the error names the
539
+ * address and never the holder. Measured on this daemon — four
540
+ * `EADDRINUSE` failures on 3005 in thirty-one seconds, `omnitron list`
541
+ * reporting the app crashed with no pid, and the port held the whole time
542
+ * by a child of this very daemon that nothing in its records claimed.
543
+ *
544
+ * Returns null when the failure was not a port clash, and says so out loud
545
+ * when it could not read the machine — a reading that failed and a port
546
+ * nobody holds are different findings.
282
547
  */
548
+ async explainPortConflict(message) {
549
+ const address = addressInUse(message);
550
+ if (!address)
551
+ return null;
552
+ const { file, args } = holderCommand(address.port, process.platform);
553
+ let output;
554
+ try {
555
+ output = await new Promise((resolve, reject) => {
556
+ execFile(file, args, { encoding: 'utf8', timeout: 5_000 }, (err, stdout) => {
557
+ // `ss` and `lsof` both exit non-zero when nothing matches, which is
558
+ // an answer rather than a failure; the empty stdout carries it.
559
+ if (err && !stdout)
560
+ reject(err);
561
+ else
562
+ resolve(stdout);
563
+ });
564
+ });
565
+ }
566
+ catch (err) {
567
+ return `could not read who holds port ${address.port}: ${file} ${err.message}`;
568
+ }
569
+ const holders = parseHolders(output, process.platform);
570
+ // One `ps` for the parents, before deciding: the decision is pure and
571
+ // takes the answers, so that it can be tested without a machine.
572
+ const parents = new Map();
573
+ for (const holder of holders) {
574
+ const ppid = await this.parentOf(holder.pid);
575
+ if (ppid !== undefined)
576
+ parents.set(holder.pid, ppid);
577
+ }
578
+ const verdict = explainConflict({
579
+ port: address.port,
580
+ holders,
581
+ parentOf: (pid) => parents.get(pid),
582
+ daemonPid: process.pid,
583
+ owned: this.collectOwnedPids(),
584
+ });
585
+ return verdict.because;
586
+ }
587
+ /** The parent of one process, or undefined when it cannot be read. */
588
+ async parentOf(pid) {
589
+ try {
590
+ const out = await new Promise((resolve, reject) => {
591
+ execFile('ps', ['-o', 'ppid=', '-p', String(pid)], { encoding: 'utf8', timeout: 5_000 }, (err, stdout) => {
592
+ if (err && !stdout)
593
+ reject(err);
594
+ else
595
+ resolve(stdout);
596
+ });
597
+ });
598
+ const parsed = Number(out.trim());
599
+ return Number.isInteger(parsed) ? parsed : undefined;
600
+ }
601
+ catch {
602
+ // A process that exited between the two readings has no parent to
603
+ // report, and that is not an error worth a line of its own — the
604
+ // verdict already says the holder was gone.
605
+ return undefined;
606
+ }
607
+ }
283
608
  collectOwnedPids() {
284
- const owned = new Set();
285
- for (const handle of this.handles.values()) {
286
- const supervisor = handle.supervisor;
287
- if (!supervisor)
609
+ return collectOwnedPids(this.handles.values(), (workerId) => this.pm.getWorkerHandle(workerId)?.pid, this.managerOwnedPids());
610
+ }
611
+ /**
612
+ * Every pid the process manager currently vouches for.
613
+ *
614
+ * Two windows, and a process is only safe if both are asked. `getForkedPids`
615
+ * holds a claim from `fork()` until `spawn()` resolves and releases it —
616
+ * after that its own docstring says "the WorkerHandle registry is the
617
+ * authority". `listProcesses()` IS that registry, and it is the only source
618
+ * that answers for a process the orchestrator did not start: the daemon's
619
+ * system workers go through this same manager but belong to no app handle,
620
+ * so walking `this.handles` can never see them.
621
+ *
622
+ * Terminal records are excluded deliberately. They keep the pid they had,
623
+ * the OS reuses pids, and a stale one here would vouch for exactly the
624
+ * leaked worker the janitor exists to reap.
625
+ */
626
+ managerOwnedPids() {
627
+ const pids = [...(this.pm.getForkedPids?.() ?? [])];
628
+ for (const info of this.pm.listProcesses?.() ?? []) {
629
+ if (typeof info.pid !== 'number')
288
630
  continue;
289
- for (const childName of supervisor.getChildNames()) {
290
- const processId = supervisor.getChildProcessId(childName);
291
- if (!processId)
292
- continue;
293
- const workerHandle = this.pm.getWorkerHandle(processId);
294
- const pid = workerHandle?.pid;
295
- if (typeof pid === 'number')
296
- owned.add(pid);
297
- }
631
+ if (TERMINAL_PROCESS_STATUSES.has(info.status))
632
+ continue;
633
+ pids.push(info.pid);
298
634
  }
299
- return owned;
635
+ return pids;
300
636
  }
301
637
  async startApp(entry, config) {
302
- // T#57: coalesce concurrent calls for the same app name. Without
303
- // this, two near-simultaneous `startApp({ name: 'main' })` calls
304
- // both see `this.handles.get('main')` as undefined-or-not-online
305
- // (the first one's handle is in status='starting'), both fall
306
- // through to `launchBootstrapMode`, and we spawn two parallel
307
- // worker pools competing for the same port. Promise coalescing
308
- // — return the in-flight promise to the second caller — gives
309
- // both callers the same handle and ensures only one launch
310
- // actually runs.
638
+ // T#63: run the one-shot cold-start reconcile (orphan reap +
639
+ // janitor arming) before ANY app spawns. Project-mode boots flow
640
+ // through `projectService.startStack → orchestrator.startApp`
641
+ // and historically bypassed the reap entirely — a previous
642
+ // daemon's children (now reparented to init) kept their HTTP
643
+ // listen ports and the fresh child failed to bind silently.
644
+ // No-op after first call thanks to `bootReconciled`.
645
+ await this.ensureBootReconciled(config);
646
+ // Defensive auto-namespacing — promotes a bare entry.name to the
647
+ // canonical `${project}/${stack}/${name}` form when the env carries
648
+ // OMNITRON_PROJECT + OMNITRON_STACK. Catches every code path that
649
+ // bypasses ProjectService.startStack:
650
+ // - direct CLI `omnitron start <name>` (no stack context)
651
+ // - legacy ecosystem configs without per-app namespace
652
+ // - daemon startAll() flows over a not-yet-namespaced config
653
+ entry = ensureNamespacedEntry(entry);
311
654
  const inflight = this.startingApps.get(entry.name);
312
655
  if (inflight) {
313
656
  this.logger.debug({ app: entry.name }, 'Coalescing concurrent startApp into in-flight launch (T#57)');
314
657
  return inflight;
315
658
  }
316
- const promise = this.startAppInternal(entry, config).finally(() => {
659
+ // Serialise against stop/restart for the same app via the
660
+ // lifecycle queue. A stopApp arriving while this start is queued
661
+ // (or in-flight) now waits its turn instead of racing through
662
+ // and leaving an orphan supervisor.
663
+ const promise = this.lifecycleQueue.enqueue(entry.name, 'start', () => this.startAppInternal(entry, config)).finally(() => {
317
664
  this.startingApps.delete(entry.name);
318
665
  });
319
666
  this.startingApps.set(entry.name, promise);
@@ -339,17 +686,37 @@ export class OrchestratorService extends EventEmitter {
339
686
  // - exact key already exists, online → return it (idempotent)
340
687
  // - exact key already exists, not online → fall through and re-launch
341
688
  // under the same key
342
- // - DIFFERENT key with the same effective short name → reject:
343
- // operator should restart by canonical name, not register an
344
- // alias.
689
+ // - DIFFERENT key with the same effective short name:
690
+ // - new is MORE specific (namespaced) and existing is bare → stop
691
+ // the bare one and re-launch under the namespaced key. Without
692
+ // this, a boot-race where bare-name auto-start beats the
693
+ // project/stack flow strands the app under the wrong key and
694
+ // webapp/CLI lookups by namespaced key see it as stopped.
695
+ // - new is LESS-or-equally specific → refuse: operator should
696
+ // restart by canonical name, not register an alias.
345
697
  const effective = effectiveAppName(entry.name);
698
+ const newIsNamespaced = entry.name.includes('/');
346
699
  for (const [key, h] of this.handles) {
347
700
  if (key === entry.name)
348
701
  continue;
349
702
  if (effectiveAppName(key) !== effective)
350
703
  continue;
704
+ const existingIsNamespaced = key.includes('/');
351
705
  // Same effective app under a different key.
352
706
  if (h.status === 'online') {
707
+ if (newIsNamespaced && !existingIsNamespaced) {
708
+ this.logger.info({ stale: key, replacing_with: entry.name, effective }, 'Stopping bare-name handle to re-register under namespaced key');
709
+ try {
710
+ await this.stopApp(key);
711
+ }
712
+ catch (err) {
713
+ this.logger.warn({ stale: key, error: err.message }, 'stopApp during dedup failed — clearing handle anyway');
714
+ }
715
+ this.handles.delete(key);
716
+ if (key !== entry.name)
717
+ this.metricsBridge?.evictApp(key);
718
+ continue;
719
+ }
353
720
  this.logger.warn({ existing: key, attempted: entry.name, effective }, 'App already running under a canonical name — refusing duplicate registration');
354
721
  return h;
355
722
  }
@@ -357,6 +724,12 @@ export class OrchestratorService extends EventEmitter {
357
724
  // failed previous launch doesn't permanently block the canonical
358
725
  // form.
359
726
  this.logger.info({ stale: key, replacing_with: entry.name, effective }, 'Replacing stale duplicate handle');
727
+ // The handle is errored or stopped, so `stopApp` never ran for it and
728
+ // its services are still registered on the daemon's Netron. Dropping the
729
+ // handle here is the last moment anything knows they exist — or that its
730
+ // processes still run: an errored app's other children and its pools.
731
+ await this.releaseExposedServices(h);
732
+ await this.tearDownProcesses(h);
360
733
  this.handles.delete(key);
361
734
  // Drop any metrics that were collected under the stale alias so it
362
735
  // doesn't haunt future snapshots as a ghost offline app.
@@ -368,9 +741,40 @@ export class OrchestratorService extends EventEmitter {
368
741
  this.logger.warn({ app: entry.name }, 'App already running');
369
742
  return existing;
370
743
  }
744
+ // A handle that is not online may still run processes. A crash restart and
745
+ // a cool-down recovery only flip its status before coming here; one child
746
+ // crashing leaves its siblings — the server among them — up; and its pools
747
+ // were never any supervisor's to stop. Stopped before the new handle
748
+ // starts, or the new one finds its own predecessor on its port: on
749
+ // daos/test (2026-09-25) paysys's crash restarts failed five times on
750
+ // `EADDRINUSE :3004`, «held by … a child this daemon still accounts for».
751
+ if (existing) {
752
+ await this.releaseExposedServices(existing);
753
+ await this.tearDownProcesses(existing);
754
+ }
371
755
  const mode = entry.bootstrap ? 'bootstrap' : 'classic';
372
756
  const handle = new AppHandle(entry, mode);
757
+ // The restart budget belongs to the APP, not to this handle instance.
758
+ // Every start replaces the handle, so a counter that starts at zero here
759
+ // can never reach `maxRestarts` — which made the budget, the escalation
760
+ // and the cool-down unreachable even once the restart itself worked. The
761
+ // window is enforced by the policy, not by forgetting.
762
+ if (existing)
763
+ handle.restarts = existing.restarts;
373
764
  this.handles.set(entry.name, handle);
765
+ // Arm metrics sampling here — the one place an app enters the registry.
766
+ //
767
+ // It used to be armed only from `startAll` (the `omnitron start` path).
768
+ // Anything else — `stack start`, a single `omnitron start <app>`, the
769
+ // daemon resuming a stack at boot — was never sampled, so `list`,
770
+ // `status` and the console's app cards reported cpu 0 / memory 0 for
771
+ // processes serving traffic. Observed: six apps online, every one at
772
+ // zero, while `ps` showed ~105 MB RSS apiece.
773
+ //
774
+ // `setConfig` looked like the right seam but is only called on config
775
+ // RELOAD, so it never fired on a normal start. This is the seam that
776
+ // actually covers every path. Idempotent: re-arming replaces the timer.
777
+ this.ensureMetricsPolling();
374
778
  this.logger.info({ app: entry.name, mode }, 'Starting app');
375
779
  try {
376
780
  if (mode === 'classic') {
@@ -379,43 +783,113 @@ export class OrchestratorService extends EventEmitter {
379
783
  else {
380
784
  await this.launchBootstrapMode(entry, handle, cfg);
381
785
  }
382
- this.logger.info({ app: entry.name, pid: handle.pid, mode }, 'App started');
786
+ // «App started» only for an app that is. It was logged in the same
787
+ // millisecond as a child's «Process failed to start», for an app whose
788
+ // processes never all came up.
789
+ if (handle.status === 'online') {
790
+ this.logger.info({ app: entry.name, pid: handle.pid, mode }, 'App started');
791
+ }
792
+ else {
793
+ this.logger.error({ app: entry.name, status: handle.status, mode }, 'App did not come up — not every declared process started');
794
+ }
383
795
  this.persistState();
384
796
  return handle;
385
797
  }
386
798
  catch (err) {
799
+ // `errored` rather than `crashed` is the right REPORT — the app never
800
+ // ran. It was also, until now, the end of the story: `'errored'` is read
801
+ // by no code in this repository outside formatters, `status` and a
802
+ // metrics label, so a start that failed was never attempted again. One
803
+ // try, then errored until a human noticed.
804
+ //
805
+ // That asymmetry is backwards. A failure to start is MORE likely to be
806
+ // transient than a crash after a successful boot — a dependency still
807
+ // coming up, a disk still cold, a machine briefly loaded. Six bootstrap
808
+ // apps here timed out at `config:loading` under a load average of 149
809
+ // and stayed down after the load passed, because nothing asked again.
810
+ //
811
+ // The counter is bumped by hand because `markErrored()` deliberately
812
+ // does not (an app that never ran has not "restarted"), and without it
813
+ // the budget below can never trip and the retry would be unbounded.
387
814
  handle.markErrored();
815
+ handle.restarts++;
388
816
  this.persistState();
817
+ this.scheduleRecovery(entry, handle, cfg);
389
818
  throw err;
390
819
  }
391
820
  }
392
- async stopApp(name, force = false, timeout = 10_000) {
393
- const canonical = this.resolveAppName(name) ?? name;
394
- const handle = this.handles.get(canonical);
395
- if (!handle || handle.status === 'stopped')
396
- return;
397
- handle.markStopping();
398
- // Unexpose topology services from daemon Netron before stopping processes
399
- if (handle.serviceRouter) {
400
- try {
401
- for (const svcName of handle.serviceRouter.getServiceNames()) {
402
- await handle.serviceRouter.unexposeService(svcName);
403
- }
821
+ /**
822
+ * Everything a handle still runs, stopped: its supervisor's children (or its
823
+ * classic child process) AND its topology pools.
824
+ *
825
+ * The pools were missed. A pool's workers are not the supervisor's children
826
+ * — they are the daemon's, siblings of the server, recorded only in
827
+ * `topologyPools` — so `supervisor.stop()` never reached them, and nothing
828
+ * in the orchestrator destroyed a pool: only the daemon's own shutdown did.
829
+ * Every restart of an app with `instances > 1` left the previous workers
830
+ * running. On daos/test (2026-09-25): priceverse's `ohlcv-aggregator` and
831
+ * storage's `transform` in three generations at once.
832
+ *
833
+ * The pools go even when the supervisor's stop throws: a stop that failed
834
+ * half-way must not also leave the half nobody asked about.
835
+ */
836
+ async tearDownProcesses(handle, force = false, timeout = 10_000) {
837
+ try {
838
+ if (handle.mode === 'bootstrap' && handle.supervisor) {
839
+ await handle.supervisor.stop();
404
840
  }
405
- catch {
406
- // Best-effort cleanup
841
+ else if (handle.childProcess) {
842
+ await this.stopChildProcess(handle, force, timeout);
843
+ }
844
+ }
845
+ finally {
846
+ for (const [name, pool] of handle.topologyPools) {
847
+ try {
848
+ await pool.destroy();
849
+ }
850
+ catch (err) {
851
+ this.logger.warn({ app: handle.name, pool: name, error: err.message }, 'Destroying a topology pool failed — its workers may still run');
852
+ }
407
853
  }
408
- handle.serviceRouter = null;
854
+ handle.topologyPools.clear();
409
855
  }
410
- if (handle.mode === 'bootstrap' && handle.supervisor) {
411
- await handle.supervisor.stop();
856
+ }
857
+ /**
858
+ * Give back every service an app exposed on the daemon's Netron.
859
+ *
860
+ * A registration outlives the process behind it: the daemon keeps answering
861
+ * `queryInterface` with the full method list and every call through the name
862
+ * fails at the socket. `takeOverExisting` rescues a name the next launch
863
+ * re-exposes; one it does not stays advertised for the life of the daemon.
864
+ */
865
+ async releaseExposedServices(handle) {
866
+ if (!handle.serviceRouter)
867
+ return;
868
+ try {
869
+ await handle.serviceRouter.releaseAll();
412
870
  }
413
- else if (handle.childProcess) {
414
- await this.stopChildProcess(handle, force, timeout);
871
+ catch (err) {
872
+ this.logger.warn({ app: handle.name, error: err.message }, 'Releasing exposed services failed — the daemon may still advertise them');
415
873
  }
416
- handle.markStopped();
417
- this.persistState();
418
- this.logger.info({ app: name }, 'App stopped');
874
+ handle.serviceRouter = null;
875
+ }
876
+ async stopApp(name, force = false, timeout = 10_000) {
877
+ const canonical = this.resolveAppName(name) ?? name;
878
+ // Outside the queue on purpose: the operator's intent lands the
879
+ // moment the command arrives, not whenever the queue drains.
880
+ this.clearCoolDown(canonical);
881
+ return this.lifecycleQueue.enqueue(canonical, 'stop', async () => {
882
+ const handle = this.handles.get(canonical);
883
+ if (!handle || handle.status === 'stopped')
884
+ return;
885
+ handle.markStopping();
886
+ // Unexpose topology services from daemon Netron before stopping processes
887
+ await this.releaseExposedServices(handle);
888
+ await this.tearDownProcesses(handle, force, timeout);
889
+ handle.markStopped();
890
+ this.persistState();
891
+ this.logger.info({ app: name }, 'App stopped');
892
+ });
419
893
  }
420
894
  async stopAll(force = false) {
421
895
  if (!this.config)
@@ -460,6 +934,19 @@ export class OrchestratorService extends EventEmitter {
460
934
  this.buildResults.clear();
461
935
  return count;
462
936
  }
937
+ /**
938
+ * Give a supervised app the definition its next start will use — its
939
+ * environment, its command. The running process is not touched; the next
940
+ * `restartApp` starts from this. `false` when nothing by that name runs
941
+ * here. See `AppHandle.redefine`.
942
+ */
943
+ redefineApp(name, entry) {
944
+ const handle = this.handles.get(this.resolveAppName(name) ?? name);
945
+ if (!handle)
946
+ return false;
947
+ handle.redefine(entry);
948
+ return true;
949
+ }
463
950
  /**
464
951
  * Restart an app, coalescing concurrent triggers.
465
952
  *
@@ -496,8 +983,58 @@ export class OrchestratorService extends EventEmitter {
496
983
  async restartAppCoalesced(name, fromExternal) {
497
984
  const canonical = this.resolveAppName(name) ?? name;
498
985
  const handle = this.handles.get(canonical);
499
- if (!handle)
986
+ if (!handle) {
987
+ // A PROJECT-QUALIFIED name with no live handle is not this daemon's to
988
+ // start, and matching it against the daemon's own config starts the
989
+ // WRONG APP.
990
+ //
991
+ // The names collide in practice. omnitron's own `omnitron.config.ts`
992
+ // declares `main`, `payments`, `messaging`, `storage`, `pricing` — the
993
+ // same five a downstream stack uses. On 2026-09-10, after payments died,
994
+ // `omnitron restart acme/dev/payments` matched omnitron's sample entry by
995
+ // its last path segment and launched `apps/payments/src/main.ts` relative
996
+ // to the DAEMON's cwd. It died with ERR_MODULE_NOT_FOUND on a path that
997
+ // exists in neither repository, and the backend stayed down while the
998
+ // command reported that it had restarted it.
999
+ //
1000
+ // Starting the wrong app is worse than refusing: the operator is told
1001
+ // their service is coming back, and the log that says otherwise is the
1002
+ // one they are not reading yet. `startApp` on the daemon already refuses
1003
+ // this exact shape and names the command that works; this is the same
1004
+ // refusal on the same reasoning, at the point where "no live handle" is
1005
+ // already known.
1006
+ const qualified = name.includes('/');
1007
+ const ownProject = this.config?.project;
1008
+ if (qualified && !ownProject) {
1009
+ const [project, stack] = name.split('/');
1010
+ throw new Error(`'${name}' belongs to a registered project, which this daemon restarts through its ` +
1011
+ `stack. Use \`omnitron stack start ${project} ${stack}\`, or restart the app by its ` +
1012
+ `bare name if you mean the daemon's own config.`);
1013
+ }
1014
+ // P1-F — fall through to startApp when there's no live handle but
1015
+ // the ecosystem config declares the app. Operators expect
1016
+ // `omnitron restart foo` to work even when `foo` was never started
1017
+ // (or was stopped, or crashed and was reaped). Previously this
1018
+ // threw "Unknown app", forcing the operator to first run start
1019
+ // explicitly.
1020
+ const effective = effectiveAppName(name);
1021
+ const entry = this.config?.apps.find((a) => a.name === name || a.name === effective || effectiveAppName(a.name) === effective);
1022
+ if (entry) {
1023
+ this.logger.info({ app: name }, 'restartApp: no live handle — falling through to startApp');
1024
+ // Restart under the name the OPERATOR asked for. The match above
1025
+ // accepts a bare entry for a qualified request, and handing it to
1026
+ // `startApp` unchanged re-registers the app bare —
1027
+ // `ensureNamespacedEntry` leaves an entry with no OMNITRON_PROJECT /
1028
+ // OMNITRON_STACK env alone by design. The app then silently leaves its
1029
+ // stack, because `ProjectService.toStackInfo` filters handles on the
1030
+ // `${project}/${stack}/` prefix.
1031
+ const requested = name.includes('/') && !entry.name.includes('/')
1032
+ ? { ...entry, name }
1033
+ : entry;
1034
+ return this.startApp(requested);
1035
+ }
500
1036
  throw new Error(`Unknown app: ${name}`);
1037
+ }
501
1038
  name = canonical;
502
1039
  const state = this.getCoalescerState(name);
503
1040
  if (fromExternal)
@@ -569,6 +1106,26 @@ export class OrchestratorService extends EventEmitter {
569
1106
  * bootstrap-config cache is cleared because changes to `bootstrap.ts`
570
1107
  * (topology definition) need to take effect on the next start.
571
1108
  */
1109
+ /**
1110
+ * Start an app this daemon already supervises, using the entry its handle
1111
+ * carries.
1112
+ *
1113
+ * `startApp(entry)` needs an ecosystem entry, which the daemon's own config
1114
+ * is only one source of — a project stack registers handles this daemon
1115
+ * never declared. `restartAppNow` has always taken the entry off the handle
1116
+ * for exactly that reason; this is the same move for a start, so the name
1117
+ * `list` prints is a name `start` accepts.
1118
+ *
1119
+ * Returns `undefined` when the app is unknown, leaving the caller to decide
1120
+ * what to say about a name that names nothing.
1121
+ */
1122
+ async startKnownApp(name) {
1123
+ const canonical = this.resolveAppName(name) ?? name;
1124
+ const handle = this.handles.get(canonical);
1125
+ if (!handle)
1126
+ return undefined;
1127
+ return this.startApp(handle.entry);
1128
+ }
572
1129
  async restartAppNow(name) {
573
1130
  const handle = this.handles.get(name);
574
1131
  if (!handle)
@@ -602,10 +1159,14 @@ export class OrchestratorService extends EventEmitter {
602
1159
  * Resolve a user-supplied name to a canonical handle key.
603
1160
  *
604
1161
  * Inputs accepted:
605
- * - exact canonical key (`omni/dev/paysys`) → returned as-is
606
- * - bare short name (`paysys`) → returns the canonical key whose
1162
+ * - exact canonical key (`omni/dev/payments`) → returned as-is
1163
+ * - bare short name (`payments`) → returns the canonical key whose
607
1164
  * last `/`-segment matches, IF AND ONLY IF the match is unique.
608
1165
  *
1166
+ * A qualified name that matches no handle exactly resolves to nothing: it
1167
+ * names a deployment this daemon does not have, and the nearest app with
1168
+ * the same last segment is a different one.
1169
+ *
609
1170
  * Returns `undefined` for no match. Throws on ambiguity (multiple
610
1171
  * stacks expose the same short name) so the caller can surface a
611
1172
  * clear error to the operator instead of operating on the wrong
@@ -614,6 +1175,18 @@ export class OrchestratorService extends EventEmitter {
614
1175
  resolveAppName(rawName) {
615
1176
  if (this.handles.has(rawName))
616
1177
  return rawName;
1178
+ // A qualified name is an ADDRESS, and the only handle that satisfies it
1179
+ // is the one with that address. Falling through to the short-name match
1180
+ // answered a question about one deployment with another's data:
1181
+ //
1182
+ // omnitron logs daos/deployed/main → daos/dev/main
1183
+ //
1184
+ // — the node's app asked for, this laptop's app printed, with nothing
1185
+ // said. `startApp` already refuses this for the same reason ("a
1186
+ // PROJECT-QUALIFIED name with no live handle is not this daemon's to
1187
+ // start"); the resolver every other caller goes through did not.
1188
+ if (rawName.includes('/'))
1189
+ return undefined;
617
1190
  const effective = effectiveAppName(rawName);
618
1191
  const matches = [];
619
1192
  for (const key of this.handles.keys()) {
@@ -667,8 +1240,11 @@ export class OrchestratorService extends EventEmitter {
667
1240
  this.logger.info({ app: name, child: childName }, 'Child reloaded successfully');
668
1241
  }
669
1242
  catch (err) {
670
- this.logger.error({ app: name, child: childName, error: err.message }, 'Failed to reload child — aborting zero-downtime reload');
671
- throw new Error(`Zero-downtime reload failed for '${name}' at child '${childName}': ${err.message}`);
1243
+ // `error: err`, not `error: err.message`: the logger aliases `error`
1244
+ // onto pino's error serializer, so the object arrives with its stack
1245
+ // and a string arrives as a string with nothing behind it.
1246
+ this.logger.error({ app: name, child: childName, error: err }, 'Failed to reload child — aborting zero-downtime reload');
1247
+ throw new Error(`Zero-downtime reload failed for '${name}' at child '${childName}': ${err.message}`, { cause: err });
672
1248
  }
673
1249
  }
674
1250
  this.logger.info({ app: name }, 'Zero-downtime reload complete');
@@ -761,6 +1337,14 @@ export class OrchestratorService extends EventEmitter {
761
1337
  getWorkerHandle(processId) {
762
1338
  return this.pm.getWorkerHandle(processId);
763
1339
  }
1340
+ /**
1341
+ * Surface the underlying IProcessInfo for a titan-pm processId
1342
+ * so callers (e.g. the daemon's `inspect` RPC) can read uptime
1343
+ * + status without reaching into the process-manager directly.
1344
+ */
1345
+ getChildProcessInfo(processId) {
1346
+ return this.pm.getProcess(processId);
1347
+ }
764
1348
  // ============================================================================
765
1349
  // Monitoring — async, delegates to PM supervisor for bootstrap mode
766
1350
  // ============================================================================
@@ -793,38 +1377,23 @@ export class OrchestratorService extends EventEmitter {
793
1377
  }
794
1378
  return null;
795
1379
  }
1380
+ /**
1381
+ * `name` is resolved like every other per-app question (`main` finds
1382
+ * `daos/dev/main`); it was compared verbatim, so `omnitron metrics main`
1383
+ * printed «(no data)». An unknown name answers `{}`, and the caller says so.
1384
+ */
796
1385
  async getMetrics(name) {
1386
+ const wanted = name === undefined ? undefined : this.resolveAppName(name);
1387
+ if (name !== undefined && wanted === undefined)
1388
+ return {};
797
1389
  const result = {};
798
1390
  for (const [appName, handle] of this.handles) {
799
- if (name && appName !== name)
1391
+ if (wanted && appName !== wanted)
800
1392
  continue;
801
1393
  if (handle.mode === 'bootstrap' && handle.supervisor) {
802
- const childNames = handle.supervisor.getChildNames();
803
- let cpu = 0, memory = 0, requests = 0, errors = 0;
804
- for (const childName of childNames) {
805
- // Try PM-level RPC metrics first
806
- const m = await handle.supervisor.getChildMetrics(childName);
807
- if (m && (m.cpu > 0 || m.memory > 0)) {
808
- cpu += m.cpu;
809
- memory += m.memory;
810
- requests += m.requests ?? 0;
811
- errors += m.errors ?? 0;
812
- }
813
- else {
814
- // Fallback: OS-level metrics via ps for the child PID
815
- const processId = handle.supervisor.getChildProcessId(childName);
816
- if (processId) {
817
- const workerHandle = this.pm.getWorkerHandle(processId);
818
- const childPid = workerHandle?.pid;
819
- if (childPid) {
820
- const osMetrics = await this.sampleProcessMetrics(childPid);
821
- cpu += osMetrics.cpu;
822
- memory += osMetrics.memory;
823
- }
824
- }
825
- }
826
- }
827
- const aggregated = { cpu, memory, requests, errors };
1394
+ // Same sampler the poller uses, so `omnitron metrics` and `omnitron
1395
+ // list` cannot report different numbers for the same app.
1396
+ const aggregated = await this.sampleAppMetrics(handle);
828
1397
  handle.lastMetrics = aggregated;
829
1398
  result[appName] = aggregated;
830
1399
  }
@@ -835,21 +1404,36 @@ export class OrchestratorService extends EventEmitter {
835
1404
  }
836
1405
  return result;
837
1406
  }
1407
+ /**
1408
+ * Each app's health, asked of every process it runs — its supervisor
1409
+ * children and its worker pools — and combined (`app-health.ts`).
1410
+ *
1411
+ * `name` is resolved like every other per-app question: `main` finds
1412
+ * `daos/dev/main`. It was compared verbatim, so `omnitron health main`
1413
+ * matched nothing and the answer was empty. An unknown name answers `{}`,
1414
+ * and the caller says so.
1415
+ */
838
1416
  async getHealth(name) {
1417
+ const wanted = name === undefined ? undefined : this.resolveAppName(name);
1418
+ if (name !== undefined && wanted === undefined)
1419
+ return {};
839
1420
  const result = {};
840
1421
  for (const [appName, handle] of this.handles) {
841
- if (name && appName !== name)
1422
+ if (wanted && appName !== wanted)
1423
+ continue;
1424
+ if (handle.status !== 'online') {
1425
+ result[appName] = notRunningHealth(handle.status);
842
1426
  continue;
1427
+ }
843
1428
  if (handle.mode === 'bootstrap' && handle.supervisor) {
844
- const childNames = handle.supervisor.getChildNames();
845
- if (childNames.length > 0) {
846
- const health = await handle.supervisor.getChildHealth(childNames[0]);
847
- handle.lastHealth = health;
848
- result[appName] = health;
849
- }
850
- else {
851
- result[appName] = handle.lastHealth;
1429
+ const supervisor = handle.supervisor;
1430
+ const answers = await Promise.all(supervisor.getChildNames().map(async (child) => [child, await supervisor.getChildHealth(child)]));
1431
+ for (const [pool, instance] of handle.topologyPools) {
1432
+ answers.push([pool, poolHealth(instance.metrics)]);
852
1433
  }
1434
+ const health = answers.length > 0 ? combineProcessHealth(answers) : handle.lastHealth;
1435
+ handle.lastHealth = health;
1436
+ result[appName] = health;
853
1437
  }
854
1438
  else {
855
1439
  result[appName] = handle.lastHealth;
@@ -952,7 +1536,7 @@ export class OrchestratorService extends EventEmitter {
952
1536
  else {
953
1537
  this.logger.error({ app: entry.name, code, signal }, 'App exited unexpectedly');
954
1538
  handle.markCrashed();
955
- this.handleClassicCrash(entry, handle, config);
1539
+ this.scheduleRecovery(entry, handle, config);
956
1540
  }
957
1541
  this.persistState();
958
1542
  });
@@ -967,29 +1551,56 @@ export class OrchestratorService extends EventEmitter {
967
1551
  async launchBootstrapMode(entry, handle, config) {
968
1552
  handle.markStarting();
969
1553
  const bootstrapAbsPath = path.resolve(entry.cwd ?? this.cwd, entry.bootstrap);
970
- // Load bootstrap definition to check for process topology and requirements
1554
+ // Load bootstrap definition to check for process topology and requirements.
1555
+ //
1556
+ // In dev, in a process of its own (`freshModuleGraph`): imported here, a
1557
+ // package the definition uses would be the one this daemon first saw, for
1558
+ // as long as it runs (bootstrap-loader.ts has the measurement). What comes
1559
+ // back is data, which is all the daemon reads of it — here, in the build
1560
+ // service and in the stack resolver; the app's own processes load the
1561
+ // definition themselves, functions and all.
971
1562
  let topology;
972
1563
  let definition;
973
1564
  try {
974
- definition = await loadBootstrapConfig(bootstrapAbsPath, { devMode: this.devMode });
1565
+ definition = await loadBootstrapConfig(bootstrapAbsPath, { devMode: this.devMode, freshModuleGraph: true });
975
1566
  topology = definition.processes;
976
1567
  }
977
1568
  catch (err) {
978
- this.logger.warn({ app: entry.name, error: err.message }, 'Could not load bootstrap config for topology — using single-process mode');
1569
+ // Refused, with the reason. This fell back to «single-process mode» at
1570
+ // warn — but every definition declares its topology (IAppDefinition:
1571
+ // `processes` is required), so the fallback started the app in a shape
1572
+ // it was never written for. Measured 2026-09-23 on the master: paysys
1573
+ // and messaging, whose definitions failed to load («Parameter
1574
+ // decorators only work when experimental decorators are enabled»), were
1575
+ // started that way and died on «Dependency 'AuthModule:JWTService' not
1576
+ // found» and a module path that single mode could not resolve — the
1577
+ // operator saw those, and the cause sat one line up at warn.
1578
+ throw new Error(`Could not load ${entry.name}'s definition from ${bootstrapAbsPath}: ${err.message}`, { cause: err });
979
1579
  }
980
- // Populate omnitronConfig from app's config/default.json if not set by bootstrap
981
- if (definition && !definition.omnitronConfig) {
982
- try {
983
- const srcDir = path.dirname(bootstrapAbsPath);
984
- const appRoot = path.resolve(srcDir, '..');
985
- const configPath = path.join(appRoot, 'config', 'default.json');
986
- const content = fs.readFileSync(configPath, 'utf-8');
987
- const json = JSON.parse(content);
988
- if (json.omnitron) {
989
- definition.omnitronConfig = json.omnitron;
1580
+ // The app's `config/default.json`, read on every start and put on a COPY
1581
+ // of the definition. This wrote the section onto the loader's cached
1582
+ // definition «if not set», so the first start's reading was every later
1583
+ // start's — and the logger level beside it, read in the same block, was
1584
+ // never read again at all (`project/declared-config.ts`).
1585
+ if (definition) {
1586
+ const declared = readDeclaredConfig(bootstrapAbsPath);
1587
+ if (declared.malformed) {
1588
+ this.logger.error({ app: entry.name, configPath: declared.configPath, error: declared.malformed }, 'config/default.json does not parse — its `omnitron` section is being ignored');
1589
+ }
1590
+ definition = withDeclaredConfig(definition, declared);
1591
+ // The only place the parent can learn the level the app chose; children
1592
+ // are spawned with it below. Validated rather than trusted: this is a
1593
+ // hand-edited JSON file, and a level pino does not know ("verbose", a
1594
+ // typo) would be handed straight to a child's logger constructor.
1595
+ const level = declared.loggerLevel;
1596
+ if (typeof level === 'string') {
1597
+ if (PINO_LEVELS.has(level)) {
1598
+ handle.appLogLevel = level;
1599
+ }
1600
+ else {
1601
+ this.logger.warn({ app: entry.name, configPath: declared.configPath, level }, 'config/default.json declares a logger level this runtime does not know — children keep the default');
990
1602
  }
991
1603
  }
992
- catch { /* config file missing — skip */ }
993
1604
  }
994
1605
  // Resolve infrastructure config and inject as env vars.
995
1606
  // Skip if already injected by project.service.ts (stack flow sets OMNITRON_STACK).
@@ -1102,9 +1713,12 @@ export class OrchestratorService extends EventEmitter {
1102
1713
  // fail with a cryptic "Cannot find module" downstream. Surface
1103
1714
  // the build error directly so the operator sees the actual
1104
1715
  // problem on `omnitron start/restart`.
1105
- this.logger.error({ app: entry.name, error: err.message }, 'esbuild build failed — app cannot start without bundled output');
1716
+ // esbuild's error carries an `errors[]` with file, line and column.
1717
+ // `.message` is the summary; the locations are what an operator needs
1718
+ // to fix the build, and they exist only on the original.
1719
+ this.logger.error({ app: entry.name, error: err }, 'esbuild build failed — app cannot start without bundled output');
1106
1720
  throw new Error(`Cannot launch '${entry.name}' in dev mode: esbuild build failed (${err.message}). ` +
1107
- `Fix the build error and retry, or run with bundled output present in apps/${entry.name}/.omnitron-build/.`);
1721
+ `Fix the build error and retry, or run with bundled output present in apps/${entry.name}/.omnitron-build/.`, { cause: err });
1108
1722
  }
1109
1723
  }
1110
1724
  const buildResult = this.buildResults.get(entry.name);
@@ -1157,14 +1771,20 @@ export class OrchestratorService extends EventEmitter {
1157
1771
  size: instances,
1158
1772
  strategy: PoolStrategy.POWER_OF_TWO,
1159
1773
  metrics: true,
1160
- replaceUnhealthy: true,
1161
- requestTimeout: 120_000,
1774
+ // `replaceUnhealthy` is deliberately not set: titan-pm declares it
1775
+ // and reads it nowhere. An unhealthy worker is replaced by the
1776
+ // health path regardless, so `true` was a no-op that happened to
1777
+ // match the behaviour — and `false` would have been a no-op that did
1778
+ // not, leaving an operator believing they had turned replacement off.
1779
+ // What actually governs it is `healthCheck.unhealthyThreshold`.
1780
+ requestTimeout: CHILD_REQUEST_TIMEOUT,
1162
1781
  maxQueueSize: 200,
1163
1782
  spawnOptions: {
1164
1783
  name: `${entry.name}/${procEntry.name}`,
1165
1784
  version: '1.0.0',
1166
1785
  allMethodsPublic: true,
1167
- startupTimeout: procEntry.startupTimeout ?? 60_000,
1786
+ ...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
1787
+ startupTimeout: procEntry.startupTimeout ?? entry.startupTimeout ?? DEFAULT_CHILD_STARTUP_TIMEOUT,
1168
1788
  ...(entry.env && { env: entry.env }),
1169
1789
  ...(entry.cwd && { cwd: entry.cwd }),
1170
1790
  dependencies: {
@@ -1187,25 +1807,31 @@ export class OrchestratorService extends EventEmitter {
1187
1807
  enabled: procEntry.health?.enabled !== false,
1188
1808
  interval: procEntry.health?.interval ?? config.monitoring.healthCheck.interval,
1189
1809
  },
1810
+ // `scaling.strategy` is what opts a pool into autoscaling; anything
1811
+ // else — including the default of not declaring it — means the pool
1812
+ // stays at the declared `instances`, which is what 'fixed' says on
1813
+ // the tin.
1814
+ //
1815
+ // Both arms of this ternary used to set `enabled: true`, differing
1816
+ // only in where the numbers came from. The setting existed, the
1817
+ // mechanism existed, and nothing connected them: every pool in the
1818
+ // platform was autoscaled whether or not it asked to be. Live effect
1819
+ // on two pools declared `instances: 2` — grown to their cap of eight
1820
+ // and held there, sixteen processes where four were configured.
1821
+ //
1822
+ // `min` follows the declared instance count rather than 1: a pool
1823
+ // asked for two workers should not be scaled below two.
1190
1824
  autoScale: procEntry.scaling?.strategy === 'auto'
1191
1825
  ? {
1192
1826
  enabled: true,
1193
- min: 1,
1827
+ min: instances,
1194
1828
  max: procEntry.scaling?.maxInstances ?? instances * 4,
1195
1829
  targetCPU: procEntry.scaling?.targetCPU ?? 70,
1196
1830
  targetMemory: procEntry.scaling?.targetMemory ?? 80,
1197
1831
  queueThreshold: procEntry.scaling?.queueThreshold ?? 10,
1198
1832
  cooldownPeriod: procEntry.scaling?.cooldownPeriod ?? 30_000,
1199
1833
  }
1200
- : {
1201
- enabled: true,
1202
- min: 1,
1203
- max: instances * 4,
1204
- targetCPU: 70,
1205
- targetMemory: 80,
1206
- queueThreshold: 10,
1207
- cooldownPeriod: 30_000,
1208
- },
1834
+ : { enabled: false, min: instances, max: instances },
1209
1835
  };
1210
1836
  this.logger.info({ app: entry.name, process: procEntry.name, instances }, 'Creating topology process pool');
1211
1837
  // pm.pool() returns a Proxy wrapping ProcessPool — execute() dispatches to pool workers
@@ -1217,24 +1843,50 @@ export class OrchestratorService extends EventEmitter {
1217
1843
  // which returns null for pool workers (they aren't supervisor
1218
1844
  // children) and the entry is mis-reported as `status: 'stopped'`.
1219
1845
  handle.topologyPools.set(procEntry.name, pool);
1846
+ this.attachPoolLogCapture(entry.name, procEntry.name, handle, pool);
1220
1847
  // Auto-discover @Service metadata from pool workers and expose via ServiceRouter.
1221
1848
  // No manual `exports` needed — @Service decorator IS the source of truth.
1222
1849
  // Requires explicit `topology: { expose: true }` — opt-in for security.
1223
1850
  if (procEntry.topology?.expose === true) {
1224
1851
  try {
1852
+ // A pool worker reports ready before its Application has started, and
1853
+ // `@Service` classes are exposed to Netron during start — so asking
1854
+ // once, or once more 500ms later, asks a worker that is still booting.
1855
+ // What it answers then is an empty list, or a service whose method
1856
+ // list has not been filled in yet, and both used to end the same way:
1857
+ // nothing registered on the daemon, the consumer process starting
1858
+ // seconds later, querying, and being told the service does not exist.
1859
+ // The consumer resolves its topology proxy exactly once, so that
1860
+ // answer disabled the feature for the life of the process — in the
1861
+ // case that surfaced this, every OHLCV candle aggregation.
1862
+ //
1863
+ // Poll instead, and treat "a service with no methods" as not ready
1864
+ // rather than as a service, because an empty method list produces a
1865
+ // proxy that can do nothing.
1866
+ const usable = (list) => Array.isArray(list) && list.length > 0 && list.some((svc) => svc.methods?.length > 0);
1225
1867
  let services = await pool.execute('getExposedServices');
1226
- // Defensive: if services list is empty, the worker's Application may still
1227
- // be registering services. Retry once after a short delay.
1228
- if (!services || services.length === 0) {
1229
- await new Promise((r) => setTimeout(r, 500));
1868
+ const deadline = Date.now() + TOPOLOGY_EXPOSE_TIMEOUT_MS;
1869
+ while (!usable(services) && Date.now() < deadline) {
1870
+ await new Promise((r) => setTimeout(r, TOPOLOGY_EXPOSE_POLL_MS));
1230
1871
  services = await pool.execute('getExposedServices');
1231
1872
  }
1232
- if (!services || services.length === 0) {
1233
- this.logger.warn({ app: entry.name, process: procEntry.name }, 'No @Service found on pool worker — topology.expose has no effect');
1873
+ if (!usable(services)) {
1874
+ this.logger.warn({
1875
+ app: entry.name,
1876
+ process: procEntry.name,
1877
+ waitedMs: TOPOLOGY_EXPOSE_TIMEOUT_MS,
1878
+ reported: services?.map((svc) => `${svc.name}(${svc.methods?.length ?? 0})`) ?? [],
1879
+ }, 'No usable @Service found on pool worker — topology.expose has no effect, and any ' +
1880
+ 'process declaring topology.access on it will start without a proxy');
1234
1881
  }
1235
- for (const svc of services) {
1236
- if (svc.methods.length === 0)
1882
+ for (const svc of services ?? []) {
1883
+ if (svc.methods.length === 0) {
1884
+ // Silently skipped before. A consumer then failed to find a
1885
+ // service its own declaration named, with nothing anywhere
1886
+ // connecting the two facts.
1887
+ this.logger.warn({ app: entry.name, process: procEntry.name, service: svc.name }, 'Pool service exposes no methods — not registered on the daemon');
1237
1888
  continue;
1889
+ }
1238
1890
  await serviceRouter.exposePoolService(procEntry.name, svc.name, svc.version ?? '1.0.0', pool, svc.methods);
1239
1891
  this.logger.info({ app: entry.name, process: procEntry.name, service: svc.name, methods: svc.methods.length }, 'Auto-discovered and exposed pool service on daemon Netron');
1240
1892
  }
@@ -1253,7 +1905,10 @@ export class OrchestratorService extends EventEmitter {
1253
1905
  name: `${entry.name}/${procEntry.name}`,
1254
1906
  version: '1.0.0',
1255
1907
  allMethodsPublic: true,
1256
- startupTimeout: procEntry.startupTimeout ?? entry.startupTimeout ?? 30_000,
1908
+ // Without this the child's client takes netron's 5000 ms; the pool
1909
+ // path beside it has always stated one. See CHILD_REQUEST_TIMEOUT.
1910
+ requestTimeout: CHILD_REQUEST_TIMEOUT,
1911
+ startupTimeout: procEntry.startupTimeout ?? entry.startupTimeout ?? DEFAULT_CHILD_STARTUP_TIMEOUT,
1257
1912
  ...(entry.env && { env: entry.env }),
1258
1913
  ...(entry.cwd && { cwd: entry.cwd }),
1259
1914
  dependencies: {
@@ -1271,6 +1926,7 @@ export class OrchestratorService extends EventEmitter {
1271
1926
  timeout: procEntry.health?.timeout ?? config.monitoring.healthCheck.timeout,
1272
1927
  },
1273
1928
  observability: { metrics: true },
1929
+ ...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
1274
1930
  };
1275
1931
  children.push({
1276
1932
  name: `${entry.name}/${procEntry.name}`,
@@ -1295,18 +1951,11 @@ export class OrchestratorService extends EventEmitter {
1295
1951
  };
1296
1952
  // Create supervisor WITHOUT starting — wire events first to avoid race condition
1297
1953
  const supervisor = this.pm.createSupervisor(supervisorConfig);
1298
- supervisor.on('child:started', (childName) => {
1299
- const processId = supervisor.getChildProcessId(childName);
1300
- if (processId) {
1301
- const workerHandle = this.pm.getWorkerHandle(processId);
1302
- const info = this.pm.getProcess(processId);
1303
- const childPid = workerHandle?.pid ?? info?.pid;
1304
- handle.markOnline(childPid ?? process.pid);
1305
- }
1306
- this.persistState();
1307
- this.startPostOnlineCooldown(entry.name);
1308
- this.emit('app:online', entry.name);
1309
- });
1954
+ // The app's pid is its server's: the single process with a transport, the
1955
+ // one `handle.port` names. A topology with none has no pid to give.
1956
+ const server = singleEntries.find((p) => p.transports?.http) ?? singleEntries.find((p) => p.transports) ?? null;
1957
+ handle.declareChildren(children.map((child) => child.name), server ? `${entry.name}/${server.name}` : null);
1958
+ this.onChildStarted(entry, handle, supervisor);
1310
1959
  // Set supervisor on handle BEFORE start so attachLogCapture can access it
1311
1960
  handle.supervisor = supervisor;
1312
1961
  handle.serviceRouter = serviceRouter;
@@ -1315,6 +1964,73 @@ export class OrchestratorService extends EventEmitter {
1315
1964
  this.wireSupervisorEvents(entry, handle, supervisor);
1316
1965
  // Now start — events will be captured correctly
1317
1966
  await supervisor.start();
1967
+ // Step 4: register the single processes that declare `topology.expose`.
1968
+ //
1969
+ // This was a pool-only feature, ignored in silence everywhere else: a
1970
+ // process declaring `expose: true` got no registration and no warning,
1971
+ // and every consumer naming it in `topology.access` started without a
1972
+ // proxy. Measured on priceverse, whose `collector` is a single process
1973
+ // holding the only objects that know whether the exchange WebSockets are
1974
+ // up — the server process could not ask, and answered `ready: down —
1975
+ // exchanges unavailable` while three were connected.
1976
+ //
1977
+ // After `supervisor.start()` because there is no child to ask before it.
1978
+ // A consumer among these children may therefore start before the service
1979
+ // is registered; consumers re-resolve their proxy on use for exactly that
1980
+ // reason (see priceverse's `OhlcvSchedulerService`).
1981
+ await this.exposeChildTopologyServices(entry, supervisor, serviceRouter, singleEntries);
1982
+ }
1983
+ /**
1984
+ * Register each single process's `@Service` classes on the daemon.
1985
+ *
1986
+ * Mirrors the pool path above, including its poll: a child reports ready
1987
+ * before its Application has started, and `@Service` classes are exposed to
1988
+ * Netron during start, so asking once asks a process that is still booting
1989
+ * and gets an empty list back.
1990
+ */
1991
+ async exposeChildTopologyServices(entry, supervisor, serviceRouter, singleEntries) {
1992
+ for (const procEntry of singleEntries) {
1993
+ if (procEntry.topology?.expose !== true)
1994
+ continue;
1995
+ const childName = `${entry.name}/${procEntry.name}`;
1996
+ const getProxy = () => supervisor.getChildProxy(childName);
1997
+ try {
1998
+ const usable = (list) => Array.isArray(list) && list.length > 0 && list.some((svc) => svc.methods?.length > 0);
1999
+ const ask = async () => {
2000
+ const child = getProxy();
2001
+ if (!child?.getExposedServices)
2002
+ return undefined;
2003
+ return (await child.getExposedServices());
2004
+ };
2005
+ let services = await ask();
2006
+ const deadline = Date.now() + TOPOLOGY_EXPOSE_TIMEOUT_MS;
2007
+ while (!usable(services) && Date.now() < deadline) {
2008
+ await new Promise((r) => setTimeout(r, TOPOLOGY_EXPOSE_POLL_MS));
2009
+ services = await ask();
2010
+ }
2011
+ if (!usable(services)) {
2012
+ this.logger.warn({
2013
+ app: entry.name,
2014
+ process: procEntry.name,
2015
+ waitedMs: TOPOLOGY_EXPOSE_TIMEOUT_MS,
2016
+ reported: services?.map((svc) => `${svc.name}(${svc.methods?.length ?? 0})`) ?? [],
2017
+ }, 'No usable @Service found on this process — topology.expose has no effect, and any ' +
2018
+ 'process declaring topology.access on it will start without a proxy');
2019
+ continue;
2020
+ }
2021
+ for (const svc of services ?? []) {
2022
+ if (svc.methods.length === 0) {
2023
+ this.logger.warn({ app: entry.name, process: procEntry.name, service: svc.name }, 'Process service exposes no methods — not registered on the daemon');
2024
+ continue;
2025
+ }
2026
+ await serviceRouter.exposeChildService(procEntry.name, svc.name, svc.version ?? '1.0.0', getProxy, svc.methods);
2027
+ this.logger.info({ app: entry.name, process: procEntry.name, service: svc.name, methods: svc.methods.length }, 'Auto-discovered and exposed process service on daemon Netron');
2028
+ }
2029
+ }
2030
+ catch (err) {
2031
+ this.logger.error({ app: entry.name, process: procEntry.name, error: err.message }, "Failed to auto-discover/expose this process's services");
2032
+ }
2033
+ }
1318
2034
  }
1319
2035
  /**
1320
2036
  * Launch single-process bootstrap app (no topology — original behavior).
@@ -1330,11 +2046,22 @@ export class OrchestratorService extends EventEmitter {
1330
2046
  const spawnOpts = {
1331
2047
  name: entry.name,
1332
2048
  version: '1.0.0',
2049
+ // As above: a single child's client otherwise runs on netron's default.
2050
+ requestTimeout: CHILD_REQUEST_TIMEOUT,
1333
2051
  ...(entry.env && { env: entry.env }),
1334
2052
  dependencies: {
1335
2053
  bootstrapPath: bootstrapAbsPath,
1336
2054
  ...(bundledModulePath && { bundledModulePath }),
1337
2055
  },
2056
+ // Omitted when unset, which delegates the budget to titan-pm —
2057
+ // `process-spawner.ts`: `options.startupTimeout ?? resources?.timeout ??
2058
+ // 30_000`. Worth naming, because it is otherwise invisible from here:
2059
+ // an APP that declares nothing gets 30s while its CHILD processes get
2060
+ // `DEFAULT_CHILD_STARTUP_TIMEOUT`, 60s. Backwards if anything, since the
2061
+ // app usually does the heavier work — and the downstream project's own
2062
+ // config carries a note about a 30s default that kept a whole stack from
2063
+ // starting. Left as delegation rather than changed here: raising it is a
2064
+ // decision about titan-pm's contract, not about this call site.
1338
2065
  ...(entry.startupTimeout != null && { startupTimeout: entry.startupTimeout }),
1339
2066
  health: {
1340
2067
  enabled: true,
@@ -1342,6 +2069,7 @@ export class OrchestratorService extends EventEmitter {
1342
2069
  timeout: config.monitoring.healthCheck.timeout,
1343
2070
  },
1344
2071
  observability: { metrics: true },
2072
+ ...(handle.appLogLevel && { logLevel: handle.appLogLevel }),
1345
2073
  };
1346
2074
  const childConfig = {
1347
2075
  name: entry.name,
@@ -1354,7 +2082,8 @@ export class OrchestratorService extends EventEmitter {
1354
2082
  size: instances,
1355
2083
  strategy: PoolStrategy.ROUND_ROBIN,
1356
2084
  metrics: true,
1357
- replaceUnhealthy: true,
2085
+ // See the note on the other pool: `replaceUnhealthy` is declared by
2086
+ // titan-pm and read by nothing.
1358
2087
  healthCheck: {
1359
2088
  enabled: true,
1360
2089
  interval: config.monitoring.healthCheck.interval,
@@ -1392,20 +2121,10 @@ export class OrchestratorService extends EventEmitter {
1392
2121
  };
1393
2122
  // Create supervisor WITHOUT starting — wire events first to avoid race condition
1394
2123
  const supervisor = this.pm.createSupervisor(supervisorConfig);
1395
- // Wire supervisor events BEFORE start so we catch the initial child:started
1396
- supervisor.on('child:started', (childName) => {
1397
- const processId = supervisor.getChildProcessId(childName);
1398
- if (processId) {
1399
- // Prefer WorkerHandle.pid (actual child OS PID), fallback to IProcessInfo.pid
1400
- const workerHandle = this.pm.getWorkerHandle(processId);
1401
- const info = this.pm.getProcess(processId);
1402
- const childPid = workerHandle?.pid ?? info?.pid;
1403
- handle.markOnline(childPid ?? process.pid);
1404
- }
1405
- this.persistState();
1406
- this.startPostOnlineCooldown(entry.name);
1407
- this.emit('app:online', entry.name);
1408
- });
2124
+ // Wire supervisor events BEFORE start so we catch the initial child:started.
2125
+ // One child, and it is the whole app — so it is the server too.
2126
+ handle.declareChildren([childConfig.name], childConfig.name);
2127
+ this.onChildStarted(entry, handle, supervisor);
1409
2128
  // Set supervisor on handle BEFORE start so attachLogCapture can access it
1410
2129
  handle.supervisor = supervisor;
1411
2130
  handle.instanceCount = instances;
@@ -1413,14 +2132,50 @@ export class OrchestratorService extends EventEmitter {
1413
2132
  // Now start — events will be captured correctly
1414
2133
  await supervisor.start();
1415
2134
  }
2135
+ /**
2136
+ * A child's start, for both launch paths: the app is online when the start
2137
+ * completes it, and not before (`AppHandle.childStarted`).
2138
+ *
2139
+ * Each path had its own copy of this handler, and both marked the app
2140
+ * online — with that child's pid — on every child's start. `app:online`
2141
+ * went out once per child with it: three STARTED events for one boot of a
2142
+ * three-process app, the first two about an app still coming up.
2143
+ */
2144
+ onChildStarted(entry, handle, supervisor) {
2145
+ supervisor.on('child:started', (childName) => {
2146
+ // A pool child's proxy answers every property with a function —
2147
+ // `__processId` included — so only a string is a process id. The old
2148
+ // handler took that function for one, found no worker behind it, and
2149
+ // gave such an app the DAEMON's pid.
2150
+ const processId = supervisor.getChildProcessId(childName);
2151
+ const pid = typeof processId === 'string'
2152
+ ? (this.pm.getWorkerHandle(processId)?.pid ?? this.pm.getProcess(processId)?.pid ?? null)
2153
+ : null;
2154
+ const whole = handle.childStarted(childName, pid);
2155
+ this.persistState();
2156
+ if (!whole)
2157
+ return;
2158
+ this.startPostOnlineCooldown(entry.name);
2159
+ this.emit('app:online', entry.name);
2160
+ });
2161
+ }
1416
2162
  /**
1417
2163
  * Wire common supervisor events (crash, restart, escalate) and log capture.
1418
2164
  */
1419
2165
  wireSupervisorEvents(entry, handle, supervisor) {
1420
2166
  supervisor.on('child:started', (childName) => {
1421
2167
  this.attachLogCapture(entry.name, childName, handle);
2168
+ this.attachExitCapture(entry.name, childName, handle);
2169
+ // The app re-entered a live state — drop the previous crash
2170
+ // context so a future `omnitron inspect` shows the *next*
2171
+ // crash, not the last one.
2172
+ handle.clearLastExit();
1422
2173
  });
2174
+ // A child that is down leaves the app partial until it is back — a
2175
+ // sibling's restart must not bring the app `online` around it.
2176
+ supervisor.on('child:stopped', (childName) => handle.childGone(childName));
1423
2177
  supervisor.on('child:crash', (childName, error) => {
2178
+ handle.childGone(childName);
1424
2179
  // If app never reached 'online', mark as errored (not crashed — it never ran)
1425
2180
  if (handle.status === 'starting') {
1426
2181
  handle.markErrored();
@@ -1428,10 +2183,94 @@ export class OrchestratorService extends EventEmitter {
1428
2183
  else {
1429
2184
  handle.markCrashed();
1430
2185
  }
2186
+ // Stash the supervisor-provided error message onto whatever
2187
+ // exit context the workerHandle.onExit hook already wrote. If
2188
+ // the exit event hasn't fired yet (race) we still need to
2189
+ // persist *something* — record a synthetic entry that the
2190
+ // exit hook below will overwrite when it does fire.
2191
+ if (handle.lastExit) {
2192
+ handle.lastExit.message = error.message;
2193
+ }
2194
+ else {
2195
+ handle.recordExit({ code: null, signal: null, expected: false, message: error.message });
2196
+ }
1431
2197
  this.logger.error({ app: entry.name, child: childName, error: error.message }, 'Process crashed');
1432
2198
  this.persistState();
1433
2199
  this.emit('app:crash', entry.name, error);
1434
2200
  });
2201
+ // T#60 — startup failures (child throws/rejects in waitForReady
2202
+ // before ever reporting ready) come through this event, NOT
2203
+ // child:crash. Without a handler the orchestrator just sees
2204
+ // status='errored' with no clue what went wrong. titan-pm
2205
+ // already stuffs the captured stderr into `error.details.stderr`
2206
+ // so we surface it on `lastExit.stderrTail`.
2207
+ supervisor.on('child:start-failed', (childName, error) => {
2208
+ const details = error.details;
2209
+ const stderrTail = typeof details?.stderr === 'string'
2210
+ ? details.stderr.split('\n').filter((l) => l.length > 0).slice(-80)
2211
+ : [];
2212
+ for (const line of stderrTail)
2213
+ handle.appendStderr(line);
2214
+ // A child that never came up because WE stopped the app is not a
2215
+ // failure, and reporting it as one is worse than saying nothing: the
2216
+ // message titan-pm builds for an aborted start says the process "did not
2217
+ // choose to exit, so its own logs will not explain this" — true, and it
2218
+ // sends the reader hunting a fault in an application that was doing
2219
+ // nothing wrong. Every `stack stop` issued while anything was still
2220
+ // booting wrote one of these per child, at error level, into the app's
2221
+ // own log.
2222
+ //
2223
+ // `'stopping'` only — NOT `'stopped'`, which is also an AppHandle's
2224
+ // initial value, so accepting it would swallow the first genuine failure
2225
+ // of any app whose handle had not yet been marked starting.
2226
+ const expected = handle.status === 'stopping' ||
2227
+ error.stoppedDuringStartup === true;
2228
+ if (expected) {
2229
+ handle.recordExit({
2230
+ code: null,
2231
+ signal: null,
2232
+ expected: true,
2233
+ message: error.message,
2234
+ });
2235
+ this.logger.info({ app: entry.name, child: childName, err: error.message }, 'Process stopped before it finished starting');
2236
+ this.persistState();
2237
+ return;
2238
+ }
2239
+ // Put what the child printed into the app's LOG FILE, not only into the
2240
+ // in-memory ring `omnitron inspect` reads.
2241
+ //
2242
+ // A child that dies during startup never produces a WorkerHandle at all
2243
+ // — titan-pm registers one only after `waitForReady` resolves — so there
2244
+ // is no `onLog` to subscribe to here and nothing to replay. What does
2245
+ // exist is the output `waitForReady` captured while waiting, which it
2246
+ // attaches to the error. Feeding that to the log handlers is the only
2247
+ // way a failed boot reaches disk, and a failed boot is the case where
2248
+ // reading it matters most: an application that dies before its logger
2249
+ // exists has said everything it is ever going to say on these streams.
2250
+ // The failure itself first. A child can fail without printing a word —
2251
+ // it reports the error over the IPC channel and exits, which is what a
2252
+ // module that throws on import does — so the streams below are often
2253
+ // empty and the only account of what happened is on the error.
2254
+ this.persistChildFailure(entry.name, childName, error);
2255
+ this.persistChildOutput(entry.name, childName, details?.stdout, 'stdout');
2256
+ this.persistChildOutput(entry.name, childName, details?.stderr, 'stderr');
2257
+ handle.recordExit({
2258
+ code: null,
2259
+ signal: null,
2260
+ expected: false,
2261
+ message: error.message,
2262
+ });
2263
+ // The supervisor gives up on a non-critical child whose start failed (a
2264
+ // critical one throws, and `startApp` says so). That went unseen while
2265
+ // the first child's start made the app `online`; now the app would wait
2266
+ // for a child that is not coming, in `starting` — which `omnitron start`
2267
+ // takes for a start in progress and leaves alone. A declared process
2268
+ // that did not start is an app that did not start.
2269
+ if (handle.status === 'starting')
2270
+ handle.markErrored();
2271
+ this.logger.error({ app: entry.name, child: childName, err: error.message }, 'Process failed to start');
2272
+ this.persistState();
2273
+ });
1435
2274
  supervisor.on('child:restart', (childName, attempt) => {
1436
2275
  handle.restarts = attempt;
1437
2276
  this.logger.info({ app: entry.name, child: childName, attempt }, 'Process restarting');
@@ -1458,9 +2297,45 @@ export class OrchestratorService extends EventEmitter {
1458
2297
  const workerHandle = this.pm.getWorkerHandle(processId);
1459
2298
  if (!workerHandle?.onLog)
1460
2299
  return;
1461
- workerHandle.onLog((line, _stream) => {
2300
+ workerHandle.onLog(this.childLogForwarder(appName, handle));
2301
+ }
2302
+ /**
2303
+ * Attach log capture to the workers of a topology POOL.
2304
+ *
2305
+ * `attachLogCapture` above is driven by `child:started`, which only
2306
+ * supervisor children emit — a pool worker is not one, so nothing ever
2307
+ * subscribed to its output. Everything those workers print went nowhere:
2308
+ * the OHLCV aggregator and the image transformer have been mute since they
2309
+ * were written, and since titan-pm started holding pre-capture output it was
2310
+ * also a megabyte per worker retained for a reader that never arrived.
2311
+ *
2312
+ * Attached once, at pool creation. A worker replaced later by autoscaling or
2313
+ * a health restart is not covered; these pools are fixed-size, and the
2314
+ * alternative is an event this package does not expose yet.
2315
+ */
2316
+ attachPoolLogCapture(appName, processName, handle, pool) {
2317
+ const forward = this.childLogForwarder(appName, handle);
2318
+ let attached = 0;
2319
+ for (const workerId of pool.getWorkerIds()) {
2320
+ const workerHandle = pool.getWorkerHandle(workerId);
2321
+ if (!workerHandle?.onLog)
2322
+ continue;
2323
+ workerHandle.onLog(forward);
2324
+ attached++;
2325
+ }
2326
+ this.logger.debug({ app: appName, process: processName, workers: attached }, 'Pool log capture attached');
2327
+ }
2328
+ /** The one place a child's line becomes an app's line. */
2329
+ childLogForwarder(appName, handle) {
2330
+ return (line, stream) => {
1462
2331
  // In-memory ring buffer (for live CLI tailing)
1463
2332
  handle.appendLog(line);
2333
+ // Separate stderr ring — feeds `lastExit.stderrTail` on death.
2334
+ // Without this, the operator running `omnitron inspect` after
2335
+ // a crash sees no hint at WHY (the in-flight pipe is gone).
2336
+ if (stream === 'stderr') {
2337
+ handle.appendStderr(line);
2338
+ }
1464
2339
  // Persist to disk via registered log handlers (LogManager)
1465
2340
  for (const handler of this.appLogHandlers) {
1466
2341
  try {
@@ -1470,6 +2345,108 @@ export class OrchestratorService extends EventEmitter {
1470
2345
  // Log handler failure must not break the app
1471
2346
  }
1472
2347
  }
2348
+ };
2349
+ }
2350
+ /**
2351
+ * Write the failure a child reported before it could start to the app's log
2352
+ * handlers, as one structured line.
2353
+ *
2354
+ * `error.childStack` is the stack from inside the child when it managed to
2355
+ * send one — for a module that throws on import that is the entire
2356
+ * diagnosis, and it exists nowhere else once the process is gone.
2357
+ */
2358
+ persistChildFailure(appName, childName, error) {
2359
+ const childStack = error.childStack;
2360
+ const entry = JSON.stringify({
2361
+ level: 50,
2362
+ time: new Date().toISOString(),
2363
+ processName: qualifyChildName(appName, childName),
2364
+ event: 'process.start_failed',
2365
+ err: { message: error.message, ...(childStack ? { stack: childStack } : {}) },
2366
+ msg: `Process failed to start: ${error.message}`,
2367
+ });
2368
+ for (const handler of this.appLogHandlers) {
2369
+ try {
2370
+ handler(appName, entry);
2371
+ }
2372
+ catch {
2373
+ // A log handler must not break failure reporting.
2374
+ }
2375
+ }
2376
+ }
2377
+ /**
2378
+ * Write a dead child's captured output to the app's log handlers.
2379
+ *
2380
+ * Emitted as pino-shaped JSON so it lands in the same file, at the same
2381
+ * level, as everything else that app ever logged — a reader tailing
2382
+ * `app.log` after a failed start finds the reason there rather than having
2383
+ * to know that this particular failure hides in a different place.
2384
+ */
2385
+ persistChildOutput(appName, childName, output, stream) {
2386
+ if (!output)
2387
+ return;
2388
+ const time = new Date().toISOString();
2389
+ for (const line of output.split('\n')) {
2390
+ if (!line.trim())
2391
+ continue;
2392
+ // Already-structured lines pass through untouched; the child may well
2393
+ // have got far enough to produce them.
2394
+ const entry = line.trimStart().startsWith('{')
2395
+ ? line
2396
+ : JSON.stringify({
2397
+ // stderr is not error. A boot script's own progress markers go
2398
+ // there, and a failing start emits a run of them — classifying
2399
+ // every one at 50 filled the log with errors that are nothing of
2400
+ // the sort, next to the single line that IS the failure. The
2401
+ // failure is reported at 50 by `persistChildFailure`; this is the
2402
+ // context around it, and `stream` keeps it greppable.
2403
+ level: stream === 'stderr' ? 40 : 30,
2404
+ time,
2405
+ processName: qualifyChildName(appName, childName),
2406
+ stream,
2407
+ msg: line,
2408
+ });
2409
+ for (const handler of this.appLogHandlers) {
2410
+ try {
2411
+ handler(appName, entry);
2412
+ }
2413
+ catch {
2414
+ // A log handler must not break failure reporting.
2415
+ }
2416
+ }
2417
+ }
2418
+ }
2419
+ /**
2420
+ * Subscribe to the WorkerHandle's `exit` event so the orchestrator
2421
+ * keeps the exit code / signal / "expected" flag past the
2422
+ * WorkerHandle's GC. Pre-fix the supervisor's `child:crash` event
2423
+ * fired with an Error but never surfaced the underlying
2424
+ * code/signal — operators had no way to tell SIGKILL from a
2425
+ * thrown TypeError. The captured info lands on
2426
+ * `handle.lastExit` and is surfaced by `omnitron inspect`.
2427
+ */
2428
+ attachExitCapture(appName, childName, handle) {
2429
+ const supervisor = handle.supervisor;
2430
+ if (!supervisor)
2431
+ return;
2432
+ const processId = supervisor.getChildProcessId(childName);
2433
+ if (!processId)
2434
+ return;
2435
+ const workerHandle = this.pm.getWorkerHandle(processId);
2436
+ if (!workerHandle?.onExit)
2437
+ return;
2438
+ workerHandle.onExit((info) => {
2439
+ // Preserve the supervisor's error message if `child:crash`
2440
+ // already fired before us. recordExit overwrites the synthetic
2441
+ // entry with the authoritative code/signal.
2442
+ const message = handle.lastExit?.message;
2443
+ handle.recordExit({
2444
+ code: info.code,
2445
+ signal: info.signal,
2446
+ expected: info.expected,
2447
+ ...(message !== undefined && { message }),
2448
+ });
2449
+ this.logger.debug({ app: appName, child: childName, code: info.code, signal: info.signal, expected: info.expected }, 'Captured worker exit context');
1473
2450
  });
1474
2451
  }
1475
2452
  // ============================================================================
@@ -1479,30 +2456,58 @@ export class OrchestratorService extends EventEmitter {
1479
2456
  const child = handle.childProcess;
1480
2457
  if (!child || !child.pid)
1481
2458
  return;
2459
+ const pid = child.pid;
2460
+ // P0-D — Signal the whole process group, not just the direct
2461
+ // child. classic-launcher spawns with `detached: true` so the
2462
+ // child becomes group leader (pgid === pid). `process.kill(-pid)`
2463
+ // delivers the signal to every process in the group, reaping
2464
+ // grandchildren (ffmpeg, docker, worker pools …) that ignore the
2465
+ // direct-child kill. Falls back to `child.kill` when the negative-
2466
+ // pid call fails (test environments, restricted permissions, the
2467
+ // child wasn't started detached — bootstrap-mode currently isn't).
2468
+ const sendSignal = (sig) => {
2469
+ try {
2470
+ process.kill(-pid, sig);
2471
+ return;
2472
+ }
2473
+ catch (err) {
2474
+ const code = err.code;
2475
+ if (code !== 'ESRCH' && code !== 'EPERM') {
2476
+ // ESRCH/EPERM = group already gone or not a group leader.
2477
+ // Anything else (EINVAL etc.) → log via best-effort.
2478
+ this.logger.debug({ app: handle.name, pid, sig, code }, 'process.kill(-pid) fallback');
2479
+ }
2480
+ }
2481
+ try {
2482
+ child.kill(sig);
2483
+ }
2484
+ catch { /* best-effort */ }
2485
+ };
1482
2486
  await new Promise((resolve) => {
1483
2487
  const timer = setTimeout(() => {
1484
- try {
1485
- child.kill('SIGKILL');
1486
- }
1487
- catch {
1488
- // SIGKILL send is best-effort
1489
- }
2488
+ sendSignal('SIGKILL');
1490
2489
  resolve();
1491
2490
  }, timeout);
1492
2491
  child.once('exit', () => {
1493
2492
  clearTimeout(timer);
1494
2493
  resolve();
1495
2494
  });
1496
- try {
1497
- child.kill(force ? 'SIGKILL' : 'SIGTERM');
1498
- }
1499
- catch {
1500
- clearTimeout(timer);
1501
- resolve();
1502
- }
2495
+ sendSignal(force ? 'SIGKILL' : 'SIGTERM');
1503
2496
  });
1504
2497
  }
1505
- async handleClassicCrash(entry, handle, config) {
2498
+ /**
2499
+ * Cancel a pending crash cool-down for `name`, if one is armed.
2500
+ * An explicit stop means "this app is going down now, stay down",
2501
+ * which the cool-down must not undo ten minutes later.
2502
+ */
2503
+ clearCoolDown(name) {
2504
+ const timer = this.coolDownTimers.get(name);
2505
+ if (timer) {
2506
+ clearTimeout(timer);
2507
+ this.coolDownTimers.delete(name);
2508
+ }
2509
+ }
2510
+ async scheduleRecovery(entry, handle, config) {
1506
2511
  const policy = buildRestartPolicy(entry, config);
1507
2512
  if (!policy.enabled)
1508
2513
  return;
@@ -1510,87 +2515,416 @@ export class OrchestratorService extends EventEmitter {
1510
2515
  handle.status = 'crashed';
1511
2516
  this.logger.error({ app: entry.name }, 'Max restarts exceeded');
1512
2517
  this.emit('app:escalated', entry.name);
2518
+ // P1-I — auto-recovery after a cool-down. pm2 has this as
2519
+ // `restart_delay` + a fresh attempt window; pre-fix omnitron
2520
+ // left the app permanently 'crashed' until human intervention.
2521
+ // We reset the counter + schedule one fresh attempt 10 minutes
2522
+ // out. If THAT attempt also crashes through maxRestarts we
2523
+ // land back here and pause another 10 minutes — bounded
2524
+ // self-healing without thrashing.
2525
+ //
2526
+ // The wait happens OUTSIDE the lifecycle queue, and that is the
2527
+ // whole point. It used to be a queued `crash-restart` whose
2528
+ // `run()` slept for the entire cool-down — but `enqueue` will
2529
+ // not preempt a RUNNING head, and with an empty queue the
2530
+ // sleeper always became the head immediately. So the `onPreempt`
2531
+ // hook meant to cancel it could never fire, and an operator's
2532
+ // `stop` or `restart` simply queued behind ten minutes of sleep:
2533
+ // the CLI's RPC timed out after 60s while the op stayed queued,
2534
+ // to fire whenever the cool-down finally elapsed. An app that
2535
+ // exhausted its restart budget was unreachable by every command
2536
+ // the CLI offers — `restart` hung, `stop` hung, `stack start`
2537
+ // skipped it — and the only recovery was restarting the daemon.
2538
+ const COOL_DOWN_MS = 10 * 60_000;
2539
+ this.clearCoolDown(entry.name);
2540
+ const timer = setTimeout(() => {
2541
+ this.coolDownTimers.delete(entry.name);
2542
+ // Re-read the handle: every start replaces it (see
2543
+ // `startAppInternal`), so the one captured above is stale the
2544
+ // moment anything restarts this app. Only a still-crashed app
2545
+ // wants this attempt — an operator who started it meanwhile
2546
+ // has already settled the question.
2547
+ const current = this.handles.get(entry.name);
2548
+ if (!current || current.status !== 'crashed')
2549
+ return;
2550
+ this.logger.info({ app: entry.name }, 'Cool-down elapsed — attempting fresh recovery');
2551
+ current.restarts = 0; // fresh attempt window
2552
+ current.status = 'stopped';
2553
+ void this.lifecycleQueue
2554
+ .enqueue(entry.name, 'crash-restart', () => this.startAppInternal(entry, config))
2555
+ .catch((err) => {
2556
+ if (err instanceof LifecyclePreempted)
2557
+ return;
2558
+ this.logger.error({ app: entry.name, err: err.message }, 'Cool-down recovery failed');
2559
+ });
2560
+ }, COOL_DOWN_MS);
2561
+ timer.unref?.();
2562
+ this.coolDownTimers.set(entry.name, timer);
1513
2563
  return;
1514
2564
  }
1515
2565
  const backoff = policy.backoff ?? config.supervision.backoff;
1516
2566
  const delay = this.calculateBackoff(handle.restarts, backoff);
1517
2567
  this.logger.info({ app: entry.name, delay, attempt: handle.restarts + 1 }, 'Scheduling restart');
1518
- const timer = setTimeout(async () => {
1519
- // Guard: don't restart if orchestrator was stopped during backoff
1520
- if (handle.status === 'stopping' || handle.status === 'stopped')
1521
- return;
1522
- try {
1523
- handle.status = 'stopped';
1524
- await this.startApp(entry, config);
1525
- }
1526
- catch (err) {
1527
- this.logger.error({ app: entry.name, error: err.message }, 'Restart failed');
2568
+ // P0-I — schedule the crash-restart through the lifecycle queue
2569
+ // so an explicit stop preempts the pending restart before it
2570
+ // fires. The setTimeout is still the timing source, but the
2571
+ // queue.enqueue with `onPreempt` guarantees the timer is cleared
2572
+ // (and the promise rejected with LifecyclePreempted) the moment a
2573
+ // higher-priority op for the same app arrives. Pre-fix the timer
2574
+ // could fire during stopApp's `await stopChild(...)` window and
2575
+ // spawn a zombie under the same handle.
2576
+ let timer = null;
2577
+ const restartPromise = this.lifecycleQueue.enqueue(entry.name, 'crash-restart', () => new Promise((resolve, reject) => {
2578
+ timer = setTimeout(async () => {
2579
+ timer = null;
2580
+ // Guard: don't restart if orchestrator was stopped during backoff
2581
+ if (handle.status === 'stopping' || handle.status === 'stopped') {
2582
+ resolve();
2583
+ return;
2584
+ }
2585
+ try {
2586
+ handle.status = 'stopped';
2587
+ // `startAppInternal`, not `startApp`.
2588
+ //
2589
+ // This task holds the lifecycle-queue slot for `entry.name`.
2590
+ // `startApp` enqueues 'start' on that same key, and the queue
2591
+ // cannot run it until this task returns — so the await never
2592
+ // settles and the restart silently never happens. Measured:
2593
+ // `startApp:enter` with no matching exit, and the app left in
2594
+ // whatever state the crash put it in.
2595
+ //
2596
+ // This is why the whole policy below — bounded attempts,
2597
+ // exponential backoff, the ten-minute cool-down described in its
2598
+ // own comment as "bounded self-healing" — had never restarted
2599
+ // anything. The queue was introduced (P0-I) to stop a restart
2600
+ // timer racing a stop; it also made the restart unreachable, and
2601
+ // nothing failed loudly enough to say so.
2602
+ await this.startAppInternal(entry, config);
2603
+ resolve();
2604
+ }
2605
+ catch (err) {
2606
+ const message = err.message;
2607
+ this.logger.error({ app: entry.name, error: message }, 'Restart failed');
2608
+ // Said separately and after, because reading the process table
2609
+ // takes a moment and the failure itself must not wait for it.
2610
+ void this.explainPortConflict(message).then((held) => {
2611
+ if (held)
2612
+ this.logger.error({ app: entry.name }, `Restart failed — ${held}`);
2613
+ });
2614
+ reject(err);
2615
+ }
2616
+ }, delay);
2617
+ }), () => {
2618
+ // onPreempt — explicit stop / restart arrived. Cancel the
2619
+ // pending timer so the restart never fires.
2620
+ if (timer) {
2621
+ clearTimeout(timer);
2622
+ timer = null;
1528
2623
  }
1529
- }, delay);
1530
- // Allow cleanup on explicit stop
2624
+ });
2625
+ // Swallow LifecyclePreempted — that's the explicit-stop happy
2626
+ // path, not an error. Anything else gets logged.
2627
+ restartPromise.catch((err) => {
2628
+ if (err instanceof LifecyclePreempted)
2629
+ return;
2630
+ this.logger.warn({ app: entry.name, err: err.message }, 'Crash restart task rejected');
2631
+ });
2632
+ // Legacy field — callers still poke handle.crashRestartTimer to
2633
+ // check "is a restart pending"; keep it in sync with the queued
2634
+ // timer so existing inspect / status code paths don't break.
1531
2635
  handle.crashRestartTimer = timer;
1532
2636
  }
1533
2637
  calculateBackoff(attempt, backoff) {
1534
- const initial = backoff.initial ?? 1000;
1535
- const max = backoff.max ?? 30_000;
1536
- const factor = backoff.factor ?? 2;
1537
- if (backoff.type === 'exponential')
1538
- return Math.min(initial * Math.pow(factor, attempt), max);
1539
- if (backoff.type === 'linear')
1540
- return Math.min(initial * (attempt + 1), max);
1541
- return initial;
1542
- }
1543
- /** Sample CPU/memory for a single OS process via `ps` (async — does not block event loop) */
2638
+ // Delegate to the shared backoff library in titan-pm. Pre-fix the
2639
+ // inline impl here defaulted `initial=1000` while supervisor's
2640
+ // copy defaulted to 300 — same config produced 3.3× longer
2641
+ // delays in classic-mode than bootstrap-mode. Now both routes
2642
+ // run the same math.
2643
+ return computeBackoff(attempt + 1, backoff);
2644
+ }
2645
+ /**
2646
+ * The live OS processes behind one topology entry.
2647
+ *
2648
+ * Single-instance entries run as supervisor children; entries with
2649
+ * `instances > 1` run in a pool the supervisor knows nothing about. Both
2650
+ * shapes are resolved here, in one place, because metrics sampling and
2651
+ * status reporting each used to resolve them separately — and disagreed.
2652
+ * The pool branch asserted `online` from `pool.size > 0` before looking at
2653
+ * a pid at all, so a worker whose process had exited still read as online
2654
+ * (observed live: two pool workers reporting `online` on pids that `kill
2655
+ * -0` said were gone).
2656
+ */
2657
+ resolveTopologyPids(handle, topo) {
2658
+ return this.resolveTopologyMembers(handle, topo).members.map((m) => m.pid);
2659
+ }
2660
+ /**
2661
+ * The live processes behind one topology entry, keyed by the PM process id
2662
+ * as well as the OS pid.
2663
+ *
2664
+ * The pid alone answers "is it alive"; the process id is what any question
2665
+ * ABOUT that process has to be asked with — `pm.getProcess(id)` carries the
2666
+ * spawn time, and the supervisor's restart counter is keyed by child name.
2667
+ * `resolveTopologyPids` used to discard both on the way out, which is why
2668
+ * the `omnitron list` DTO had nothing to report but the parent's numbers.
2669
+ */
2670
+ resolveTopologyMembers(handle, topo) {
2671
+ const isPoolEntry = (topo.instances ?? 1) > 1;
2672
+ if (isPoolEntry) {
2673
+ const pool = handle.topologyPools.get(topo.name);
2674
+ if (!pool || pool.size === 0)
2675
+ return { members: [], childName: null };
2676
+ const members = pool
2677
+ .getWorkerIds()
2678
+ .map((processId) => ({ processId, pid: this.pm.getWorkerHandle(processId)?.pid }))
2679
+ .filter((m) => typeof m.pid === 'number');
2680
+ return { members, childName: null };
2681
+ }
2682
+ const childNames = handle.supervisor?.getChildNames() ?? [];
2683
+ const childName = childNames.find((cn) => cn.includes(topo.name)) ?? topo.name;
2684
+ const processId = handle.supervisor?.getChildProcessId(childName);
2685
+ if (!processId)
2686
+ return { members: [], childName };
2687
+ const pid = this.pm.getWorkerHandle(processId)?.pid;
2688
+ return { members: typeof pid === 'number' ? [{ processId, pid }] : [], childName };
2689
+ }
2690
+ /**
2691
+ * Sample every process an app owns, and record the per-entry readings on
2692
+ * the handle on the way through.
2693
+ *
2694
+ * A bootstrap app is several OS processes, so "the memory this app uses"
2695
+ * is their sum. The periodic poller used to sample the single pid stored
2696
+ * on the handle — which for a multi-process app is whichever child was
2697
+ * registered last — and overwrite `lastMetrics` with it every five
2698
+ * seconds, while `getMetrics()` right beside it aggregated correctly and
2699
+ * wrote to the same field. The poller won on frequency, so the aggregate
2700
+ * was never what anyone saw: `acme/dev/main` reported 122 MB against 437 MB
2701
+ * actually resident across its three processes.
2702
+ *
2703
+ * ## Why `ps` and not the child's own report
2704
+ *
2705
+ * The aggregating path preferred the PM-level metrics a child reports over
2706
+ * its RPC channel, falling back to `ps`. Those two are not the same
2707
+ * quantities:
2708
+ *
2709
+ * - `cpu` from the child is `(cpuUsage().user + .system) / 1e6` — CPU
2710
+ * SECONDS CONSUMED SINCE START, a monotonically growing counter. `ps`
2711
+ * `%cpu` is a percentage. The dashboard column is labelled `CPU %`,
2712
+ * so an app running for a day would have read `86400`. Observed
2713
+ * climbing live: 12.845 to 13.297 across twelve seconds of idling.
2714
+ * - `memory` from the child is `memoryUsage().heapUsed` — the V8 heap.
2715
+ * `ps` reports RSS, which is what "memory this app uses" means to
2716
+ * whoever is sizing a host. Observed: 66 MB heap against 149 MB
2717
+ * resident for the same process.
2718
+ *
2719
+ * Mixing them per-child, depending on which source answered first, gave a
2720
+ * column that was neither quantity. `ps` is asked for both figures here so
2721
+ * that every number under one label means one thing.
2722
+ */
2723
+ async sampleAppMetrics(handle) {
2724
+ const topology = handle.topologyProcesses ?? [];
2725
+ const childNames = handle.supervisor?.getChildNames() ?? [];
2726
+ // Which pids belong to which topology entry. Everything is sampled in a
2727
+ // single `ps` call: sampling one child at a time would fork a process per
2728
+ // child per polling interval, and a monitor that costs more than what it
2729
+ // monitors is its own kind of defect.
2730
+ const pidsByEntry = new Map();
2731
+ for (const topo of topology) {
2732
+ pidsByEntry.set(topo.name, this.resolveTopologyPids(handle, topo));
2733
+ }
2734
+ if (topology.length === 0) {
2735
+ // No declared topology: fall back to the supervisor's children, and
2736
+ // failing that to the handle's own pid.
2737
+ for (const childName of childNames) {
2738
+ const processId = handle.supervisor?.getChildProcessId(childName);
2739
+ const pid = processId ? this.pm.getWorkerHandle(processId)?.pid : undefined;
2740
+ if (typeof pid === 'number')
2741
+ pidsByEntry.set(childName, [pid]);
2742
+ }
2743
+ if (pidsByEntry.size === 0 && handle.pid) {
2744
+ pidsByEntry.set(handle.name, [handle.pid]);
2745
+ }
2746
+ }
2747
+ const batch = await this.sampleProcessMetricsBatch([...pidsByEntry.values()].flat());
2748
+ let cpu = 0;
2749
+ let memory = 0;
2750
+ const sampled = new Map();
2751
+ for (const [name, pids] of pidsByEntry) {
2752
+ const sample = { cpu: 0, memory: 0 };
2753
+ for (const pid of pids) {
2754
+ const os = batch.get(pid);
2755
+ if (!os)
2756
+ continue;
2757
+ sample.cpu += os.cpu;
2758
+ sample.memory += os.memory;
2759
+ }
2760
+ sampled.set(name, sample);
2761
+ cpu += sample.cpu;
2762
+ memory += sample.memory;
2763
+ }
2764
+ // Traffic can only come from inside the child; `ps` knows nothing about
2765
+ // it. Its CPU and memory figures are deliberately NOT used — see the note
2766
+ // on this method. Only what a child reports as TRAFFIC is summed: the
2767
+ // `requests`/`errors` a child on an older runtime reports are the call
2768
+ // counters of its wrapper — the supervisor's own health and metrics
2769
+ // calls — and adding them up counted this daemon's questions as the
2770
+ // app's requests (`combineProcessTraffic`).
2771
+ const reports = [];
2772
+ for (const childName of childNames) {
2773
+ const reported = await handle.supervisor?.getChildMetrics(childName);
2774
+ reports.push(reported?.traffic);
2775
+ }
2776
+ const traffic = combineProcessTraffic(reports);
2777
+ // A sample that produced nothing for an app that HAS live processes is a
2778
+ // failed sample, not a measurement of zero — `ps` can time out under load,
2779
+ // and it did: an app was observed reporting 0 MB one second and 143 MB the
2780
+ // next. Overwriting the last known reading with zeros would put the exact
2781
+ // placeholder this method exists to remove back on the dashboard, so the
2782
+ // previous reading is kept instead.
2783
+ const sampledNothing = batch.size === 0 && [...pidsByEntry.values()].some((pids) => pids.length > 0);
2784
+ if (sampledNothing && handle.lastMetrics) {
2785
+ return handle.lastMetrics;
2786
+ }
2787
+ handle.childMetrics = sampled;
2788
+ return traffic
2789
+ ? {
2790
+ cpu,
2791
+ memory,
2792
+ requests: traffic.requests,
2793
+ errors: traffic.serverErrors,
2794
+ traffic,
2795
+ ...(traffic.latency
2796
+ ? {
2797
+ latency: {
2798
+ p50: traffic.latency.p50,
2799
+ p75: traffic.latency.p75,
2800
+ p90: traffic.latency.p90,
2801
+ p95: traffic.latency.p95,
2802
+ p99: traffic.latency.p99,
2803
+ mean: traffic.latency.mean,
2804
+ },
2805
+ }
2806
+ : {}),
2807
+ }
2808
+ : { cpu, memory };
2809
+ }
2810
+ /**
2811
+ * Sample several processes in one `ps` call.
2812
+ *
2813
+ * Pids the OS no longer knows are simply absent from the result — an
2814
+ * omission the caller can see, unlike a zeroed reading it cannot tell
2815
+ * apart from an idle process.
2816
+ */
2817
+ async sampleProcessMetricsBatch(pids) {
2818
+ const out = new Map();
2819
+ const unique = [...new Set(pids)];
2820
+ if (unique.length === 0)
2821
+ return out;
2822
+ try {
2823
+ const psOutput = await new Promise((resolve, reject) => {
2824
+ execFile('ps', ['-p', unique.join(','), '-o', 'pid=,rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
2825
+ // `ps` exits non-zero when NONE of the pids exist, but still
2826
+ // prints the ones that do — so stdout is worth reading either way.
2827
+ if (err && !stdout)
2828
+ reject(err);
2829
+ else
2830
+ resolve(stdout.trim());
2831
+ });
2832
+ });
2833
+ for (const [pid, sample] of parsePsBatch(psOutput))
2834
+ out.set(pid, sample);
2835
+ }
2836
+ catch {
2837
+ // Leave the map empty: no reading is honest, a zero reading is not.
2838
+ }
2839
+ return out;
2840
+ }
2841
+ /**
2842
+ * Sample CPU/memory for a single OS process via `ps`.
2843
+ *
2844
+ * `null` when it could not be read. This used to return
2845
+ * `{ cpu: 0, memory: 0 }` on any failure, and the caller wrote that
2846
+ * straight over `handle.lastMetrics` — so a `ps` that timed out under load
2847
+ * made a running classic-mode app read as idle, indistinguishable from one
2848
+ * that genuinely was. The batch path above already treated absence as
2849
+ * absence; these two now agree.
2850
+ */
1544
2851
  async sampleProcessMetrics(pid) {
1545
2852
  try {
1546
- const { execFile } = await import('node:child_process');
1547
2853
  const psOutput = await new Promise((resolve, reject) => {
1548
- execFile('ps', ['-p', String(pid), '-o', 'rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
1549
- if (err)
2854
+ execFile('ps', ['-p', String(pid), '-o', 'pid=,rss=,%cpu='], { timeout: 5000 }, (err, stdout) => {
2855
+ if (err && !stdout)
1550
2856
  reject(err);
1551
2857
  else
1552
2858
  resolve(stdout.trim());
1553
2859
  });
1554
2860
  });
1555
- const [rssKb, cpuPercent] = psOutput.split(/\s+/);
1556
- return {
1557
- cpu: parseFloat(cpuPercent ?? '0'),
1558
- memory: parseInt(rssKb ?? '0', 10) * 1024,
1559
- };
2861
+ return parsePsLine(psOutput)?.sample ?? null;
1560
2862
  }
1561
2863
  catch {
1562
- return { cpu: 0, memory: 0 };
2864
+ return null;
1563
2865
  }
1564
2866
  }
1565
- /** Periodic metrics polling for classic mode processes */
2867
+ /**
2868
+ * Periodic CPU/memory sampling for every managed process.
2869
+ *
2870
+ * Idempotent: re-arming replaces the previous timer rather than stacking a
2871
+ * second one, so `setConfig` can call it on every config reload.
2872
+ */
1566
2873
  startMetricsPolling(interval) {
2874
+ if (this.metricsTimer) {
2875
+ clearInterval(this.metricsTimer);
2876
+ this.metricsTimer = null;
2877
+ }
1567
2878
  this.metricsTimer = setInterval(async () => {
1568
2879
  for (const handle of this.handles.values()) {
1569
2880
  if (handle.status !== 'online')
1570
2881
  continue;
1571
2882
  if (handle.mode === 'classic' && handle.childProcess?.pid) {
1572
2883
  const m = await this.sampleProcessMetrics(handle.childProcess.pid);
1573
- handle.lastMetrics = { ...m, requests: 0, errors: 0 };
2884
+ // A failed read keeps the previous numbers rather than replacing
2885
+ // them with zeros — the same rule the bootstrap path below uses.
2886
+ if (m) {
2887
+ handle.lastMetrics = {
2888
+ ...m,
2889
+ requests: handle.lastMetrics?.requests ?? 0,
2890
+ errors: handle.lastMetrics?.errors ?? 0,
2891
+ };
2892
+ }
1574
2893
  }
1575
- else if (handle.mode === 'bootstrap' && handle.pid) {
1576
- const m = await this.sampleProcessMetrics(handle.pid);
1577
- handle.lastMetrics = {
1578
- ...m,
1579
- requests: handle.lastMetrics?.requests ?? 0,
1580
- errors: handle.lastMetrics?.errors ?? 0,
1581
- };
2894
+ else if (handle.mode === 'bootstrap') {
2895
+ // Every process the app owns, not just the one on the handle.
2896
+ // As sampled. The counts used to fall back to the previous reading
2897
+ // whenever a sample said 0 — which kept a stale figure alive
2898
+ // across a process restart (traffic counts restart from zero with
2899
+ // the process) and, while `requests` was the wrapper's call count,
2900
+ // mixed the two meanings in one field.
2901
+ handle.lastMetrics = await this.sampleAppMetrics(handle);
1582
2902
  }
1583
2903
  }
1584
2904
  }, interval);
2905
+ this.metricsTimer.unref();
1585
2906
  }
1586
2907
  // ============================================================================
1587
2908
  // Private — Utilities
1588
2909
  // ============================================================================
1589
2910
  toProcessInfo(handle) {
2911
+ // Ghost-online guard. The in-memory `handle.status` reflects the
2912
+ // last event the supervisor observed. If a child died between
2913
+ // the last sweep and now (kill -9 by an operator, OOM kill,
2914
+ // container restart) the handle still claims 'online' until the
2915
+ // next event lands. A cheap signal-0 verification at DTO-mapping
2916
+ // time keeps `omnitron list` honest: when the OS reports the pid
2917
+ // is gone, surface 'crashed' instead of lying to the operator.
2918
+ // `isAlive` covers EPERM as alive (correct — process exists, we
2919
+ // just can't signal it).
2920
+ let status = handle.status;
2921
+ if (status === 'online' && handle.pid != null && !isAlive(handle.pid)) {
2922
+ status = 'crashed';
2923
+ }
1590
2924
  const info = {
1591
2925
  name: handle.name,
1592
2926
  pid: handle.pid,
1593
- status: handle.status,
2927
+ status,
1594
2928
  cpu: handle.cpu,
1595
2929
  memory: handle.memory,
1596
2930
  uptime: handle.uptime,
@@ -1605,49 +2939,48 @@ export class OrchestratorService extends EventEmitter {
1605
2939
  info.processes = [];
1606
2940
  for (const topo of handle.topologyProcesses) {
1607
2941
  const isPoolEntry = (topo.instances ?? 1) > 1;
1608
- let childPid = null;
1609
- let childStatus = 'stopped';
1610
- if (isPoolEntry) {
1611
- // Pool-managed topology — the supervisor doesn't know about
1612
- // these (workers are owned by ProcessPool). Read directly
1613
- // from the pool ref we stashed at creation time.
1614
- const pool = handle.topologyPools.get(topo.name);
1615
- if (pool && pool.size > 0) {
1616
- childStatus = 'online';
1617
- // Surface the first worker's PID. Pool workers are
1618
- // interchangeable so any one is representative; the CLI
1619
- // prints "PID" as a quick liveness signal, not a unique
1620
- // identifier.
1621
- const workerIds = pool.getWorkerIds();
1622
- if (workerIds.length > 0) {
1623
- const firstId = workerIds[0];
1624
- const wh = this.pm.getWorkerHandle(firstId);
1625
- childPid = wh?.pid ?? null;
1626
- }
1627
- }
1628
- }
1629
- else {
1630
- // Single-instance topology — supervisor manages the child directly.
1631
- const childNames = handle.supervisor.getChildNames();
1632
- const childName = childNames.find((cn) => cn.includes(topo.name)) ?? topo.name;
1633
- const processId = handle.supervisor.getChildProcessId(childName);
1634
- if (processId) {
1635
- const wh = this.pm.getWorkerHandle(processId);
1636
- childPid = wh?.pid ?? null;
1637
- childStatus = childPid ? 'online' : 'stopped';
1638
- }
1639
- }
2942
+ // The same ghost-online guard the parent gets a few lines above.
2943
+ // Without it the pool branch reported `online` from the pool's own
2944
+ // bookkeeping, never checking whether the pid it printed alongside
2945
+ // still existed.
2946
+ const { members, childName } = this.resolveTopologyMembers(handle, topo);
2947
+ const alive = members.filter((m) => isAlive(m.pid));
2948
+ const declaredInstances = topo.instances ?? 1;
2949
+ const liveInstances = isPoolEntry
2950
+ ? (handle.topologyPools.get(topo.name)?.size ?? 0)
2951
+ : alive.length;
2952
+ const chosen = alive[0] ?? members[0];
2953
+ const childPid = chosen?.pid ?? null;
2954
+ const childStatus = alive.length > 0 ? 'online' : members.length > 0 ? 'crashed' : 'stopped';
2955
+ // The row prints one pid, so the row's clock is that pid's clock.
2956
+ // `startTime` is written when the process id is registered and a
2957
+ // restart registers a new one, so it moves when the process does.
2958
+ // (`IProcessInfo.restartCount` does NOT: it is assigned 0 there and
2959
+ // incremented nowhere. The supervisor keeps the real count, by child
2960
+ // name — reading the field instead would have replaced a wrong
2961
+ // constant with a wrong lookup.)
2962
+ const chosenInfo = chosen ? this.pm.getProcess(chosen.processId) : undefined;
2963
+ const childUptime = typeof chosenInfo?.startTime === 'number' && chosenInfo.startTime > 0
2964
+ ? Math.max(0, Date.now() - chosenInfo.startTime)
2965
+ : 0;
2966
+ // A pool is not a supervisor child and nothing counts its restarts.
2967
+ // `null` says so; `0` would be absence written as a value from the
2968
+ // domain, leaving "never restarted" and "not tracked" identical.
2969
+ const childRestarts = isPoolEntry || !childName ? null : (handle.supervisor?.getRestartCount(childName) ?? null);
1640
2970
  // Derive process type from declarations for backward-compatible DTO
1641
2971
  const derivedType = topo.transports ? 'server' : isPoolEntry ? 'worker' : 'custom';
2972
+ const sample = handle.childMetrics.get(topo.name);
1642
2973
  info.processes.push({
1643
2974
  name: topo.name,
1644
2975
  type: derivedType,
1645
2976
  pid: childPid,
1646
2977
  status: childStatus,
1647
- cpu: 0,
1648
- memory: 0,
1649
- uptime: handle.uptime,
1650
- restarts: 0,
2978
+ cpu: sample?.cpu ?? 0,
2979
+ memory: sample?.memory ?? 0,
2980
+ uptime: childUptime,
2981
+ restarts: childRestarts,
2982
+ instances: liveInstances,
2983
+ declaredInstances,
1651
2984
  });
1652
2985
  }
1653
2986
  }