@monoes/monomindcli 2.16.12 → 2.16.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (386) hide show
  1. package/.claude/agents/core/coder.md +1 -1
  2. package/.claude/agents/engineering/engineering-devops-automator.md +1 -1
  3. package/.claude/agents/github/repo-architect.md +1 -1
  4. package/.claude/agents/testing/production-validator.md +1 -1
  5. package/.claude/agents/testing/testing-tool-evaluator.md +1 -1
  6. package/.claude/commands/github/pr-manager.md +1 -1
  7. package/.claude/helpers/pick-rank.cjs +34 -4
  8. package/.claude/skills/monolean-audit/SKILL.md +1 -1
  9. package/README.md +1 -1
  10. package/dist/src/commands/cleanup-data.d.ts +30 -0
  11. package/dist/src/commands/cleanup-data.d.ts.map +1 -0
  12. package/dist/src/commands/cleanup-data.js +138 -0
  13. package/dist/src/commands/cleanup-data.js.map +1 -0
  14. package/dist/src/commands/cleanup-scratch.d.ts +43 -0
  15. package/dist/src/commands/cleanup-scratch.d.ts.map +1 -0
  16. package/dist/src/commands/cleanup-scratch.js +125 -0
  17. package/dist/src/commands/cleanup-scratch.js.map +1 -0
  18. package/dist/src/commands/cleanup.d.ts +2 -39
  19. package/dist/src/commands/cleanup.d.ts.map +1 -1
  20. package/dist/src/commands/cleanup.js +5 -247
  21. package/dist/src/commands/cleanup.js.map +1 -1
  22. package/dist/src/commands/doctor-gitignore-checks.d.ts +10 -0
  23. package/dist/src/commands/doctor-gitignore-checks.d.ts.map +1 -0
  24. package/dist/src/commands/doctor-gitignore-checks.js +141 -0
  25. package/dist/src/commands/doctor-gitignore-checks.js.map +1 -0
  26. package/dist/src/commands/doctor-helpers-checks.d.ts +10 -0
  27. package/dist/src/commands/doctor-helpers-checks.d.ts.map +1 -0
  28. package/dist/src/commands/doctor-helpers-checks.js +292 -0
  29. package/dist/src/commands/doctor-helpers-checks.js.map +1 -0
  30. package/dist/src/commands/doctor-memory-checks.d.ts +47 -0
  31. package/dist/src/commands/doctor-memory-checks.d.ts.map +1 -0
  32. package/dist/src/commands/doctor-memory-checks.js +298 -0
  33. package/dist/src/commands/doctor-memory-checks.js.map +1 -0
  34. package/dist/src/commands/doctor-monograph-checks.d.ts +9 -0
  35. package/dist/src/commands/doctor-monograph-checks.d.ts.map +1 -0
  36. package/dist/src/commands/doctor-monograph-checks.js +331 -0
  37. package/dist/src/commands/doctor-monograph-checks.js.map +1 -0
  38. package/dist/src/commands/doctor-project-checks.d.ts +7 -66
  39. package/dist/src/commands/doctor-project-checks.d.ts.map +1 -1
  40. package/dist/src/commands/doctor-project-checks.js +10 -1331
  41. package/dist/src/commands/doctor-project-checks.js.map +1 -1
  42. package/dist/src/commands/doctor-routing-checks.d.ts +10 -0
  43. package/dist/src/commands/doctor-routing-checks.d.ts.map +1 -0
  44. package/dist/src/commands/doctor-routing-checks.js +194 -0
  45. package/dist/src/commands/doctor-routing-checks.js.map +1 -0
  46. package/dist/src/commands/doctor-worker-checks.d.ts +17 -0
  47. package/dist/src/commands/doctor-worker-checks.d.ts.map +1 -0
  48. package/dist/src/commands/doctor-worker-checks.js +122 -0
  49. package/dist/src/commands/doctor-worker-checks.js.map +1 -0
  50. package/dist/src/commands/hooks-routing-commands.js +1 -1
  51. package/dist/src/commands/hooks-routing-commands.js.map +1 -1
  52. package/dist/src/commands/init.d.ts.map +1 -1
  53. package/dist/src/commands/init.js +6 -15
  54. package/dist/src/commands/init.js.map +1 -1
  55. package/dist/src/commands/org-control.d.ts +117 -0
  56. package/dist/src/commands/org-control.d.ts.map +1 -0
  57. package/dist/src/commands/org-control.js +274 -0
  58. package/dist/src/commands/org-control.js.map +1 -0
  59. package/dist/src/commands/org-lifecycle.d.ts +7 -0
  60. package/dist/src/commands/org-lifecycle.d.ts.map +1 -0
  61. package/dist/src/commands/org-lifecycle.js +298 -0
  62. package/dist/src/commands/org-lifecycle.js.map +1 -0
  63. package/dist/src/commands/org-manage.d.ts +7 -0
  64. package/dist/src/commands/org-manage.d.ts.map +1 -0
  65. package/dist/src/commands/org-manage.js +357 -0
  66. package/dist/src/commands/org-manage.js.map +1 -0
  67. package/dist/src/commands/org-memory-command.d.ts +3 -0
  68. package/dist/src/commands/org-memory-command.d.ts.map +1 -0
  69. package/dist/src/commands/org-memory-command.js +178 -0
  70. package/dist/src/commands/org-memory-command.js.map +1 -0
  71. package/dist/src/commands/org-observe-approvals.d.ts +8 -0
  72. package/dist/src/commands/org-observe-approvals.d.ts.map +1 -0
  73. package/dist/src/commands/org-observe-approvals.js +211 -0
  74. package/dist/src/commands/org-observe-approvals.js.map +1 -0
  75. package/dist/src/commands/org-observe-checkpoint.d.ts +26 -0
  76. package/dist/src/commands/org-observe-checkpoint.d.ts.map +1 -0
  77. package/dist/src/commands/org-observe-checkpoint.js +180 -0
  78. package/dist/src/commands/org-observe-checkpoint.js.map +1 -0
  79. package/dist/src/commands/org-observe-config.d.ts +8 -0
  80. package/dist/src/commands/org-observe-config.d.ts.map +1 -0
  81. package/dist/src/commands/org-observe-config.js +160 -0
  82. package/dist/src/commands/org-observe-config.js.map +1 -0
  83. package/dist/src/commands/org-observe-gates.d.ts +4 -0
  84. package/dist/src/commands/org-observe-gates.d.ts.map +1 -0
  85. package/dist/src/commands/org-observe-gates.js +182 -0
  86. package/dist/src/commands/org-observe-gates.js.map +1 -0
  87. package/dist/src/commands/org-observe-inbox.d.ts +18 -0
  88. package/dist/src/commands/org-observe-inbox.d.ts.map +1 -0
  89. package/dist/src/commands/org-observe-inbox.js +154 -0
  90. package/dist/src/commands/org-observe-inbox.js.map +1 -0
  91. package/dist/src/commands/org-observe-logs.d.ts +31 -0
  92. package/dist/src/commands/org-observe-logs.d.ts.map +1 -0
  93. package/dist/src/commands/org-observe-logs.js +272 -0
  94. package/dist/src/commands/org-observe-logs.js.map +1 -0
  95. package/dist/src/commands/org-observe-questions.d.ts +8 -0
  96. package/dist/src/commands/org-observe-questions.d.ts.map +1 -0
  97. package/dist/src/commands/org-observe-questions.js +201 -0
  98. package/dist/src/commands/org-observe-questions.js.map +1 -0
  99. package/dist/src/commands/org-observe-report.d.ts +8 -0
  100. package/dist/src/commands/org-observe-report.d.ts.map +1 -0
  101. package/dist/src/commands/org-observe-report.js +366 -0
  102. package/dist/src/commands/org-observe-report.js.map +1 -0
  103. package/dist/src/commands/org-observe-shared.d.ts +20 -0
  104. package/dist/src/commands/org-observe-shared.d.ts.map +1 -0
  105. package/dist/src/commands/org-observe-shared.js +55 -0
  106. package/dist/src/commands/org-observe-shared.js.map +1 -0
  107. package/dist/src/commands/org-observe.d.ts +9 -101
  108. package/dist/src/commands/org-observe.d.ts.map +1 -1
  109. package/dist/src/commands/org-observe.js +12 -1675
  110. package/dist/src/commands/org-observe.js.map +1 -1
  111. package/dist/src/commands/org-poll.d.ts +57 -0
  112. package/dist/src/commands/org-poll.d.ts.map +1 -0
  113. package/dist/src/commands/org-poll.js +194 -0
  114. package/dist/src/commands/org-poll.js.map +1 -0
  115. package/dist/src/commands/org-run.d.ts +3 -0
  116. package/dist/src/commands/org-run.d.ts.map +1 -0
  117. package/dist/src/commands/org-run.js +449 -0
  118. package/dist/src/commands/org-run.js.map +1 -0
  119. package/dist/src/commands/org-serve.d.ts +19 -0
  120. package/dist/src/commands/org-serve.d.ts.map +1 -0
  121. package/dist/src/commands/org-serve.js +395 -0
  122. package/dist/src/commands/org-serve.js.map +1 -0
  123. package/dist/src/commands/org-subcommands-config.d.ts +8 -0
  124. package/dist/src/commands/org-subcommands-config.d.ts.map +1 -0
  125. package/dist/src/commands/org-subcommands-config.js +93 -0
  126. package/dist/src/commands/org-subcommands-config.js.map +1 -0
  127. package/dist/src/commands/org-subcommands-hil.d.ts +11 -0
  128. package/dist/src/commands/org-subcommands-hil.d.ts.map +1 -0
  129. package/dist/src/commands/org-subcommands-hil.js +216 -0
  130. package/dist/src/commands/org-subcommands-hil.js.map +1 -0
  131. package/dist/src/commands/org-subcommands-observe.d.ts +12 -0
  132. package/dist/src/commands/org-subcommands-observe.d.ts.map +1 -0
  133. package/dist/src/commands/org-subcommands-observe.js +249 -0
  134. package/dist/src/commands/org-subcommands-observe.js.map +1 -0
  135. package/dist/src/commands/org-subcommands-runtime.d.ts +12 -0
  136. package/dist/src/commands/org-subcommands-runtime.d.ts.map +1 -0
  137. package/dist/src/commands/org-subcommands-runtime.js +163 -0
  138. package/dist/src/commands/org-subcommands-runtime.js.map +1 -0
  139. package/dist/src/commands/org.d.ts +3 -151
  140. package/dist/src/commands/org.d.ts.map +1 -1
  141. package/dist/src/commands/org.js +43 -2735
  142. package/dist/src/commands/org.js.map +1 -1
  143. package/dist/src/index.js +2 -2
  144. package/dist/src/index.js.map +1 -1
  145. package/dist/src/init/never-commit.d.ts +3 -3
  146. package/dist/src/init/types.d.ts +1 -1
  147. package/dist/src/init/types.js +1 -1
  148. package/dist/src/knowledge/document-store.js +1 -1
  149. package/dist/src/knowledge/document-store.js.map +1 -1
  150. package/dist/src/mcp-tools/hooks-edit-command.d.ts +10 -0
  151. package/dist/src/mcp-tools/hooks-edit-command.d.ts.map +1 -0
  152. package/dist/src/mcp-tools/hooks-edit-command.js +247 -0
  153. package/dist/src/mcp-tools/hooks-edit-command.js.map +1 -0
  154. package/dist/src/mcp-tools/hooks-learning.d.ts +9 -0
  155. package/dist/src/mcp-tools/hooks-learning.d.ts.map +1 -0
  156. package/dist/src/mcp-tools/hooks-learning.js +408 -0
  157. package/dist/src/mcp-tools/hooks-learning.js.map +1 -0
  158. package/dist/src/mcp-tools/hooks-metrics-list.d.ts +8 -0
  159. package/dist/src/mcp-tools/hooks-metrics-list.d.ts.map +1 -0
  160. package/dist/src/mcp-tools/hooks-metrics-list.js +205 -0
  161. package/dist/src/mcp-tools/hooks-metrics-list.js.map +1 -0
  162. package/dist/src/mcp-tools/hooks-route.d.ts +12 -0
  163. package/dist/src/mcp-tools/hooks-route.d.ts.map +1 -0
  164. package/dist/src/mcp-tools/hooks-route.js +251 -0
  165. package/dist/src/mcp-tools/hooks-route.js.map +1 -0
  166. package/dist/src/mcp-tools/hooks-routing.d.ts +6 -27
  167. package/dist/src/mcp-tools/hooks-routing.d.ts.map +1 -1
  168. package/dist/src/mcp-tools/hooks-routing.js +6 -1727
  169. package/dist/src/mcp-tools/hooks-routing.js.map +1 -1
  170. package/dist/src/mcp-tools/hooks-session.d.ts +9 -0
  171. package/dist/src/mcp-tools/hooks-session.d.ts.map +1 -0
  172. package/dist/src/mcp-tools/hooks-session.js +243 -0
  173. package/dist/src/mcp-tools/hooks-session.js.map +1 -0
  174. package/dist/src/mcp-tools/hooks-task.d.ts +17 -0
  175. package/dist/src/mcp-tools/hooks-task.d.ts.map +1 -0
  176. package/dist/src/mcp-tools/hooks-task.js +427 -0
  177. package/dist/src/mcp-tools/hooks-task.js.map +1 -0
  178. package/dist/src/mcp-tools/hooks-tools.d.ts +2 -1
  179. package/dist/src/mcp-tools/hooks-tools.d.ts.map +1 -1
  180. package/dist/src/mcp-tools/hooks-tools.js +2 -1
  181. package/dist/src/mcp-tools/hooks-tools.js.map +1 -1
  182. package/dist/src/mcp-tools/memory-tools.js +1 -1
  183. package/dist/src/mcp-tools/memory-tools.js.map +1 -1
  184. package/dist/src/memory/bm25-index.d.ts +2 -2
  185. package/dist/src/memory/bm25-index.js +2 -2
  186. package/dist/src/memory/embedding-operations.js +2 -2
  187. package/dist/src/memory/embedding-operations.js.map +1 -1
  188. package/dist/src/memory/entity-name-key.d.ts +1 -1
  189. package/dist/src/memory/entity-name-key.js +1 -1
  190. package/dist/src/memory/hnsw-operations.d.ts +1 -1
  191. package/dist/src/memory/hnsw-operations.js +1 -1
  192. package/dist/src/memory/memory-bridge-backend.d.ts +43 -0
  193. package/dist/src/memory/memory-bridge-backend.d.ts.map +1 -0
  194. package/dist/src/memory/memory-bridge-backend.js +363 -0
  195. package/dist/src/memory/memory-bridge-backend.js.map +1 -0
  196. package/dist/src/memory/memory-bridge-core.d.ts +42 -0
  197. package/dist/src/memory/memory-bridge-core.d.ts.map +1 -0
  198. package/dist/src/memory/memory-bridge-core.js +126 -0
  199. package/dist/src/memory/memory-bridge-core.js.map +1 -0
  200. package/dist/src/memory/memory-bridge-entries.d.ts +131 -0
  201. package/dist/src/memory/memory-bridge-entries.d.ts.map +1 -0
  202. package/dist/src/memory/memory-bridge-entries.js +227 -0
  203. package/dist/src/memory/memory-bridge-entries.js.map +1 -0
  204. package/dist/src/memory/memory-bridge-feedback.d.ts +60 -0
  205. package/dist/src/memory/memory-bridge-feedback.d.ts.map +1 -0
  206. package/dist/src/memory/memory-bridge-feedback.js +118 -0
  207. package/dist/src/memory/memory-bridge-feedback.js.map +1 -0
  208. package/dist/src/memory/memory-bridge-paths.d.ts +123 -0
  209. package/dist/src/memory/memory-bridge-paths.d.ts.map +1 -0
  210. package/dist/src/memory/memory-bridge-paths.js +335 -0
  211. package/dist/src/memory/memory-bridge-paths.js.map +1 -0
  212. package/dist/src/memory/memory-bridge-search.d.ts +48 -0
  213. package/dist/src/memory/memory-bridge-search.d.ts.map +1 -0
  214. package/dist/src/memory/memory-bridge-search.js +383 -0
  215. package/dist/src/memory/memory-bridge-search.js.map +1 -0
  216. package/dist/src/memory/memory-bridge-sessions.d.ts +38 -0
  217. package/dist/src/memory/memory-bridge-sessions.d.ts.map +1 -0
  218. package/dist/src/memory/memory-bridge-sessions.js +109 -0
  219. package/dist/src/memory/memory-bridge-sessions.js.map +1 -0
  220. package/dist/src/memory/memory-bridge-store.d.ts +58 -0
  221. package/dist/src/memory/memory-bridge-store.d.ts.map +1 -0
  222. package/dist/src/memory/memory-bridge-store.js +201 -0
  223. package/dist/src/memory/memory-bridge-store.js.map +1 -0
  224. package/dist/src/memory/memory-bridge.d.ts +10 -438
  225. package/dist/src/memory/memory-bridge.d.ts.map +1 -1
  226. package/dist/src/memory/memory-bridge.js +14 -1787
  227. package/dist/src/memory/memory-bridge.js.map +1 -1
  228. package/dist/src/memory/memory-kg-claims.d.ts +52 -0
  229. package/dist/src/memory/memory-kg-claims.d.ts.map +1 -0
  230. package/dist/src/memory/memory-kg-claims.js +99 -0
  231. package/dist/src/memory/memory-kg-claims.js.map +1 -0
  232. package/dist/src/memory/memory-kg-index.d.ts +86 -0
  233. package/dist/src/memory/memory-kg-index.d.ts.map +1 -0
  234. package/dist/src/memory/memory-kg-index.js +256 -0
  235. package/dist/src/memory/memory-kg-index.js.map +1 -0
  236. package/dist/src/memory/memory-kg-ingest.d.ts +44 -0
  237. package/dist/src/memory/memory-kg-ingest.d.ts.map +1 -0
  238. package/dist/src/memory/memory-kg-ingest.js +361 -0
  239. package/dist/src/memory/memory-kg-ingest.js.map +1 -0
  240. package/dist/src/memory/memory-kg-model.d.ts +168 -0
  241. package/dist/src/memory/memory-kg-model.d.ts.map +1 -0
  242. package/dist/src/memory/memory-kg-model.js +158 -0
  243. package/dist/src/memory/memory-kg-model.js.map +1 -0
  244. package/dist/src/memory/memory-kg-names.d.ts +64 -0
  245. package/dist/src/memory/memory-kg-names.d.ts.map +1 -0
  246. package/dist/src/memory/memory-kg-names.js +187 -0
  247. package/dist/src/memory/memory-kg-names.js.map +1 -0
  248. package/dist/src/memory/memory-kg-promote.d.ts +79 -0
  249. package/dist/src/memory/memory-kg-promote.d.ts.map +1 -0
  250. package/dist/src/memory/memory-kg-promote.js +248 -0
  251. package/dist/src/memory/memory-kg-promote.js.map +1 -0
  252. package/dist/src/memory/memory-kg-rebuild.d.ts +77 -0
  253. package/dist/src/memory/memory-kg-rebuild.d.ts.map +1 -0
  254. package/dist/src/memory/memory-kg-rebuild.js +299 -0
  255. package/dist/src/memory/memory-kg-rebuild.js.map +1 -0
  256. package/dist/src/memory/memory-kg-rollback.d.ts +47 -0
  257. package/dist/src/memory/memory-kg-rollback.d.ts.map +1 -0
  258. package/dist/src/memory/memory-kg-rollback.js +193 -0
  259. package/dist/src/memory/memory-kg-rollback.js.map +1 -0
  260. package/dist/src/memory/memory-kg-rules.d.ts +64 -0
  261. package/dist/src/memory/memory-kg-rules.d.ts.map +1 -0
  262. package/dist/src/memory/memory-kg-rules.js +251 -0
  263. package/dist/src/memory/memory-kg-rules.js.map +1 -0
  264. package/dist/src/memory/memory-kg-scan.d.ts +118 -0
  265. package/dist/src/memory/memory-kg-scan.d.ts.map +1 -0
  266. package/dist/src/memory/memory-kg-scan.js +209 -0
  267. package/dist/src/memory/memory-kg-scan.js.map +1 -0
  268. package/dist/src/memory/memory-kg-search.d.ts +80 -0
  269. package/dist/src/memory/memory-kg-search.d.ts.map +1 -0
  270. package/dist/src/memory/memory-kg-search.js +239 -0
  271. package/dist/src/memory/memory-kg-search.js.map +1 -0
  272. package/dist/src/memory/memory-kg.d.ts +19 -547
  273. package/dist/src/memory/memory-kg.d.ts.map +1 -1
  274. package/dist/src/memory/memory-kg.js +10 -2389
  275. package/dist/src/memory/memory-kg.js.map +1 -1
  276. package/dist/src/orgrt/daemon-types.d.ts +196 -0
  277. package/dist/src/orgrt/daemon-types.d.ts.map +1 -0
  278. package/dist/src/orgrt/daemon-types.js +38 -0
  279. package/dist/src/orgrt/daemon-types.js.map +1 -0
  280. package/dist/src/orgrt/daemon.d.ts +70 -300
  281. package/dist/src/orgrt/daemon.d.ts.map +1 -1
  282. package/dist/src/orgrt/daemon.js +41 -2536
  283. package/dist/src/orgrt/daemon.js.map +1 -1
  284. package/dist/src/orgrt/idle-watchdog.d.ts +39 -0
  285. package/dist/src/orgrt/idle-watchdog.d.ts.map +1 -0
  286. package/dist/src/orgrt/idle-watchdog.js +289 -0
  287. package/dist/src/orgrt/idle-watchdog.js.map +1 -0
  288. package/dist/src/orgrt/org-reload.d.ts +7 -0
  289. package/dist/src/orgrt/org-reload.d.ts.map +1 -0
  290. package/dist/src/orgrt/org-reload.js +111 -0
  291. package/dist/src/orgrt/org-reload.js.map +1 -0
  292. package/dist/src/orgrt/org-start-steps.d.ts +21 -0
  293. package/dist/src/orgrt/org-start-steps.d.ts.map +1 -0
  294. package/dist/src/orgrt/org-start-steps.js +305 -0
  295. package/dist/src/orgrt/org-start-steps.js.map +1 -0
  296. package/dist/src/orgrt/org-start.d.ts +7 -0
  297. package/dist/src/orgrt/org-start.d.ts.map +1 -0
  298. package/dist/src/orgrt/org-start.js +435 -0
  299. package/dist/src/orgrt/org-start.js.map +1 -0
  300. package/dist/src/orgrt/org-state-file.d.ts +8 -0
  301. package/dist/src/orgrt/org-state-file.d.ts.map +1 -0
  302. package/dist/src/orgrt/org-state-file.js +116 -0
  303. package/dist/src/orgrt/org-state-file.js.map +1 -0
  304. package/dist/src/orgrt/org-stop.d.ts +6 -0
  305. package/dist/src/orgrt/org-stop.d.ts.map +1 -0
  306. package/dist/src/orgrt/org-stop.js +332 -0
  307. package/dist/src/orgrt/org-stop.js.map +1 -0
  308. package/dist/src/orgrt/org-tools.d.ts +11 -0
  309. package/dist/src/orgrt/org-tools.d.ts.map +1 -0
  310. package/dist/src/orgrt/org-tools.js +305 -0
  311. package/dist/src/orgrt/org-tools.js.map +1 -0
  312. package/dist/src/orgrt/role-incarnation.d.ts +18 -0
  313. package/dist/src/orgrt/role-incarnation.d.ts.map +1 -0
  314. package/dist/src/orgrt/role-incarnation.js +374 -0
  315. package/dist/src/orgrt/role-incarnation.js.map +1 -0
  316. package/dist/src/orgrt/role-respawn.d.ts +7 -0
  317. package/dist/src/orgrt/role-respawn.d.ts.map +1 -0
  318. package/dist/src/orgrt/role-respawn.js +318 -0
  319. package/dist/src/orgrt/role-respawn.js.map +1 -0
  320. package/dist/src/orgrt/role-session-opts.d.ts +389 -0
  321. package/dist/src/orgrt/role-session-opts.d.ts.map +1 -0
  322. package/dist/src/orgrt/role-session-opts.js +289 -0
  323. package/dist/src/orgrt/role-session-opts.js.map +1 -0
  324. package/dist/src/orgrt/runner-resolve.d.ts +43 -0
  325. package/dist/src/orgrt/runner-resolve.d.ts.map +1 -0
  326. package/dist/src/orgrt/runner-resolve.js +77 -0
  327. package/dist/src/orgrt/runner-resolve.js.map +1 -0
  328. package/dist/src/orgrt/session-gate.d.ts +34 -0
  329. package/dist/src/orgrt/session-gate.d.ts.map +1 -0
  330. package/dist/src/orgrt/session-gate.js +74 -0
  331. package/dist/src/orgrt/session-gate.js.map +1 -0
  332. package/dist/src/orgrt/session-prompt.d.ts +31 -0
  333. package/dist/src/orgrt/session-prompt.d.ts.map +1 -0
  334. package/dist/src/orgrt/session-prompt.js +115 -0
  335. package/dist/src/orgrt/session-prompt.js.map +1 -0
  336. package/dist/src/orgrt/session-run.d.ts +18 -0
  337. package/dist/src/orgrt/session-run.d.ts.map +1 -0
  338. package/dist/src/orgrt/session-run.js +429 -0
  339. package/dist/src/orgrt/session-run.js.map +1 -0
  340. package/dist/src/orgrt/session-stream.d.ts +21 -0
  341. package/dist/src/orgrt/session-stream.d.ts.map +1 -0
  342. package/dist/src/orgrt/session-stream.js +173 -0
  343. package/dist/src/orgrt/session-stream.js.map +1 -0
  344. package/dist/src/orgrt/session-types.d.ts +219 -0
  345. package/dist/src/orgrt/session-types.d.ts.map +1 -0
  346. package/dist/src/orgrt/session-types.js +2 -0
  347. package/dist/src/orgrt/session-types.js.map +1 -0
  348. package/dist/src/orgrt/session-usage.d.ts +29 -0
  349. package/dist/src/orgrt/session-usage.d.ts.map +1 -0
  350. package/dist/src/orgrt/session-usage.js +64 -0
  351. package/dist/src/orgrt/session-usage.js.map +1 -0
  352. package/dist/src/orgrt/session.d.ts +5 -281
  353. package/dist/src/orgrt/session.d.ts.map +1 -1
  354. package/dist/src/orgrt/session.js +8 -1117
  355. package/dist/src/orgrt/session.js.map +1 -1
  356. package/dist/src/ui/org-hil.mjs +1 -1
  357. package/dist/src/ui/routes-org-agents.mjs +318 -0
  358. package/dist/src/ui/routes-org-approvals.mjs +133 -0
  359. package/dist/src/ui/routes-org-config.mjs +459 -0
  360. package/dist/src/ui/routes-org-control.mjs +300 -0
  361. package/dist/src/ui/routes-org-files.mjs +172 -0
  362. package/dist/src/ui/routes-org-helpers.mjs +84 -0
  363. package/dist/src/ui/routes-org-knowledge.mjs +262 -0
  364. package/dist/src/ui/routes-org-lifecycle.mjs +224 -0
  365. package/dist/src/ui/routes-org-live.mjs +278 -0
  366. package/dist/src/ui/routes-org-mastermind.mjs +346 -0
  367. package/dist/src/ui/routes-org-planning.mjs +398 -0
  368. package/dist/src/ui/routes-org-runs.mjs +430 -0
  369. package/dist/src/ui/routes-org-status.mjs +342 -0
  370. package/dist/src/ui/routes-org-workflows.mjs +212 -0
  371. package/dist/src/ui/routes-org.mjs +26 -3847
  372. package/dist/src/ui/server.mjs +1 -1
  373. package/dist/src/utils/native-error.d.ts +1 -1
  374. package/dist/src/utils/native-error.js +1 -1
  375. package/dist/src/utils/paths.d.ts +2 -1
  376. package/dist/src/utils/paths.d.ts.map +1 -1
  377. package/dist/src/utils/paths.js +2 -1
  378. package/dist/src/utils/paths.js.map +1 -1
  379. package/dist/tsconfig.tsbuildinfo +1 -1
  380. package/org-skills/adversarial-reviewer/SKILL.md +1 -1
  381. package/org-skills/board-deck-builder/SKILL.md +1 -1
  382. package/org-skills/code-reviewer/SKILL.md +1 -1
  383. package/org-skills/debugging-wizard/SKILL.md +1 -1
  384. package/org-skills/scientific-thinking-literature-review/SKILL.md +1 -1
  385. package/org-skills/scientific-thinking-scholar-evaluation/SKILL.md +1 -1
  386. package/package.json +5 -5
@@ -1,64 +1,25 @@
1
1
  // packages/@monomind/cli/src/orgrt/daemon.ts
2
2
  // monolean: single-process inter-org — upgrade path = daemon-to-daemon HTTP when multi-host is real
3
- import { execFileSync } from 'node:child_process';
4
- import { randomUUID } from 'node:crypto';
5
- import { existsSync, mkdirSync, readFileSync, unlinkSync } from 'node:fs';
3
+ import { existsSync } from 'node:fs';
6
4
  import { isAbsolute, join } from 'node:path';
7
- import { resolveOrgDefBlueprints } from '../catalog/blueprints.js';
8
- import { writeJsonFileAtomic } from '../utils/json-file.js';
9
- import { configureResourceLimits, getResourceLimits, reapOrphanedSdkProcesses, } from '../utils/resource-governor.js';
10
- import { AntigravityAgentRunner } from './antigravity-runner.js';
11
5
  // ── Extracted module imports ────────────────────────────────────────────
12
6
  import * as approvalOps from './approvals.js';
13
- import { wakeDueBlockRechecks } from './block-recheck.js';
14
- import { BrokerLease, normalizeCredential } from './broker.js';
15
- import { onBudgetBusEvent, reopenBudgetClosedRoles, rolesOnDefTokenCaps, } from './budget-closure.js';
16
- import { OrgBus } from './bus.js';
17
- import { captureCheckpoint, generateChecksum, isCheckpointExpired, migrateCheckpoint, restoredRoleStatus, restoreMailboxQueue, validateCheckpoint, } from './checkpoint.js';
18
7
  import * as checkpointOps from './checkpoint-ops.js';
19
- import { CodexAgentRunner } from './codex-runner.js';
20
- import { checkCompletion } from './completion-gate.js';
21
- import { CopilotAgentRunner } from './copilot-runner.js';
22
8
  import * as crossOrg from './cross-org.js';
23
- import { CrushAgentRunner } from './crush-runner.js';
24
9
  import * as decisionOps from './decisions.js';
25
- import { openTaskCount } from './decisions.js';
26
- import { agentRoles, hasActiveEndpointWait, isEndpointRole, retryQueuedEndpoints, startEndpointRetryLoop, stopEndpointRetries, } from './endpoint-roles.js';
27
- import { createFenceForRole, loadGlobalFenceConfig, mergeFenceConfigs, } from './fence.js';
28
- import { fileToolRoots } from './file-roots.js';
29
- import { attachForwarder } from './forwarder.js';
30
- import { GrokAgentRunner } from './grok-runner.js';
31
- import { HermesAgentRunner } from './hermes-runner.js';
32
- import { advanceHold, clearIdleRecord, hookedOnWork, noProgressRoles, projectIdleStop, writeIdleRecord, } from './idle-deadline.js';
33
- import { drainInbox, newMessageId, queueMessage } from './inbox.js';
34
- import { KimiCodeAgentRunner } from './kimicode-runner.js';
35
- import { loadoutCatalog, resolveLoadout, sessionLoadoutFor, taskTag } from './loadouts.js';
36
- import { isRecoverableCloseReason, Mailbox } from './mailbox.js';
37
- import { OpencodeAgentRunner } from './opencode-runner.js';
38
10
  import * as orgMemory from './org-memory.js';
39
- import { PiRpcAgentRunner } from './pi-rpc-runner.js';
40
- import { PiAgentRunner } from './pi-runner.js';
41
- import { PolicyEngine } from './policy.js';
42
- import { expandOrgPolicyPathVars, promptVarsFor } from './prompt-vars.js';
43
- import { resolveRoleProvider } from './provider.js';
11
+ import * as orgReload from './org-reload.js';
12
+ import * as orgStart from './org-start.js';
13
+ import * as orgStateFile from './org-state-file.js';
14
+ import * as orgStop from './org-stop.js';
44
15
  import * as questionOps from './questions.js';
45
- import { QwenRpcAgentRunner } from './qwen-rpc-runner.js';
46
- import { QwenAgentRunner } from './qwen-runner.js';
47
- import { historyFile, readHistory, readRunEvents, summarizeRun, } from './reporting.js';
48
- import { buildRespawnReceipt, computeReplacementBudget, mergeEffectiveRoleConfig, redactRoleConfig, validateRespawnInput, } from './role-slot.js';
49
- import { currentRoleTrace, endTurn, withTrace } from './role-trace.js';
16
+ import * as roleIncarnation from './role-incarnation.js';
17
+ import * as roleRespawn from './role-respawn.js';
18
+ import { currentRoleTrace } from './role-trace.js';
50
19
  import { buildRuntimeOptions } from './runtime-options.js';
51
- import { sandboxStubs } from './sandbox-stubs.js';
52
20
  import * as scheduler from './scheduler-integration.js';
53
- import { runAgentSession } from './session.js';
54
- import { SessionLedger } from './session-ledger.js';
55
- import { effectiveToolProviders } from './skill-library.js';
56
- import { TaskProcesses } from './task-cancel.js';
57
- import { TaskDag } from './task-dag.js';
58
- import { resolveAutoAssignee } from './task-match.js';
59
- import { roleProviderPrefixes, ToolProviderHub } from './tool-providers.js';
60
- import { ORG_DIR, OrgDefSchema, } from './types.js';
61
- import { VercelAgentRunner } from './vercel-runner.js';
21
+ import { ToolProviderHub } from './tool-providers.js';
22
+ import { ORG_DIR, } from './types.js';
62
23
  /** OpenTelemetry tracing helper - creates spans for major operations */
63
24
  class _OtelTracer {
64
25
  enabled = false;
@@ -87,178 +48,11 @@ class _OtelTracer {
87
48
  // Could emit to bus for collection
88
49
  }
89
50
  }
90
- /** Drain window for a PLANNED stop (the boss called org_complete). Long enough
91
- * for a sibling mid-build or mid-test to finish and flush its work. A hard
92
- * stop keeps the short bound — see finishStop. */
93
- const COMPLETE_DRAIN_MS = 5 * 60_000;
94
- /** Auto-resolve runtime from provider kind. Returns undefined for Claude default. */
95
- function autoRuntimeFromProvider(kind) {
96
- if (kind === 'vercel-api-key')
97
- return 'vercel';
98
- if (kind === 'codex')
99
- return 'codex';
100
- if (kind === 'antigravity')
101
- return 'antigravity';
102
- return undefined;
103
- }
104
- export function resolveRunner(orgRuntime, providerKind, provider) {
105
- const selected = orgRuntime ??
106
- autoRuntimeFromProvider(providerKind) ??
107
- process.env.MONOMIND_RUNTIME;
108
- if (selected === 'opencode')
109
- return new OpencodeAgentRunner();
110
- if (selected === 'kimicode')
111
- return new KimiCodeAgentRunner();
112
- if (selected === 'vercel')
113
- return new VercelAgentRunner();
114
- if (selected === 'codex')
115
- return new CodexAgentRunner();
116
- if (selected === 'antigravity')
117
- return new AntigravityAgentRunner();
118
- if (selected === 'grok')
119
- return new GrokAgentRunner();
120
- if (selected === 'qwen')
121
- return new QwenAgentRunner();
122
- if (selected === 'crush') {
123
- // Issue #177: usage-proxy accounting is opt-in via provider.usageProxy +
124
- // provider.baseUrl (the upstream the crush CLI's own provider config
125
- // points at). Absent either, CrushAgentRunner falls back to its
126
- // documented 0-token behavior — this never blocks a turn either way.
127
- if (provider?.usageProxy && provider.baseUrl) {
128
- return new CrushAgentRunner({
129
- usageProxy: { upstreamBaseUrl: provider.baseUrl, baseUrlEnvVar: provider.usageProxyEnvVar },
130
- });
131
- }
132
- return new CrushAgentRunner();
133
- }
134
- if (selected === 'copilot')
135
- return new CopilotAgentRunner();
136
- if (selected === 'pi')
137
- return new PiAgentRunner();
138
- if (selected === 'pi-rpc')
139
- return new PiRpcAgentRunner();
140
- if (selected === 'qwen-rpc')
141
- return new QwenRpcAgentRunner();
142
- if (selected === 'hermes')
143
- return new HermesAgentRunner();
144
- return undefined;
145
- }
146
- /** Per-session variant: a role's own `runtime` field wins over the org-level
147
- * one (and the env var) — including `role.runtime === 'claude'`, which forces
148
- * the default Claude path even when the org/env select another runtime.
149
- * Roles without a `runtime` inherit the org-level resolution unchanged.
150
- * If no explicit runtime is set, auto-resolve from the provider kind. */
151
- export function resolveRoleRunner(roleRuntime, orgRuntime, roleProviderKind, orgProviderKind, roleProvider) {
152
- const explicit = roleRuntime ?? orgRuntime;
153
- if (explicit)
154
- return resolveRunner(explicit, undefined, roleProvider);
155
- return resolveRunner(undefined, roleProviderKind ?? orgProviderKind, roleProvider);
156
- }
51
+ export { activeRoleCount, roleTokenBudget, ScrollbackBuffer, } from './daemon-types.js';
52
+ export { resolvedIdleNudgeCount } from './idle-watchdog.js';
53
+ export { resolveOrgComplete } from './role-session-opts.js';
54
+ export { resolveRoleRunner, resolveRunner, } from './runner-resolve.js';
157
55
  export { resolveAutoAssignee } from './task-match.js';
158
- /** Per-role token budget: a role's own `budget_tokens` wins; otherwise the
159
- * even split of run_config.budget_tokens across all roles. */
160
- export function roleTokenBudget(role, def) {
161
- return (role.budget_tokens ??
162
- Math.floor((def.run_config.budget_tokens ?? 1_000_000) / Math.max(1, agentRoles(def.roles).length)));
163
- }
164
- /** Idle watchdog's per-tick recovery check: given the previous nudge timestamp,
165
- * the cumulative nudge count, and the timestamp of the most recent real tool
166
- * call, returns the nudge count that should carry forward now that fresh
167
- * activity means the org is no longer idle.
168
- *
169
- * `nudgedAt !== 0` means we're recovering from an outstanding nudge. Resetting
170
- * the counter here means it only ever tracks UNRESOLVED idle spells in a row,
171
- * not a lifetime total — a long-running org that goes idle and recovers any
172
- * number of times (e.g. periodic checkpoints on a slow background task) is
173
- * never punished for having had several separate, healthy idle spells over
174
- * its lifetime. Before this existed, `nudges` only ever incremented, so a run
175
- * that answered every single nudge with real work still hit the "org idle
176
- * again after 3 nudges" cap and got force-stopped on its 4th idle spell —
177
- * observed live killing an in-progress 24h soak test under 90 minutes in.
178
- *
179
- * But "recovering" must mean genuine forward progress, not just any bus
180
- * event — a bare, content-free reply to the nudge (a boss that answers with
181
- * "✓ Complete" and calls no tools at all) still updates lastActivity, so the
182
- * org isn't flagged as silent, but it accomplishes nothing: nobody outside
183
- * the boss's own turn ever sees it, since role coordination only happens via
184
- * tool calls (org_send, org_task, ...). Requiring `lastToolActivity >=
185
- * nudgedAt` — a real tool call happened AFTER this nudge was sent — closes
186
- * that gap: observed live, a boss stuck responding to four consecutive
187
- * 10-minute nudges with one-line acknowledgments and zero tool calls looped
188
- * indefinitely making no progress, because every trivial reply reset the cap
189
- * that was supposed to catch exactly this. */
190
- export function resolvedIdleNudgeCount(nudgedAt, nudges, lastToolActivity) {
191
- return nudgedAt !== 0 && lastToolActivity >= nudgedAt ? 0 : nudges;
192
- }
193
- /** Bounded ring buffer for agent terminal scrollback. */
194
- export class ScrollbackBuffer {
195
- maxLines;
196
- lines = [];
197
- constructor(maxLines = 500) {
198
- this.maxLines = maxLines;
199
- }
200
- push(line) {
201
- this.lines.push(line);
202
- if (this.lines.length > this.maxLines)
203
- this.lines.splice(0, this.lines.length - this.maxLines);
204
- }
205
- snapshot() {
206
- return [...this.lines];
207
- }
208
- clear() {
209
- this.lines.length = 0;
210
- }
211
- }
212
- /** Bug 4: number of roles for this org that are actually spawned and running
213
- * right now — the live count run_config.max_concurrent_agents caps. A role
214
- * that crashed or ended no longer counts, so a slot frees up automatically
215
- * the moment that happens; nothing needs to explicitly decrement a counter. */
216
- export function activeRoleCount(org) {
217
- let n = 0;
218
- for (const rt of org.agents.values())
219
- if (rt.status === 'running')
220
- n++;
221
- return n;
222
- }
223
- /** #302: the `org_complete` consent gate's emit-and-return logic, extracted
224
- * (matching `resolvedIdleNudgeCount`'s and `runOutcomeResult`'s precedent)
225
- * so it is unit-testable without a live daemon, a running org, or the SDK's
226
- * own tool-calling loop — none of which a test can drive directly, since
227
- * `queryFn` replaces the whole SDK `query()`, tool orchestration included.
228
- * `onComplete` gathers the facts (needs `this`/`running`) and calls this;
229
- * this does the one decision (`checkCompletion`) and its two possible
230
- * side effects. Returns the refusal string, or `null` to allow — exactly
231
- * what the org_complete tool handler (session.ts) relays as the result. */
232
- export function resolveOrgComplete(bus, role, outcome, summary, blocker, blockerDetail, runFacts) {
233
- const refusal = checkCompletion({ outcome, blocker, blockerDetail, ...runFacts });
234
- if (refusal) {
235
- // Visible in `org logs` — a silent refusal reads to an operator as a
236
- // hung boss, not a boss that was told no.
237
- bus.emit({
238
- type: 'audit',
239
- from: role,
240
- reason: 'org-complete-refused',
241
- msg: refusal,
242
- data: { outcome, blocker, blockerDetail },
243
- });
244
- return refusal;
245
- }
246
- // #302 AC6: the blocker must show up in RENDERED output, not just `data`
247
- // — `org logs`'s formatter prints an event's `msg` verbatim and never
248
- // looks at `data`, so a blocker recorded only there would satisfy a unit
249
- // assertion and never reach a human reading the actual log.
250
- const blockerSuffix = blocker
251
- ? ` (blocker: ${blocker}${blockerDetail ? ` — ${blockerDetail}` : ''})`
252
- : '';
253
- bus.emit({
254
- type: 'status',
255
- from: role,
256
- reason: 'org-complete',
257
- msg: `run outcome: ${outcome}${blockerSuffix}`,
258
- data: { outcome, summary, blocker, blockerDetail },
259
- });
260
- return null;
261
- }
262
56
  export class OrgDaemon {
263
57
  root;
264
58
  opts;
@@ -290,6 +84,7 @@ export class OrgDaemon {
290
84
  /** @internal */ restarting = new Set();
291
85
  // #3: recognizes provider context-window-overflow errors so the boss can be told
292
86
  // to chunk the work instead of re-dispatching the same oversized task verbatim.
87
+ /** @internal */
293
88
  static CONTEXT_LIMIT_RE = /context[- ]?(window|length|size|limit)|maximum context|exceeds?.{0,12}(context|token)|too many tokens|prompt is too long/i;
294
89
  /** @internal */ recallUsage = new Map();
295
90
  /** @internal */ orgLearnedRuns = new Set();
@@ -328,104 +123,7 @@ export class OrgDaemon {
328
123
  * Removed roles are NOT killed — they finish their current work and won't be re-spawned.
329
124
  * Returns a summary of what changed. */
330
125
  reloadOrgDef(name) {
331
- const running = this.orgs.get(name);
332
- if (!running)
333
- throw new Error(`org ${name} is not running`);
334
- const defPath = join(this.root, ORG_DIR, `${name}.json`);
335
- const parsedDef = OrgDefSchema.parse(JSON.parse(readFileSync(defPath, 'utf8')));
336
- const bp = resolveOrgDefBlueprints(parsedDef, this.root);
337
- if (bp.errors.length)
338
- throw new Error(`org ${name}: ${bp.errors.join('; ')}`);
339
- const newDef = expandOrgPolicyPathVars(bp.def, promptVarsFor(this.root));
340
- const changed = [];
341
- const newRoles = [];
342
- const removedRoles = [];
343
- const onDefTokenCaps = rolesOnDefTokenCaps(running);
344
- if (newDef.goal !== running.def.goal) {
345
- running.def.goal = newDef.goal;
346
- changed.push('goal');
347
- }
348
- const oldRc = running.def.run_config;
349
- const newRc = newDef.run_config;
350
- for (const key of new Set([...Object.keys(oldRc), ...Object.keys(newRc)])) {
351
- if (JSON.stringify(oldRc[key]) !== JSON.stringify(newRc[key])) {
352
- oldRc[key] = newRc[key];
353
- changed.push(`run_config.${key}`);
354
- }
355
- }
356
- // M1 (C-37): apply changes to EXISTING roles' tool_providers, endpoint,
357
- // kind and policy. Fields are replaced on the live role object (sessions
358
- // read tool_providers at their next start, checkApproval reads policy
359
- // live) and a running role's PolicyEngine gets the new policy now.
360
- // #343: budget_usd / budget_tokens too — the live PolicyEngine gets the
361
- // new caps with its spend kept, and a role closed for budget reopens
362
- // below once it is no longer over them.
363
- const RELOADABLE_ROLE_FIELDS = [
364
- 'tool_providers',
365
- 'endpoint',
366
- 'kind',
367
- 'policy',
368
- 'budget_usd',
369
- 'budget_tokens',
370
- ];
371
- for (const next of newDef.roles) {
372
- const live = running.def.roles.find((r) => r.id === next.id);
373
- if (!live)
374
- continue;
375
- const liveRec = live;
376
- const nextRec = next;
377
- for (const field of RELOADABLE_ROLE_FIELDS) {
378
- if (JSON.stringify(liveRec[field]) === JSON.stringify(nextRec[field]))
379
- continue;
380
- const targets = new Set([liveRec]);
381
- const slotRole = running.roleSlots.get(next.id)?.effectiveRole;
382
- if (slotRole)
383
- targets.add(slotRole);
384
- const pending = running.pendingRoles?.get(next.id);
385
- if (pending)
386
- targets.add(pending);
387
- for (const t of targets) {
388
- if (nextRec[field] === undefined)
389
- delete t[field];
390
- else
391
- t[field] = nextRec[field];
392
- }
393
- if (field === 'policy')
394
- running.agents.get(next.id)?.policy.updatePolicy(next.policy ?? {});
395
- if (field === 'budget_usd' || field === 'budget_tokens')
396
- running.agents.get(next.id)?.policy.setBudgetCaps({
397
- maxTokens: live.policy?.maxTokens ?? computeReplacementBudget(running.def, next.id),
398
- maxUsd: live.policy?.maxUsd ?? live.budget_usd,
399
- });
400
- changed.push(`role:${next.id}:${field}`);
401
- }
402
- }
403
- const reopened = reopenBudgetClosedRoles(this, name, running, onDefTokenCaps);
404
- const existingRoleIds = new Set(running.def.roles.map((r) => r.id));
405
- const newRoleIds = new Set(newDef.roles.map((r) => r.id));
406
- for (const role of newDef.roles) {
407
- if (!existingRoleIds.has(role.id)) {
408
- running.def.roles.push(role);
409
- // M2: an endpoint role never gets a session — nothing to lazy-spawn.
410
- if (!isEndpointRole(role)) {
411
- if (!running.pendingRoles)
412
- running.pendingRoles = new Map();
413
- running.pendingRoles.set(role.id, role);
414
- }
415
- newRoles.push(role.id);
416
- }
417
- }
418
- for (const id of existingRoleIds) {
419
- if (!newRoleIds.has(id))
420
- removedRoles.push(id);
421
- }
422
- running.bus.emit({
423
- type: 'audit',
424
- reason: 'hot-reload',
425
- msg: `org def reloaded: ${changed.length} fields changed, ${newRoles.length} new roles, ${removedRoles.length} removed roles${reopened.length ? `, reopened ${reopened.join(', ')}` : ''}`,
426
- data: { changed, newRoles, removedRoles },
427
- });
428
- return { changed, newRoles, removedRoles };
126
+ return orgReload.reloadOrgDef(this, name);
429
127
  }
430
128
  /** Names of the orgs this daemon currently has running. Snapshot — safe to
431
129
  * iterate while stopOrg() mutates the underlying map. */
@@ -470,6 +168,7 @@ export class OrgDaemon {
470
168
  * A relative path is resolved against the project root rather than the
471
169
  * daemon's cwd, which is not the same directory when `org serve` is started
472
170
  * from a subdirectory. */
171
+ /** @internal */
473
172
  workspaceSetting(def) {
474
173
  const ws = def.run_config.workspace ?? 'repo';
475
174
  if (ws === 'repo' || ws === 'isolated' || ws === 'worktree' || ws === 'worktree-per-role')
@@ -477,921 +176,7 @@ export class OrgDaemon {
477
176
  return isAbsolute(ws) ? ws : join(this.root, ws);
478
177
  }
479
178
  async startOrg(name, taskOverride, options) {
480
- // A restart-driven start (scheduleBossRestart) keeps its crash counter so the
481
- // cap holds; any other (explicit) start resets it so a manual re-run gets a
482
- // fresh budget.
483
- if (!this.restarting.has(name))
484
- this.bossRestartCounts.delete(name);
485
- // Join any in-flight stop for this org before checking `this.orgs` — otherwise a
486
- // start racing a stop's drain window (up to stopWaitMs) can share the stopping
487
- // run's worktree path while it's still being force-removed.
488
- const inflightStop = this.stopping.get(name);
489
- if (inflightStop)
490
- await inflightStop;
491
- // Bug 2 (TOCTOU race): this existence check is synchronous, but the real
492
- // registration into `this.orgs` doesn't happen until deep inside
493
- // startOrgInner, after several genuine `await` points (the provider
494
- // validation dynamic import, `git worktree add` for workspace:
495
- // 'worktree'). Two concurrent startOrg(name) calls — e.g. the
496
- // scheduler's tick, the runfile poll loop, and autoWake firing close
497
- // together — could each pass this check before either registered,
498
- // spawning two duplicate runs with separate budget/policy counters that
499
- // both write the same shared per-org files. Reserve the name in
500
- // `startingOrgs` synchronously, in the same tick as the check, so a
501
- // second concurrent call sees the reservation and is rejected instead of
502
- // racing ahead to spawn a duplicate.
503
- if (this.orgs.has(name))
504
- throw new Error(`org ${name} already running`);
505
- if (this.startingOrgs.has(name))
506
- throw new Error(`org ${name} already starting`);
507
- this.startingOrgs.add(name);
508
- // #345: the run's --auto-approve list — kept by a boss auto-restart like
509
- // the crash counter, replaced by any other start. Set only after the
510
- // guards above, so a rejected start can't rewrite a live run's list.
511
- if (!this.restarting.has(name)) {
512
- if (options?.autoApprove?.length)
513
- this.runAutoApprove.set(name, options.autoApprove);
514
- else
515
- this.runAutoApprove.delete(name);
516
- }
517
- try {
518
- return await this.startOrgInner(name, taskOverride, options);
519
- }
520
- catch (err) {
521
- // startOrgInner registers the org in `this.orgs` (and spawns the boss,
522
- // installs the exit listener, starts the broker lease) well before it
523
- // returns; persistState (ENOSPC/EACCES) and BrokerLease.start() can
524
- // still throw after that. Left alone, that was a live, unreachable org:
525
- // sessions running, `this.orgs` still holding it, every later startOrg
526
- // rejected with "already running", and nothing ever calling stopOrg.
527
- // Only this call can have registered the name (the reservation above
528
- // holds until `finally`), so anything in the map is ours to tear down.
529
- if (this.orgs.has(name)) {
530
- // #302: tag the real cause so a run's history/report can never read
531
- // this as a boss-attributed outcome — nothing here asked the boss.
532
- await this.stopOrg(name, { closedBy: 'failed-start' }).catch((stopErr) => console.error(`org ${name}: teardown after failed start failed:`, stopErr instanceof Error ? stopErr.message : stopErr));
533
- }
534
- throw err;
535
- }
536
- finally {
537
- this.startingOrgs.delete(name);
538
- }
539
- }
540
- /** The actual startOrg implementation. Split out of startOrg() so the
541
- * reservation guard above runs synchronously, before any `await` in here —
542
- * see the bug 2 comment in startOrg(). */
543
- async startOrgInner(name, taskOverride, options) {
544
- // #301: the PRIMARY fix — the stop-side backstops below (finishStop,
545
- // process 'exit') can only run for a run that ends through code we
546
- // control; a SIGKILL leaves .git/worktrees/<name> metadata behind with
547
- // nothing left to clean it up, and so did every run that leaked before
548
- // this fix existed. Pruning here turns "did the last run clean up,
549
- // however it died?" into "a run always begins clean" — the only thing
550
- // that recovers both the SIGKILL case and worktrees already orphaned
551
- // before this daemon process started. Unconditional and best-effort for
552
- // the same reasons as the stop-side prune (see finishStop): it only
553
- // drops metadata whose worktree directory is already gone, so it cannot
554
- // touch a live worktree, including the owner's.
555
- try {
556
- execFileSync('git', ['worktree', 'prune'], {
557
- cwd: this.root,
558
- stdio: 'ignore',
559
- timeout: 30_000,
560
- });
561
- }
562
- catch {
563
- /* best-effort: not a git repo, git missing, or a wedged hook */
564
- }
565
- const defPath = join(this.root, ORG_DIR, `${name}.json`);
566
- const parsedDef = OrgDefSchema.parse(JSON.parse(readFileSync(defPath, 'utf8')));
567
- const bp = resolveOrgDefBlueprints(parsedDef, this.root);
568
- // {{home}} / {{org_root}} in policy paths, before any root or sandbox sees them.
569
- const def = expandOrgPolicyPathVars(bp.def, promptVarsFor(this.root));
570
- const autoApproveError = approvalOps.unknownAutoApproveError(options?.autoApprove ?? [], def.roles);
571
- if (autoApproveError)
572
- throw new Error(autoApproveError);
573
- let run;
574
- let checkpoint;
575
- if (options?.resume) {
576
- const rtPath = join(this.root, ORG_DIR, name, 'runtime.json');
577
- if (!existsSync(rtPath))
578
- throw new Error(`cannot resume org "${name}": runtime.json not found`);
579
- const rt = JSON.parse(readFileSync(rtPath, 'utf8'));
580
- if (!rt?.run || !rt?.checkpoint)
581
- throw new Error(`cannot resume org "${name}": no valid checkpoint found`);
582
- if (isCheckpointExpired(rt.checkpoint))
583
- throw new Error(`cannot resume org "${name}": checkpoint expired`);
584
- // Migrate an older-schema checkpoint (verifying ITS OWN stored checksum
585
- // first) before validating it against CHECKPOINT_VERSION — see
586
- // migrateCheckpoint's doc comment in checkpoint.ts.
587
- const migrated = migrateCheckpoint(rt.checkpoint);
588
- if (!migrated || !validateCheckpoint(migrated))
589
- throw new Error(`cannot resume org "${name}": checkpoint validation failed`);
590
- rt.checkpoint = migrated;
591
- run = rt.run;
592
- checkpoint = rt.checkpoint;
593
- if (rt.abandonedRoles) {
594
- this.abandoned.set(name, new Set(rt.abandonedRoles));
595
- }
596
- }
597
- else {
598
- this.abandoned.delete(name); // a previous run's missing roles say nothing about this one
599
- this.memoryErrors.delete(name); // nor does its memory-store failure (#293)
600
- approvalOps.clearApprovalsForFreshStart(this, name); // a previous run's approvals are moot for this one
601
- questionOps.clearQuestionsForFreshStart(this, name); // nor do its unanswered questions (#248)
602
- // random suffix: second-precision stamps collide across processes (two CLI
603
- // invocations in the same second would share a run dir and its bus.jsonl)
604
- run = `run-${new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14)}-${Math.random().toString(36).slice(2, 6)}`;
605
- }
606
- const dir = join(this.root, ORG_DIR, name, run);
607
- mkdirSync(dir, { recursive: true });
608
- // Role sessions run at the project root by default. They used to run in an
609
- // empty scratch dir under .monomind/orgs/<name>/workspace, which the policy
610
- // engine's workdir check ("path escapes org workdir") then confined every
611
- // path to — so a development org could not Read or Edit a single file of
612
- // the project it was created to work on. Roles fell back to Bash, which is
613
- // not path-scoped, meaning the sandbox blocked the safe tools and let the
614
- // unrestricted one through. Opt back in with run_config.workspace:
615
- // 'isolated', or pin an absolute path.
616
- const ws = this.workspaceSetting(def);
617
- let cwd;
618
- let worktreePath;
619
- if (ws === 'worktree') {
620
- worktreePath = join(this.root, ORG_DIR, name, 'worktree');
621
- const { execFileSync } = await import('node:child_process');
622
- try {
623
- // Remove stale worktree from a previous run
624
- if (existsSync(worktreePath)) {
625
- // R4: bound the call — a wedged git hook (git-lfs, gc lock, gpg sign
626
- // prompt) would otherwise hang the whole daemon indefinitely.
627
- // SEC-5: execFileSync + argv — no shell interpolation of worktreePath.
628
- execFileSync('git', ['worktree', 'remove', '--force', worktreePath], {
629
- cwd: this.root,
630
- stdio: 'ignore',
631
- timeout: 30_000,
632
- });
633
- }
634
- }
635
- catch {
636
- /* best-effort cleanup */
637
- }
638
- execFileSync('git', ['worktree', 'add', worktreePath, 'HEAD', '--detach'], {
639
- cwd: this.root,
640
- stdio: 'ignore',
641
- timeout: 30_000,
642
- });
643
- cwd = worktreePath;
644
- }
645
- else {
646
- cwd =
647
- ws === 'repo'
648
- ? this.root
649
- : ws === 'isolated'
650
- ? join(this.root, ORG_DIR, name, 'workspace')
651
- : ws;
652
- }
653
- mkdirSync(cwd, { recursive: true });
654
- // An org must be able to seat its whole roster. maxSdkProcesses is sized for
655
- // the machine (cpus - 2), so any org with more roles than that had its tail
656
- // roles deferred forever — a 7-role org on an 8-core box permanently lost
657
- // its 7th, and the work that role owned simply never happened. Raise the
658
- // ceiling to the role count. An explicit MONOMIND_MAX_SDK_PROCS still wins:
659
- // if the operator named a number, that number is the answer.
660
- //
661
- // Runner process model (relevant for sizing): ClaudeAgentRunner and
662
- // VercelAgentRunner are in-process (no subprocess per role). KimiCodeAgentRunner,
663
- // OpencodeAgentRunner, and CodexAgentRunner each spawn one subprocess per role.
664
- // The current sizing (def.roles.length) is therefore safe — it over-provisions
665
- // for in-process runners but never under-provisions for subprocess runners.
666
- const sessionRoleCount = agentRoles(def.roles).length;
667
- if (!process.env.MONOMIND_MAX_SDK_PROCS &&
668
- getResourceLimits().maxSdkProcesses < sessionRoleCount) {
669
- configureResourceLimits({ maxSdkProcesses: sessionRoleCount });
670
- }
671
- // ADR-O001 D8: a cost tier that can't resolve a model for a role's
672
- // provider must stop the run here. The alternative — resolving it at
673
- // session start — would either crash one role ten minutes in or, worse,
674
- // quietly leave that role on a different model than the tier claimed.
675
- const { validateCostTiers } = await import('./cost-tier.js');
676
- const tierErrors = validateCostTiers(def);
677
- if (tierErrors.length) {
678
- throw new Error(`org ${name}: ${tierErrors.join('; ')}`);
679
- }
680
- // ADR-O001 D7: an oversized or unresolvable loadout catalog stops the run
681
- // here, for the same reason — not ten minutes in, at some role's spawn.
682
- const { validateLoadouts } = await import('./loadouts.js');
683
- const loadoutErrors = validateLoadouts(def, this.root).errors;
684
- if (loadoutErrors.length) {
685
- throw new Error(`org ${name}: ${loadoutErrors.join('; ')}`);
686
- }
687
- const { validateRoleSkills } = await import('./skill-library.js');
688
- const skillErrors = [
689
- ...bp.errors,
690
- ...def.roles.flatMap((r) => validateRoleSkills(r, this.root)),
691
- ];
692
- if (skillErrors.length) {
693
- throw new Error(`org ${name}: ${skillErrors.join('; ')}`);
694
- }
695
- // Validate per-role providers before spawning anything (fail-fast: a
696
- // missing env var discovered 10 minutes into a run wastes the entire run).
697
- const { resolveProviderEnv: validateProvider, resolveRoleProvider } = await import('./provider.js');
698
- for (const role of def.roles) {
699
- try {
700
- if (role.provider) {
701
- validateProvider(role.provider);
702
- }
703
- else if (role.adapter_config?.provider) {
704
- // Named provider (`monomind providers configure`): resolve now so a
705
- // missing/misconfigured entry fails the run at start, not mid-flight.
706
- resolveRoleProvider(role, this.root);
707
- }
708
- }
709
- catch (err) {
710
- throw new Error(`org ${name}: role "${role.id}" provider validation failed — ${err instanceof Error ? err.message : err}`);
711
- }
712
- // provider.kind 'gemini'/'openai' only sets env vars (GEMINI_API_KEY /
713
- // OPENAI_API_KEY — see provider.ts) for a CLI that never reads them:
714
- // autoRuntimeFromProvider has no case for either kind, so
715
- // resolveRoleRunner falls through to `undefined` and session.ts spawns
716
- // the default ClaudeAgentRunner. The role silently runs on Claude while
717
- // its config claims gemini/openai — surface that loudly at start time
718
- // instead of leaving it to be discovered mid-run.
719
- const kind = role.provider?.kind;
720
- if ((kind === 'gemini' || kind === 'openai') &&
721
- !resolveRoleRunner(role.runtime, def.runtime, kind, undefined, role.provider)) {
722
- console.error(`org ${name}: role "${role.id}" sets provider.kind="${kind}" but no runtime honors it — ` +
723
- `this role will actually run on the Claude Agent SDK, not ${kind}. ` +
724
- `Set role.runtime (or the org's runtime) explicitly, or use provider.kind="vercel-api-key" ` +
725
- `with vendor="${kind === 'gemini' ? 'google' : 'openai'}" to route through a real ${kind} model.`);
726
- }
727
- }
728
- const bus = new OrgBus(name, run, dir);
729
- // Lightweight in-memory tail for busEvents() (test-loop, /api/history).
730
- // Full events (including Write content snapshots) live on disk in bus.jsonl;
731
- // the in-memory copy strips bulky data.content to keep RAM flat.
732
- const MAX_COLLECTED = 1000;
733
- const collected = [];
734
- let lastActivity = Date.now();
735
- // Separate from lastActivity: only real tool calls (org_send, org_task,
736
- // Bash, ...) count here, not status pings or chat-only turns. The idle
737
- // watchdog's nudge-recovery check (resolvedIdleNudgeCount) uses this to
738
- // tell genuine forward progress apart from a boss that "answers" a nudge
739
- // with a bare acknowledgment ("✓ Complete") and does nothing — a
740
- // content-free reply still updates lastActivity (so the org isn't
741
- // flagged as silent), but must not reset the cumulative nudge cap, or a
742
- // boss that's genuinely out of ideas can loop forever making zero
743
- // progress without ever tripping the watchdog.
744
- let lastToolActivity = 0;
745
- // ADR-O001 D4 (no-progress detector): the idle clock above is org-wide and
746
- // says nothing at all while a hold is in force — the 8.3-hour stall looked
747
- // perfectly healthy from it. Per-role last-activity is what tells a role
748
- // that is nominally working but producing nothing from one that is simply
749
- // waiting its turn.
750
- const roleActivity = new Map();
751
- const noProgressAlarmed = new Set();
752
- bus.subscribe((e) => {
753
- const slim = e.data?.content != null ? { ...e, data: { ...e.data, content: undefined } } : e;
754
- collected.push(slim);
755
- if (collected.length > MAX_COLLECTED)
756
- collected.splice(0, collected.length - MAX_COLLECTED);
757
- // The watchdog's own events must not count as org activity, or a hung
758
- // boss would never trip the "nudge produced no activity" stop and a
759
- // silent role would clear its own no-progress alarm.
760
- const selfEmitted = e.reason === 'idle-nudge' || e.reason === 'no-progress' || e.reason === 'hold-expired';
761
- if (!selfEmitted)
762
- lastActivity = Date.now();
763
- if (e.type === 'tool')
764
- lastToolActivity = Date.now();
765
- if (e.from && !selfEmitted) {
766
- roleActivity.set(e.from, Date.now());
767
- noProgressAlarmed.delete(e.from);
768
- }
769
- // org_complete IS the end of the run — self-stop instead of sitting
770
- // "running" forever after a recorded outcome. Deferred (unref'd) so the
771
- // tool call's receipt reaches the boss and its final turn text still
772
- // lands on the bus before mailboxes close; stopOrg is reentrant-safe
773
- // against a concurrent manual stop.
774
- if (e.type === 'status' && e.reason === 'org-complete') {
775
- const t = setTimeout(() => {
776
- // #206: closedBy: 'org-complete' is the ONLY signal `org run`
777
- // trusts to mean "the run ended cleanly, exit 0" — every other
778
- // stop path (idle watchdog, boss-restart-exhausted, manual stop)
779
- // leaves it unset.
780
- this.stopOrg(name, { drainMs: COMPLETE_DRAIN_MS, closedBy: 'org-complete' }).catch((err) => console.error(`org ${name}: auto-stop after org_complete failed:`, err instanceof Error ? err.message : err));
781
- }, 1000);
782
- t.unref?.();
783
- }
784
- // Accumulate cost from usage events into per-role metrics
785
- if (e.type === 'usage' && e.from && e.data) {
786
- const runtime = running.agents.get(e.from);
787
- if (runtime) {
788
- const cost = Number(e.data.cost_usd ?? 0);
789
- if (Number.isFinite(cost)) {
790
- runtime.metrics.costUsd += cost;
791
- }
792
- }
793
- }
794
- // Bug 1 / #343: enforce the org-wide run_config.budget_tokens ceiling,
795
- // hold a budget-closed role's tasks, warn near a budget.
796
- onBudgetBusEvent(running, e);
797
- // Track last message ID for threading responses
798
- if ((e.type === 'message' || e.type === 'xorg') && e.from) {
799
- const runtime = running.agents.get(e.from);
800
- if (runtime) {
801
- runtime.lastMessageId = e.id;
802
- }
803
- }
804
- // Apply audit filter if configured (skip filtered tool events before forwarding)
805
- if (this.opts.auditFilter && e.type === 'tool') {
806
- const { tool, decision } = this.opts.auditFilter;
807
- if (tool && e.tool !== tool)
808
- return; // Skip: tool name doesn't match
809
- if (decision && e.decision !== decision)
810
- return; // Skip: decision doesn't match
811
- }
812
- for (const fn of this.globalSubscribers)
813
- fn(e);
814
- });
815
- if (this.opts.forward !== false)
816
- this.forwarders.set(name, attachForwarder(bus, this.opts.controlJson ?? join(this.root, '.monomind/control.json')));
817
- const running = {
818
- def,
819
- run,
820
- sessionLedger: new SessionLedger(join(dir, 'sessions.json')),
821
- gates: decisionOps.readGates(this.root, name),
822
- bus,
823
- agents: new Map(),
824
- roleSlots: new Map(),
825
- bossRoleId: '', // set below, once bossRole is computed
826
- glossary: [],
827
- respawning: new Set(),
828
- busEvents: () => [...collected],
829
- workdir: cwd,
830
- credential: randomUUID(),
831
- };
832
- this.orgs.set(name, running);
833
- // ── MonoFence guardrail: pre-create per-role instances ────────────────
834
- const globalFence = loadGlobalFenceConfig(this.root);
835
- const orgFence = def.fence;
836
- const roleFences = new Map();
837
- for (const role of def.roles) {
838
- const roleFenceCfg = role.policy?.fence;
839
- const merged = mergeFenceConfigs(globalFence ?? undefined, orgFence, roleFenceCfg);
840
- if (merged.enabled === false)
841
- continue;
842
- if (!globalFence && !orgFence && !roleFenceCfg)
843
- continue;
844
- try {
845
- const instance = await createFenceForRole(merged);
846
- if (instance) {
847
- roleFences.set(role.id, {
848
- instance,
849
- abortThreshold: typeof merged.abortThreshold === 'number' ? merged.abortThreshold : 0.8,
850
- scanMessages: merged.scanMessages !== false,
851
- });
852
- }
853
- }
854
- catch {
855
- /* monofence-ai not installed — skip silently */
856
- }
857
- }
858
- if (roleFences.size > 0)
859
- running.fences = roleFences;
860
- // Even-split budget; a role's own budget_tokens overrides it (roleTokenBudget).
861
- // Bug 1: roles WITH an explicit override spend on top of the even split
862
- // rather than out of it, so the roster's ceilings could sum to well over
863
- // the declared org-wide budget (e.g. 4 roles @ 250k + one role overridden
864
- // to 2M = 2.75M achievable against a declared 1M cap). Subtract the sum of
865
- // every role's explicit override from the org-wide budget first, then
866
- // split only the remainder among the roles WITHOUT an override, so the
867
- // static split is honest about what's left. (Live usage is still tracked
868
- // and enforced as a real ceiling above, independent of this static split.)
869
- const orgBudgetTokens = def.run_config.budget_tokens ?? 1_000_000;
870
- const overriddenTokenSum = def.roles.reduce((sum, r) => sum + (r.budget_tokens ?? 0), 0);
871
- const unoverriddenRoleCount = agentRoles(def.roles).filter((r) => r.budget_tokens == null).length;
872
- const _perRoleBudget = unoverriddenRoleCount > 0
873
- ? Math.max(0, Math.floor((orgBudgetTokens - overriddenTokenSum) / unoverriddenRoleCount))
874
- : 0;
875
- // Single boss-selection rule for kickoff AND org_complete gating — the
876
- // session layer previously keyed the tool on reports_to===null while the
877
- // kickoff went to (type==='boss' || reports_to===null || roles[0]), so a
878
- // fallback-selected boss could be told to call org_complete without having
879
- // the tool.
880
- // M2: endpoint roles are never the boss.
881
- const sessionRoles = agentRoles(def.roles);
882
- if (sessionRoles.length === 0)
883
- throw new Error(`org ${name}: no agent roles — endpoint roles cannot run an org`);
884
- const bossRole = sessionRoles.find((r) => r.type === 'boss' || r.reports_to === null) ?? sessionRoles[0];
885
- running.bossRoleId = bossRole.id;
886
- // Canonical entity names from THIS org's KG — injected into the coordinator
887
- // prompt so org_learn extractions reuse them instead of minting duplicates.
888
- // Scoped: an unscoped glossary handed every org's entity names to every
889
- // coordinator, which is how one org's claims got merged into another's.
890
- const glossary = await (async () => {
891
- try {
892
- if (!(await this.orgMemoryUsable()))
893
- return [];
894
- const kg = await import('../memory/memory-kg.js');
895
- return await kg.kgGlossary({
896
- dbPath: this.orgMemoryDbPath(),
897
- scope: orgMemory.orgKgScope(name),
898
- });
899
- }
900
- catch {
901
- return [];
902
- }
903
- })();
904
- running.glossary = glossary;
905
- // Resource-gated staggered spawn: check memory/process limits before each
906
- // NON-BOSS agent, wait if under pressure. The boss always spawns immediately
907
- // and ungated — the org has no coordinator at all without it, so gating it
908
- // behind host memory pressure would make the whole org fail to start over a
909
- // condition workers are specifically designed to ride out.
910
- // Extracted so a role that fails its gate check can be spawned later by
911
- // scheduleDeferredSpawn() once resources free up, without re-running the
912
- // gate logic or duplicating the session-wiring below.
913
- const spawnRole = (role, roleCheckpoint) => {
914
- if (running.agents.has(role.id))
915
- return;
916
- if (isEndpointRole(role))
917
- return; // M2: no session, mailbox or slot
918
- const { runtime, abort } = this.spawnRoleIncarnation(name, running, role, roleCheckpoint?.generation ?? 0, { roleCheckpoint });
919
- running.agents.set(role.id, runtime);
920
- running.roleSlots.set(role.id, {
921
- generation: roleCheckpoint?.generation ?? 0,
922
- phase: 'running',
923
- runtime,
924
- abort,
925
- effectiveRole: roleCheckpoint?.effectiveRoleOverrides &&
926
- Object.keys(roleCheckpoint.effectiveRoleOverrides).length > 0
927
- ? mergeEffectiveRoleConfig(role, roleCheckpoint.effectiveRoleOverrides)
928
- : role,
929
- respawnCount: roleCheckpoint?.respawnCount ?? 0,
930
- queuedDuringSwap: roleCheckpoint?.queuedDuringSwap ?? [],
931
- retiredUsage: roleCheckpoint?.retiredUsage ?? { tokens: 0, costUsd: 0 },
932
- });
933
- };
934
- if (options?.resume && checkpoint) {
935
- const restoredRoles = new Set(Object.keys(checkpoint.roleState));
936
- for (const [roleId, roleState] of Object.entries(checkpoint.roleState)) {
937
- const role = def.roles.find((r) => r.id === roleId);
938
- if (role)
939
- spawnRole(role, roleState);
940
- }
941
- const pendingRoles = new Map();
942
- for (const role of def.roles) {
943
- if (!restoredRoles.has(role.id) && !isEndpointRole(role)) {
944
- pendingRoles.set(role.id, role);
945
- }
946
- }
947
- running.pendingRoles = pendingRoles;
948
- running.spawnRole = spawnRole;
949
- running.taskDag =
950
- checkpoint.tasks && checkpoint.tasks.length > 0
951
- ? TaskDag.fromJSON(checkpoint.tasks)
952
- : new TaskDag();
953
- // A 'running' task's "[task:…]" message was consumed by the session that
954
- // was working it. If that role's SDK session is resumed (checkpointed
955
- // sessionId) the task is still in its context; otherwise — role not
956
- // restored at all, or restored into a fresh session — nothing knows
957
- // about the task, so put it back to 'ready' and re-dispatch.
958
- for (const task of running.taskDag.all()) {
959
- if (task.status !== 'running')
960
- continue;
961
- if (checkpoint.roleState[task.assignee]?.sessionId)
962
- continue;
963
- running.taskDag.requeue(task.id);
964
- }
965
- decisionOps.dispatchReadyTasks(this, name, running);
966
- if (worktreePath)
967
- running.worktreePath = worktreePath;
968
- }
969
- else {
970
- spawnRole(bossRole); // always, ungated — see comment above
971
- // Lazy spawn: register non-boss roles as pending. They spawn on first
972
- // message (see deliver()), avoiding the memory gate stampede at startup.
973
- const pendingRoles = new Map();
974
- for (const role of def.roles) {
975
- if (role.id === bossRole.id || isEndpointRole(role))
976
- continue;
977
- pendingRoles.set(role.id, role);
978
- }
979
- running.pendingRoles = pendingRoles;
980
- running.spawnRole = spawnRole;
981
- running.taskDag = new TaskDag();
982
- if (worktreePath)
983
- running.worktreePath = worktreePath;
984
- }
985
- // Crash cleanup: reap SDK children if this process exits abnormally.
986
- // monolean: process-scoped listener — upgrade path = per-org tracking
987
- const crashCleanup = () => {
988
- try {
989
- // Statically imported: a process 'exit' handler must be synchronous,
990
- // so `await import()` is unavailable — and a bare require() throws
991
- // "require is not defined" in this ESM package. Guarded by
992
- // no-cjs-require-in-esm.test.ts.
993
- reapOrphanedSdkProcesses(new Set(), process.pid);
994
- }
995
- catch {
996
- /* best-effort */
997
- }
998
- // #301: `process.on('exit')` is the SECOND termination path — it fires
999
- // for a normal stop too (finishStop's own prune above already covers
1000
- // that case, so this is a harmless idempotent repeat there) but also
1001
- // for every path that reaches it via an explicit `process.exit()`
1002
- // (org.ts's SIGTERM/SIGINT/SIGHUP handlers, uncaughtException,
1003
- // unhandledRejection) — none of which run finishStop's cleanup at all.
1004
- // execFileSync is already a static top-level import (see the comment
1005
- // above), so this stays synchronous-safe like the rest of this
1006
- // handler. SIGKILL cannot reach here — no in-process code runs for
1007
- // it — which is why prune-at-start exists as the complement.
1008
- try {
1009
- execFileSync('git', ['worktree', 'prune'], {
1010
- cwd: this.root,
1011
- stdio: 'ignore',
1012
- timeout: 30_000,
1013
- });
1014
- }
1015
- catch {
1016
- /* best-effort: not a git repo, git missing, or a wedged hook */
1017
- }
1018
- };
1019
- process.on('exit', crashCleanup);
1020
- running._crashCleanup = crashCleanup;
1021
- // Stale-base drift detection: if the working tree is too many commits behind
1022
- // its tracking branch, warn or refuse to start. Best-effort — git may not be
1023
- // available, or the repo may have no tracking branch.
1024
- const staleThreshold = def.run_config.stale_base_threshold;
1025
- if (staleThreshold && staleThreshold > 0 && cwd === this.root) {
1026
- try {
1027
- const { execSync } = await import('node:child_process');
1028
- const behind = execSync('git rev-list --count HEAD..@{upstream} 2>/dev/null', {
1029
- cwd,
1030
- encoding: 'utf8',
1031
- timeout: 10_000,
1032
- }).trim();
1033
- const count = parseInt(behind, 10);
1034
- if (!Number.isNaN(count) && count > staleThreshold) {
1035
- bus.emit({
1036
- type: 'audit',
1037
- reason: 'stale-base',
1038
- msg: `working tree is ${count} commits behind upstream (threshold: ${staleThreshold}) — consider pulling before running`,
1039
- data: { behind: count, threshold: staleThreshold },
1040
- });
1041
- }
1042
- }
1043
- catch {
1044
- /* no upstream tracking or git unavailable — skip silently */
1045
- }
1046
- }
1047
- const boss = bossRole;
1048
- if (options?.resume) {
1049
- if (running.agents.get(boss.id)?.mailbox.serialize().queue.length === 0) {
1050
- running.agents
1051
- .get(boss.id)
1052
- ?.mailbox.push(`Org "${name}" resumed from checkpoint (run ${run}).\nGoal: ${taskOverride ?? def.goal}\n` +
1053
- `Outstanding tasks and role states have been restored. Continue coordinating your team.`);
1054
- }
1055
- bus.emit({
1056
- type: 'status',
1057
- msg: `org resumed from checkpoint (${run})`,
1058
- data: { goal: taskOverride ?? def.goal },
1059
- });
1060
- }
1061
- else {
1062
- // Cross-run memory: brief the coordinator on the previous run so scheduled
1063
- // orgs accumulate instead of starting cold every interval.
1064
- const prev = readHistory(this.root, name).at(-1);
1065
- const prevBrief = prev
1066
- ? `\n\nPrevious run (${prev.run}${prev.endedAt ? `, ${new Date(prev.endedAt).toISOString()}` : ''}): ` +
1067
- (prev.outcome
1068
- ? `outcome "${prev.outcome.status}" — ${prev.outcome.summary}`
1069
- : `no recorded outcome (${prev.messages} messages, ${prev.assets.length} assets${prev.crashes.length ? `, ${prev.crashes.length} crashed agent(s)` : ''})`) +
1070
- `\nBuild on that work — do not redo what is already done.`
1071
- : '';
1072
- running.agents
1073
- .get(boss.id)
1074
- ?.mailbox.push(`Org "${name}" started (run ${run}).\nGoal: ${taskOverride ?? def.goal}\n` +
1075
- `Coordinate your team via org_send. Only when the FULL goal above is achieved (or clearly can't be) — not merely "this batch of dispatched tasks finished" — record it with org_complete, then end your turn. ` +
1076
- `If a batch finishes but the goal has more scope left, dispatch the next batch instead of ending the run.${prevBrief}`);
1077
- bus.emit({
1078
- type: 'status',
1079
- msg: `org started (${sessionRoles.length} agents)`,
1080
- data: { goal: taskOverride ?? def.goal },
1081
- });
1082
- }
1083
- this.persistState(name, 'running', run);
1084
- // Idle watchdog: a hung tool call (or a run that quietly finished without
1085
- // org_complete) produces no bus events, and every agent just waits. After
1086
- // idle_minutes of silence, nudge the boss to complete or reassign; if the
1087
- // nudge itself produces no activity (boss hung/crashed), or the org keeps
1088
- // going idle after MAX_IDLE_NUDGES nudges in a row without ever recovering
1089
- // (see resolvedIdleNudgeCount), stop the run instead of letting it freeze
1090
- // forever. idle_minutes: 0 disables.
1091
- const idleMs = (def.run_config.idle_minutes ?? 10) * 60_000;
1092
- if (idleMs > 0) {
1093
- const MAX_IDLE_NUDGES = 3;
1094
- let nudgedAt = 0;
1095
- let nudges = 0;
1096
- let stopping = false;
1097
- const idleStop = (msg) => {
1098
- stopping = true;
1099
- bus.emit({ type: 'audit', reason: 'idle-stop', msg });
1100
- // #302: closedBy: 'idle-stop' — the truth gate at finishStop's
1101
- // history write reads this to record what actually happened
1102
- // (including any runnable work left in org_tasks) instead of
1103
- // letting a null outcome default to a plain "completed".
1104
- this.stopOrg(name, { closedBy: 'idle-stop' }).catch((err) => console.error(`org ${name}: idle-stop failed:`, err instanceof Error ? err.message : err));
1105
- };
1106
- // #296: publish the projected stop time for `org status --json`. Only
1107
- // written when it changes; a failed write must not throw out of the
1108
- // interval (that would reach the process crash handlers).
1109
- let published = '';
1110
- const publishDeadline = (hold) => {
1111
- if (stopping)
1112
- return;
1113
- const bossRt = running.agents.get(bossRole.id);
1114
- const at = hold
1115
- ? null
1116
- : new Date(projectIdleStop({
1117
- lastActivity,
1118
- nudgedAt,
1119
- nudges,
1120
- maxNudges: MAX_IDLE_NUDGES,
1121
- idleMs,
1122
- bossReachable: bossRt?.status === 'running' && !bossRt.mailbox.isClosed,
1123
- })).toISOString();
1124
- const key = `${at}|${hold?.reason ?? null}|${hold?.until ?? null}`;
1125
- if (key === published)
1126
- return;
1127
- try {
1128
- writeIdleRecord(this.root, name, {
1129
- run,
1130
- idle_minutes: idleMs / 60_000,
1131
- idle_stop_at: at,
1132
- hold,
1133
- });
1134
- published = key;
1135
- }
1136
- catch (err) {
1137
- console.error(`org ${name}: could not write the idle deadline:`, err instanceof Error ? err.message : err);
1138
- }
1139
- };
1140
- // ADR-O001 D4 — Gas Town's "30 minutes hooked without progress" alarm.
1141
- // The org-wide idle clock is silent while a hold is in force, so a role
1142
- // that is nominally running and producing nothing gets its own, loud
1143
- // audit event. Once per spell: the bus subscriber clears the flag as
1144
- // soon as the role emits anything.
1145
- const alarmNoProgress = (now) => {
1146
- // Only a role with work can be stalled on it: one with no task and no
1147
- // mail, parked on its mailbox (or with its process down after
1148
- // session_idle_exit_ms), is waiting, not hooked.
1149
- const withTask = new Set((running.taskDag?.all() ?? [])
1150
- .filter((t) => t.status === 'running')
1151
- .map((t) => t.assignee));
1152
- const stalled = noProgressRoles([...running.agents].map(([id, rt]) => ({
1153
- id,
1154
- working: rt.status === 'running' &&
1155
- !rt.mailbox.isClosed &&
1156
- hookedOnWork({
1157
- runningTask: withTask.has(id),
1158
- queuedMail: rt.mailbox.peek() !== undefined,
1159
- awaitingMail: rt.mailbox.awaitingMail,
1160
- }),
1161
- lastActivity: roleActivity.get(id) ?? lastActivity,
1162
- alarmed: noProgressAlarmed.has(id),
1163
- })), now);
1164
- for (const role of stalled) {
1165
- noProgressAlarmed.add(role.id);
1166
- bus.emit({
1167
- type: 'audit',
1168
- from: role.id,
1169
- reason: 'no-progress',
1170
- msg: `role "${role.id}" has been running for ${Math.round(role.silentMs / 60_000)}m ` +
1171
- `without a single bus event — it is hooked but producing nothing`,
1172
- data: { role: role.id, silentMinutes: Math.round(role.silentMs / 60_000) },
1173
- });
1174
- }
1175
- };
1176
- // The legitimate waits the watchdog holds through — every one of them
1177
- // with a deadline attached by advanceHold (ADR-O001 D4).
1178
- const holdReason = () => {
1179
- if (this.restarting.has(name))
1180
- return 'restarting'; // boss auto-restart in flight
1181
- // A pending gate means the org is legitimately waiting for human input
1182
- const pendingGates = this.readGates(name).gates.filter((g) => g.status === 'pending');
1183
- if (pendingGates.length > 0)
1184
- return 'pending-gate';
1185
- // Bug 3: a pending ask_human question is the same kind of legitimate
1186
- // wait as a pending gate — askHuman()'s receipt tells the role to end
1187
- // its turn and wait for the resolution, so a role that follows that
1188
- // instruction and goes quiet looks identical to a genuinely stalled
1189
- // agent. Without this check the watchdog nudges (and, after enough
1190
- // nudges, idle-stops) an org that's simply waiting on a human answer
1191
- // that's already on its way.
1192
- //
1193
- // ADR-O001 D4: only a question the asker declared BLOCKING counts. The
1194
- // 8.3-hour stall was held open by a question whose own text opened
1195
- // with "no answer needed for the run to continue; I am not blocking on
1196
- // this" — it suppressed the watchdog exactly like a real blocker.
1197
- if (questionOps.pendingBlockingQuestions(this.root, name).length > 0)
1198
- return 'pending-question';
1199
- // M1 (C-41): a pending tool approval is the same kind of legitimate
1200
- // wait — the role was told to wait for `org approve/deny`.
1201
- if ((this.approvals.get(name) ?? []).some((a) => a.approved === null))
1202
- return 'pending-approval';
1203
- // M2 (C-41): a delivered endpoint message whose reply is still due.
1204
- if (hasActiveEndpointWait(running))
1205
- return 'endpoint-reply-due';
1206
- // A task blocked on a real-world time still in the future is
1207
- // legitimate waiting, same as a pending gate — don't nudge about it.
1208
- // Its deadline is the time the asker actually named, not the default
1209
- // hold TTL, so a block set hours out is honoured exactly.
1210
- const blockedUntil = running.taskDag?.activeBlockUntil(Date.now()) ?? null;
1211
- if (blockedUntil !== null)
1212
- return { reason: 'task-blocked', until: blockedUntil };
1213
- return null;
1214
- };
1215
- // Auto-resume any task whose org_task_block time has passed: flip it
1216
- // back to 'running' and re-push it into the assignee's mailbox, same as
1217
- // a fresh dispatch. This IS real activity, so it feeds the normal
1218
- // idleFor check rather than short-circuiting it — an unblocked task
1219
- // should reset the idle clock, not just silently update state nobody
1220
- // notices until the next nudge. Runs on every tick, a hold in force
1221
- // included: a block that expires mid-wait is real work again.
1222
- const resumeExpiredBlocks = () => {
1223
- wakeDueBlockRechecks(running, Date.now()); // #329: every block is re-checked
1224
- const unblocked = running.taskDag?.unblockExpired(Date.now()) ?? [];
1225
- for (const task of unblocked) {
1226
- const agent = running.agents.get(task.assignee);
1227
- if (agent && !agent.mailbox.isClosed) {
1228
- agent.mailbox.push(`${taskTag(task)} Block expired — resuming: ${task.title}`);
1229
- }
1230
- bus.emit({
1231
- type: 'status',
1232
- from: 'dag',
1233
- reason: 'task-unblocked',
1234
- msg: `task ${task.id} block expired — resumed and re-dispatched to ${task.assignee}`,
1235
- data: { taskId: task.id, assignee: task.assignee },
1236
- });
1237
- }
1238
- };
1239
- // The normal idle path: nudge the boss, then stop if the nudge produced
1240
- // nothing. Runs only when nothing (still) holds the watchdog.
1241
- const check = () => {
1242
- const idleFor = Date.now() - lastActivity;
1243
- if (idleFor < idleMs) {
1244
- nudges = resolvedIdleNudgeCount(nudgedAt, nudges, lastToolActivity);
1245
- nudgedAt = 0;
1246
- return;
1247
- }
1248
- if (nudgedAt === 0) {
1249
- if (nudges >= MAX_IDLE_NUDGES) {
1250
- idleStop(`org idle again after ${nudges} nudges — stopping run`);
1251
- return;
1252
- }
1253
- const bossRt = running.agents.get(bossRole.id);
1254
- // #205: a budget-exhausted boss closed its own mailbox on
1255
- // purpose (session.ts) — that's a recoverable pause, not the
1256
- // same "unreachable" condition as a crash. Name it distinctly so
1257
- // the operator's remedy (raise the budget, resume) is obvious
1258
- // instead of reading like the run died.
1259
- const budgetReason = bossRt?.mailbox.closeReason;
1260
- if (budgetReason === 'token-budget' || budgetReason === 'usd-budget') {
1261
- idleStop(`org idle for ${Math.round(idleFor / 60_000)}m and boss "${bossRole.id}" is over its ` +
1262
- `${budgetReason === 'token-budget' ? 'token' : 'USD'} budget — raise the role's ` +
1263
- `${budgetReason === 'token-budget' ? 'budget_tokens' : 'budget_usd'} (or run_config's) and resume from checkpoint — stopping run`);
1264
- return;
1265
- }
1266
- if (bossRt?.status !== 'running' || bossRt.mailbox.isClosed) {
1267
- idleStop(`org idle for ${Math.round(idleFor / 60_000)}m and boss "${bossRole.id}" is unreachable — stopping run`);
1268
- return;
1269
- }
1270
- nudges++;
1271
- nudgedAt = Date.now();
1272
- bus.emit({
1273
- type: 'audit',
1274
- from: bossRole.id,
1275
- reason: 'idle-nudge',
1276
- msg: `no org activity for ${Math.round(idleFor / 60_000)}m — nudging boss (${nudges}/${MAX_IDLE_NUDGES})`,
1277
- });
1278
- bossRt.mailbox.push(`[watchdog] No activity in org "${name}" for ${Math.round(idleFor / 60_000)} minute(s). ` +
1279
- `Check org_tasks first, then pick ONE: (1) the org's full stated goal is achieved or clearly cannot be — call org_complete now (this ends the run for good, not just this batch); ` +
1280
- `(2) someone has stalled or unstarted work — check on your team via org_send and reassign it; ` +
1281
- `(3) the current task batch is done but the goal has more scope left — do NOT call org_complete for this case, instead dispatch the next batch of work with org_task/createTask so the org keeps making progress; ` +
1282
- `(4) a task is stuck 'running' only because it's genuinely waiting on a real-world time (a scheduled process, a deadline) and there is nothing else to dispatch right now — do NOT just leave it and re-confirm this every time you get nudged, call org_task_block(taskId, untilIso, reason) instead so this watchdog stops nudging you about it and auto-resumes the task when the time arrives.`);
1283
- }
1284
- else if (Date.now() - nudgedAt >= idleMs) {
1285
- idleStop(`nudge produced no activity for another ${Math.round(idleMs / 60_000)}m — boss appears hung, stopping run`);
1286
- }
1287
- };
1288
- // One watchdog tick. The hold is resolved FIRST and with a deadline, so
1289
- // a wait that outlives its deadline hands the run back to the idle path
1290
- // instead of suppressing it forever (ADR-O001 D4).
1291
- let holdTrack = null;
1292
- const tick = () => {
1293
- const now = Date.now();
1294
- alarmNoProgress(now);
1295
- resumeExpiredBlocks();
1296
- const step = advanceHold(holdTrack, holdReason(), now);
1297
- holdTrack = step.track;
1298
- if (step.expired) {
1299
- bus.emit({
1300
- type: 'audit',
1301
- reason: 'hold-expired',
1302
- msg: `the "${step.expired}" hold on org "${name}" outlived its deadline — the idle ` +
1303
- `watchdog is running again and will nudge, then stop the run if nothing happens`,
1304
- data: { hold: step.expired },
1305
- });
1306
- }
1307
- if (step.hold)
1308
- return step.hold;
1309
- check();
1310
- return null;
1311
- };
1312
- const initialHold = advanceHold(null, holdReason(), Date.now());
1313
- holdTrack = initialHold.track;
1314
- publishDeadline(initialHold.hold);
1315
- const wd = setInterval(() => publishDeadline(tick()), Math.max(200, Math.min(idleMs / 2, 30_000)));
1316
- wd.unref?.();
1317
- this.watchdogs.set(name, wd);
1318
- }
1319
- else {
1320
- try {
1321
- writeIdleRecord(this.root, name, {
1322
- run,
1323
- idle_minutes: 0,
1324
- idle_stop_at: null,
1325
- hold: { reason: 'disabled', until: null },
1326
- });
1327
- }
1328
- catch (err) {
1329
- console.error(`org ${name}: could not write the idle deadline:`, err instanceof Error ? err.message : err);
1330
- }
1331
- }
1332
- if (this.opts.crossProcess && this.opts.inboxUrl) {
1333
- const operatorCred = normalizeCredential(this.opts.operatorCredential);
1334
- const lease = new BrokerLease(name, this.opts.inboxUrl, this.opts.brokerDir, undefined, running.credential, operatorCred ? { credential: operatorCred, dir: this.opts.operatorDir } : undefined, this.root);
1335
- lease.start();
1336
- this.leases.set(name, lease);
1337
- }
1338
- // Drain any messages that arrived while the org was offline
1339
- const queued = drainInbox(this.root, name);
1340
- // M2: messages for endpoint roles are delivered by POST, not into a mailbox —
1341
- // put them back (flagged) and let the endpoint retry path send them.
1342
- const endpointQueued = new Set();
1343
- for (const msg of queued) {
1344
- if (!isEndpointRole(def.roles.find((r) => r.id === msg.toRole)))
1345
- continue;
1346
- const messageId = msg.messageId ?? newMessageId();
1347
- endpointQueued.add(messageId);
1348
- queueMessage(this.root, name, { ...msg, messageId, endpoint: true });
1349
- }
1350
- startEndpointRetryLoop(this, name);
1351
- if (endpointQueued.size > 0)
1352
- void retryQueuedEndpoints(this, name, (m) => endpointQueued.has(m.messageId ?? '')).catch(() => {
1353
- /* stays queued — the periodic sweep retries */
1354
- });
1355
- for (const msg of queued) {
1356
- if (isEndpointRole(def.roles.find((r) => r.id === msg.toRole)))
1357
- continue;
1358
- // Spawn a lazy target before delivering. These messages were queued while
1359
- // the org was offline — a human's answer, or another org's request — and
1360
- // the whole point of draining is that they arrive. Skipping a role merely
1361
- // because it has not spawned yet discarded them permanently, after
1362
- // queueMessage had already reported them accepted.
1363
- if (!running.agents.has(msg.toRole) && running.pendingRoles?.has(msg.toRole)) {
1364
- const pending = running.pendingRoles.get(msg.toRole);
1365
- // Bug 4: don't spawn past run_config.max_concurrent_agents. Requeue
1366
- // this message (queueMessage, not a silent drop) and defer the spawn
1367
- // the same way a concurrency-gated lazy spawn defers elsewhere.
1368
- const concurrencyLimit = def.run_config.max_concurrent_agents;
1369
- if (concurrencyLimit != null && activeRoleCount(running) >= concurrencyLimit) {
1370
- running.pendingRoles.delete(msg.toRole);
1371
- queueMessage(this.root, name, msg);
1372
- this.scheduleConcurrencyDeferredSpawn(name, running, pending, running.spawnRole);
1373
- }
1374
- else {
1375
- running.pendingRoles.delete(msg.toRole);
1376
- running.spawnRole?.(pending);
1377
- }
1378
- }
1379
- const agent = running.agents.get(msg.toRole);
1380
- if (agent && !agent.mailbox.isClosed) {
1381
- bus.emit({
1382
- type: 'xorg',
1383
- from: msg.fromQualified,
1384
- to: `${name}:${msg.toRole}`,
1385
- subject: msg.subject,
1386
- msg: msg.body,
1387
- data: { messageId: msg.messageId ?? newMessageId() },
1388
- });
1389
- await crossOrg.pushMessage(this, name, running, msg.toRole, msg.fromQualified, msg.subject, msg.body, `inbox-${msg.ts}-${Math.random().toString(36).slice(2, 8)}`);
1390
- }
1391
- }
1392
- if (queued.length)
1393
- bus.emit({ type: 'status', msg: `drained ${queued.length} queued message(s) from inbox` });
1394
- return running;
179
+ return orgStart.startOrg(this, name, taskOverride, options);
1395
180
  }
1396
181
  /** Build one role incarnation: mailbox, policy, AgentRuntime, sessionOpts,
1397
182
  * and the supervised crash-retry loop. Used by BOTH the startup lazy-spawn
@@ -1399,896 +184,13 @@ export class OrgDaemon {
1399
184
  * and respawnRole() (generation N+1). Does not touch running.agents or
1400
185
  * running.roleSlots — callers publish the result themselves. */
1401
186
  spawnRoleIncarnation(name, running, role, generation, opts = {}) {
1402
- const { roleCheckpoint } = opts;
1403
- const abort = opts.abort ?? new AbortController();
1404
- const { def, bus, run } = running;
1405
- const cwd = running.workdir;
1406
- const ws = this.workspaceSetting(def);
1407
- const perRoleBudget = opts.budgetTokensOverride ?? computeReplacementBudget(def, role.id);
1408
- let roleCwd = cwd;
1409
- const existingSlot = running.roleSlots.get(role.id);
1410
- if (ws === 'worktree-per-role' && role.id !== running.bossRoleId) {
1411
- const wtPath = join(this.root, ORG_DIR, name, `worktree-${role.id}`);
1412
- if (existingSlot?.runtime?.worktreePath === wtPath && existsSync(wtPath)) {
1413
- // A replacement (generation > 0) reuses the SAME worktree path —
1414
- // recreating it here would delete any uncommitted work the old
1415
- // incarnation left behind (design constraint #5).
1416
- roleCwd = wtPath;
1417
- }
1418
- else {
1419
- try {
1420
- // Q7: top-level `import { execFileSync }` replaces the inlined
1421
- // `require('node:child_process')` that broke ESM at runtime —
1422
- // vitest's CJS shim masked it in tests but the built package
1423
- // threw "require is not defined" in real Node ESM execution.
1424
- // SEC-5: argv-array form, no shell.
1425
- if (existsSync(wtPath)) {
1426
- try {
1427
- execFileSync('git', ['worktree', 'remove', '--force', wtPath], {
1428
- cwd: this.root,
1429
- stdio: 'ignore',
1430
- timeout: 30_000,
1431
- });
1432
- }
1433
- catch {
1434
- /* best-effort */
1435
- }
1436
- }
1437
- execFileSync('git', ['worktree', 'add', wtPath, 'HEAD', '--detach'], {
1438
- cwd: this.root,
1439
- stdio: 'ignore',
1440
- timeout: 30_000,
1441
- });
1442
- roleCwd = wtPath;
1443
- }
1444
- catch {
1445
- /* fallback to shared cwd if git worktree fails */
1446
- }
1447
- }
1448
- }
1449
- const mailbox = new Mailbox();
1450
- if (roleCheckpoint?.mailboxQueue?.length) {
1451
- restoreMailboxQueue({ mailbox }, roleCheckpoint.mailboxQueue);
1452
- }
1453
- // A recoverable close (budget exhaustion) is left open on resume — see
1454
- // isRecoverableCloseReason's doc comment. Re-closing it here would
1455
- // make the idle watchdog's "raise the budget and resume" remedy a
1456
- // no-op, since nothing in this codebase ever reopens a closed mailbox.
1457
- if (roleCheckpoint?.mailboxClosed &&
1458
- !isRecoverableCloseReason(roleCheckpoint.mailboxCloseReason)) {
1459
- mailbox.close(roleCheckpoint.mailboxCloseReason);
1460
- }
1461
- const policy = new PolicyEngine(role.id, {
1462
- maxTokens: role.budget_tokens ?? perRoleBudget,
1463
- // ADR-O001 D1: which basis that ceiling is enforced on. Defaults to
1464
- // the historical uncached basis so the honest (cache-aware) meter
1465
- // introduced alongside it cannot exhaust an existing budget_tokens —
1466
- // including the schema's 1M default — roughly 100x early.
1467
- maxTokensBasis: def.run_config.budget_tokens_basis ?? 'uncached',
1468
- maxUsd: role.budget_usd,
1469
- ...(role.policy ?? {}),
1470
- }, bus, roleCwd,
1471
- // #303: the file tools (Read/Write/Edit/Glob/Grep) get the same extra
1472
- // roots the Bash sandbox already treats as writable (role-sandbox.ts) —
1473
- // $TMPDIR, the org root, and any policy.sandbox.allowWrite entries.
1474
- // $HOME is deliberately excluded; see file-roots.ts.
1475
- fileToolRoots({ cwd: roleCwd, orgRoot: this.root }, role.policy?.sandbox));
1476
- policy.setToolContext({
1477
- providerPrefixes: () => roleProviderPrefixes({ tool_providers: effectiveToolProviders(role, this.root) }),
1478
- trace: () => this.roleTrace(name, role.id),
1479
- });
1480
- // ADR-O001 D1: prefer the persisted four-quantity breakdown; a checkpoint
1481
- // written before it existed still resumes via the scalar, on the uncached
1482
- // basis it was recorded on.
1483
- if (roleCheckpoint?.tokenUsage) {
1484
- policy.setTokenUsage(roleCheckpoint.tokenUsage);
1485
- }
1486
- else if (roleCheckpoint?.tokensUsed) {
1487
- policy.setUsage(roleCheckpoint.tokensUsed);
1488
- }
1489
- // ORG-7: restore accumulated USD spend across resume so a stop/resume
1490
- // cycle can't reset a role's USD budget back to zero.
1491
- if (roleCheckpoint?.costUsd) {
1492
- policy.setUsageUsd(roleCheckpoint.costUsd);
1493
- }
1494
- // ADR-O001 D7: the loadout this incarnation's session is built with, fixed
1495
- // for its life. A checkpointed role keeps what it had (its SDK session was
1496
- // built with it); a replacement keeps its predecessor's; a new role takes
1497
- // the loadout of the first ready task it is being spawned for.
1498
- const loadoutName = roleCheckpoint
1499
- ? roleCheckpoint.loadout
1500
- : (existingSlot?.runtime?.loadout ?? sessionLoadoutFor(running.taskDag, role.id));
1501
- let loadout;
1502
- if (loadoutName) {
1503
- try {
1504
- loadout = resolveLoadout(def, loadoutName, this.root);
1505
- }
1506
- catch (err) {
1507
- // Validated at start, so only a config hot-reload or a deleted
1508
- // instructions_file lands here. Spawn without it, loudly: a role that
1509
- // fails to spawn would strand its task, which is worse.
1510
- bus.emit({
1511
- type: 'audit',
1512
- from: role.id,
1513
- reason: 'loadout-unresolvable',
1514
- msg: `role "${role.id}" spawned without loadout "${loadoutName}": ${err instanceof Error ? err.message : err}`,
1515
- data: { loadout: loadoutName },
1516
- });
1517
- }
1518
- }
1519
- const runtime = {
1520
- mailbox,
1521
- policy,
1522
- status: restoredRoleStatus(roleCheckpoint),
1523
- done: Promise.resolve(),
1524
- metrics: { tokens: roleCheckpoint?.tokensUsed ?? 0, costUsd: roleCheckpoint?.costUsd ?? 0 },
1525
- lastMessageId: roleCheckpoint?.lastMessageId,
1526
- error: roleCheckpoint?.error,
1527
- sessionId: roleCheckpoint?.sessionId,
1528
- worktreePath: roleCwd !== cwd ? roleCwd : undefined,
1529
- scrollback: new ScrollbackBuffer(),
1530
- ...(loadout ? { loadout: loadout.name } : {}),
1531
- taskProcesses: new TaskProcesses(),
1532
- };
1533
- if (roleCheckpoint?.scrollback?.length) {
1534
- for (const line of roleCheckpoint.scrollback)
1535
- runtime.scrollback.push(line);
1536
- }
1537
- if (roleCheckpoint?.turns && !running.turns?.has(role.id)) {
1538
- (running.turns ??= new Map()).set(role.id, roleCheckpoint.turns);
1539
- }
1540
- const sessionOpts = {
1541
- org: name,
1542
- role,
1543
- bus,
1544
- policy,
1545
- mailbox,
1546
- taskProcesses: runtime.taskProcesses,
1547
- cwd: roleCwd,
1548
- def,
1549
- // Pass the org state directory so runners that persist per-role state
1550
- // (VercelAgentRunner session files) write under .monomind/orgs/<name>
1551
- // instead of polluting the workspace cwd.
1552
- orgDir: join(this.root, ORG_DIR, name),
1553
- // Project root for named-provider (`adapter_config.provider`) config
1554
- // lookup — role cwd may be an isolated workspace with no config file.
1555
- orgRoot: this.root,
1556
- run,
1557
- // M1: role tool providers — listed at session start, processes spawned
1558
- // lazily on first call and killed when the session ends.
1559
- buildProviderTools: async () => {
1560
- const providers = effectiveToolProviders(role, this.root);
1561
- if (providers.length === 0)
1562
- return undefined;
1563
- return this.toolProviders.buildRoleTools({
1564
- ctx: { org: name, run, role: role.id, root: this.root },
1565
- providers,
1566
- trace: () => this.roleTrace(name, role.id),
1567
- bus,
1568
- cwd: roleCwd,
1569
- });
1570
- },
1571
- maxTurns: role.max_turns_per_message ?? def.run_config.max_turns_per_message,
1572
- resumeSessionId: roleCheckpoint?.sessionId,
1573
- sessionLedger: running.sessionLedger,
1574
- // ADR-O001 D3 x D7: a task-scoped session is built with its task's own
1575
- // recorded loadout. Unresolvable → no loadout, loudly (as at spawn).
1576
- loadoutFor: (taskId) => {
1577
- const name = running.taskDag?.get(taskId)?.loadout;
1578
- if (!name)
1579
- return undefined;
1580
- try {
1581
- return resolveLoadout(def, name, this.root);
1582
- }
1583
- catch (err) {
1584
- bus.emit({
1585
- type: 'audit',
1586
- from: role.id,
1587
- reason: 'loadout-unresolvable',
1588
- msg: `task ${taskId} session built without loadout "${name}": ${err instanceof Error ? err.message : err}`,
1589
- data: { loadout: name, taskId },
1590
- });
1591
- return undefined;
1592
- }
1593
- },
1594
- lastMessageId: () => runtime.lastMessageId,
1595
- onOutput: (line) => runtime.scrollback.push(line),
1596
- onSessionId: (id) => {
1597
- runtime.sessionId = id;
1598
- },
1599
- onTurnEnd: () => {
1600
- endTurn(running, role.id);
1601
- decisionOps.nudgeOpenTasksAtTurnEnd(running, role.id);
1602
- },
1603
- // #327: the role's org_send mail carries its chain at the next hop.
1604
- deliver: (from, to, subject, body) => this.deliver(name, from, to, subject, withTrace(body, this.roleTrace(name, role.id))),
1605
- askHuman: (r, question, blocking) => this.askHuman(name, r, question, blocking),
1606
- onGate: (r, gateName, gateDesc) => this.createGate(name, r, gateName, gateDesc),
1607
- circuitBreaker: (() => {
1608
- const cb = def.run_config.circuit_breaker;
1609
- if (!cb)
1610
- return undefined;
1611
- return { threshold: cb.failure_threshold ?? 5, state: { failures: 0, tripped: false } };
1612
- })(),
1613
- beforeTool: (r, toolName, input) => this.checkApproval(name, r, toolName, input),
1614
- fence: running.fences?.get(role.id),
1615
- // ORG-1: gatedCanUseTool denials are a natural decision point — record them so
1616
- // `org decisions` shows real traces instead of always reporting none.
1617
- onDecision: (r, toolName, message, kind) => {
1618
- this.recordDecision(name, r, {
1619
- type: 'tool',
1620
- kind,
1621
- context: `tool call: ${toolName}`,
1622
- reasoning: message,
1623
- outcome: 'denied',
1624
- });
1625
- },
1626
- // ORG-9: decision gates are documented as "hard-blocking" — make that
1627
- // true by actually denying tool use while this role has a pending gate,
1628
- // the same way pending approvals already do.
1629
- hasPendingGate: () => this.listGates(name, 'pending').some((g) => g.roleId === role.id),
1630
- // #302: refuse an unsatisfiable org_complete BEFORE the 'org-complete'
1631
- // status event ever exists — the bus subscriber at the top of this
1632
- // function auto-stops the run on that event alone, so a refusal that
1633
- // still emitted it would be undone by the very next tick regardless of
1634
- // what this function returns to the tool handler.
1635
- onComplete: role.id === running.bossRoleId
1636
- ? (r, outcome, summary, blocker, blockerDetail) => {
1637
- let maxBudgetFraction = 0;
1638
- for (const rt of running.agents.values()) {
1639
- const p = rt.policy;
1640
- if (p.policy.maxTokens)
1641
- maxBudgetFraction = Math.max(maxBudgetFraction, p.usage / p.policy.maxTokens);
1642
- if (p.policy.maxUsd)
1643
- maxBudgetFraction = Math.max(maxBudgetFraction, p.usageUsd / p.policy.maxUsd);
1644
- }
1645
- // Same predicates the idle watchdog uses for its own
1646
- // legitimate-wait check (:1316-1328) — a pending gate or an
1647
- // unanswered question is the same "genuinely waiting on a
1648
- // human" fact either way.
1649
- const pendingHumanWaits = this.listGates(name, 'pending').length +
1650
- questionOps.pendingBlockingQuestions(this.root, name).length;
1651
- return resolveOrgComplete(bus, r, outcome, summary, blocker, blockerDetail, {
1652
- mode: def.run_config.completion ?? 'boss',
1653
- maxBudgetFraction,
1654
- pendingHumanWaits,
1655
- hasActiveBlock: running.taskDag?.hasActiveBlock(Date.now()) ?? false,
1656
- hasPendingWork: running.taskDag?.hasPendingWork() ?? false,
1657
- });
1658
- }
1659
- : undefined,
1660
- // #11: a boss that overflows its context window isn't a crash (it keeps
1661
- // returning +0-token errors forever), so without this the idle watchdog
1662
- // just nudges it for ~30 min before idle-stopping. Restart the whole org
1663
- // with fresh sessions instead — bounded by MAX_BOSS_RESTARTS.
1664
- onContextLimit: role.id === running.bossRoleId ? () => this.scheduleBossRestart(name) : undefined,
1665
- onListRuntimeOptions: role.id === running.bossRoleId && (def.run_config.max_role_respawns ?? 0) > 0
1666
- ? () => this.listRuntimeOptions()
1667
- : undefined,
1668
- onRespawnRole: role.id === running.bossRoleId && (def.run_config.max_role_respawns ?? 0) > 0
1669
- ? (callerId, args) => this.respawnRole(name, callerId, args)
1670
- : undefined,
1671
- recall: async (r, q) => {
1672
- const answer = await this.recallOrgMemory(name, def, q, r);
1673
- bus.emit({
1674
- type: 'status',
1675
- from: r,
1676
- reason: 'org-recall',
1677
- msg: `recall: ${q.slice(0, 80)}`,
1678
- data: { hits: answer.hits },
1679
- });
1680
- return answer.text;
1681
- },
1682
- searchKnowledge: async (r, q) => {
1683
- const answer = await this.searchProjectKnowledge(q);
1684
- bus.emit({
1685
- type: 'status',
1686
- from: r,
1687
- reason: 'knowledge-search',
1688
- msg: `knowledge: ${q.slice(0, 80)}`,
1689
- data: { hits: answer.hits },
1690
- });
1691
- return answer.text;
1692
- },
1693
- glossary: running.glossary,
1694
- remember: async (r, content, scope) => {
1695
- const text = await this.rememberOrgMemory(name, def, r, content, scope, run);
1696
- bus.emit({
1697
- type: 'status',
1698
- from: r,
1699
- reason: 'org-remember',
1700
- msg: `remember (${scope}): ${content.slice(0, 80)}`,
1701
- data: { scope },
1702
- });
1703
- return text;
1704
- },
1705
- learn: async (r, payload) => {
1706
- const text = await this.learnOrgKnowledge(name, run, payload);
1707
- bus.emit({
1708
- type: 'status',
1709
- from: r,
1710
- reason: 'org-learn',
1711
- msg: `learn: ${text.slice(0, 120)}`,
1712
- data: {
1713
- nodes: payload.nodes?.length ?? 0,
1714
- edges: payload.edges?.length ?? 0,
1715
- rules: payload.rules?.length ?? 0,
1716
- },
1717
- });
1718
- return text;
1719
- },
1720
- createTask: (r, title, assignee, deps, loadout, brief, pick) => {
1721
- return this.dagCreateTask(name, r, title, assignee, deps, loadout, brief, pick);
1722
- },
1723
- pickAssignee: resolveAutoAssignee(def, (id) => openTaskCount(this.orgs.get(name), id), () => this.orgs.get(name)?.taskDag?.all() ?? []),
1724
- onSkillLoad: (r, skill) => decisionOps.recordSkillLoad(this.orgs.get(name), r, skill),
1725
- // ADR-O001 D7: only an org with a catalog gets the `loadout` argument;
1726
- // the session itself is built with the loadout frozen above.
1727
- loadoutCatalog: loadoutCatalog(def),
1728
- loadout,
1729
- completeTask: (r, taskId, result, evidence) => {
1730
- return this.dagCompleteTask(name, r, taskId, result, evidence);
1731
- },
1732
- // ADR-O001 D5: only an org that opted in advertises the evidence
1733
- // argument, so every other org's tool list stays byte-identical.
1734
- requireTaskEvidence: def.run_config.completion_evidence === true && role.deliberative !== true,
1735
- // ADR-O001 D6: only an org with an artifact-only reviewer gets org_review,
1736
- // so every other org's tool list stays byte-identical.
1737
- requestReview: def.roles.some((r) => r.review_input === 'artifact-only')
1738
- ? (r, taskId, reviewer, base) => this.dagRequestReview(name, r, taskId, reviewer, base)
1739
- : undefined,
1740
- listTasks: (taskId) => decisionOps.dagListTasks(this, name, taskId),
1741
- splitTask: (r, parentId, children) => {
1742
- return this.dagSplitTask(name, r, parentId, children);
1743
- },
1744
- mergeTask: (r, sourceId, targetId) => {
1745
- return this.dagMergeTask(name, r, sourceId, targetId);
1746
- },
1747
- cancelTask: (r, taskId, reason) => {
1748
- return this.dagCancelTask(name, r, taskId, reason);
1749
- },
1750
- blockTask: (r, taskId, untilIso, reason, every) => {
1751
- return this.dagBlockTask(name, r, taskId, untilIso, reason, every);
1752
- },
1753
- planGraph: (r, specs) => {
1754
- return this.dagPlanGraph(name, r, specs);
1755
- },
1756
- queryFn: this.opts.queryFn,
1757
- // Runner resolution: explicit opts.runner > role `runtime` field >
1758
- // org def `runtime` field > MONOMIND_RUNTIME env (opencode/kimicode) >
1759
- // undefined (session.ts falls back to ClaudeAgentRunner via queryFn).
1760
- // Leaving it undefined for the default path is what keeps
1761
- // Claude/Antigravity orgs byte-for-byte unchanged. Session opts are
1762
- // built per role here, so each role gets its own runner.
1763
- runner: this.opts.runner ??
1764
- resolveRoleRunner(role.runtime, def.runtime, role.provider?.kind, undefined, role.provider),
1765
- // Lets respawnRole force-stop THIS specific incarnation (mid-run role
1766
- // replacement's forced-stop step) without reaching into runAgentSession's
1767
- // internals.
1768
- externalAbort: abort,
1769
- silentSessionMs: this.opts.silentSessionMs,
1770
- };
1771
- // Supervised session: transient crashes (provider blips, network) restart
1772
- // with backoff; a crash with the mailbox already closed, or one that
1773
- // exhausts the retry budget, is terminal. runAgentSession already emits a
1774
- // 'status' event for the raw error; the terminal 'audit' event is for
1775
- // dashboards/alerts that filter on actionable failures (not routine
1776
- // status chatter) so a dead agent surfaces instead of a run that
1777
- // silently never progresses.
1778
- const BACKOFFS_MS = this.opts.crashBackoffsMs ?? [1000, 5000, 15000];
1779
- const myGeneration = generation;
1780
- const isStaleGeneration = () => (running.roleSlots.get(role.id)?.generation ?? 0) !== myGeneration;
1781
- if (!mailbox.isClosed && runtime.status !== 'crashed') {
1782
- runtime.done = (async () => {
1783
- for (let attempt = 0;; attempt++) {
1784
- try {
1785
- await runAgentSession(sessionOpts);
1786
- runtime.status = 'ended';
1787
- return;
1788
- }
1789
- catch (err) {
1790
- // A deliberate respawn (see respawnRole) bumps the slot's
1791
- // generation and force-stops this incarnation via its
1792
- // externalAbort - that abort makes runAgentSession reject here
1793
- // exactly like a real crash would. Recognize supersession
1794
- // FIRST: this generation's retry loop must never restart,
1795
- // never run terminal crash handling, and never notify the
1796
- // boss - the replacement (a new generation, spawned
1797
- // separately) already owns this role id.
1798
- if (isStaleGeneration())
1799
- return;
1800
- // Drop the crashed session's stale waker immediately: a push()
1801
- // during the backoff window must queue for the NEXT session, not
1802
- // wake the dead generator to swallow it.
1803
- mailbox.detach();
1804
- // #203: if the crashed session's mailbox generator was abandoned
1805
- // mid-yield (message already shift()ed for it, turn never
1806
- // finished), put that message back on the queue — otherwise the
1807
- // replacement session's stream() finds an empty queue and parks
1808
- // forever, since the "delivered" message is gone for good.
1809
- mailbox.reclaimInFlight();
1810
- const message = err instanceof Error ? err.message : String(err);
1811
- const isTurnLimit = /Reached maximum number of turns|error_max_turns/i.test(message);
1812
- // Bounded like every other recovery: attempt counts every pass
1813
- // through this loop, so a role that keeps surfacing max-turns
1814
- // errors here (session.ts already swallows the normal ones)
1815
- // falls through to crash handling instead of looping forever.
1816
- if (isTurnLimit && !mailbox.isClosed && attempt < BACKOFFS_MS.length) {
1817
- sessionOpts.resumeSessionId = undefined;
1818
- mailbox.push(`${Mailbox.CONTINUE_PREFIX} You reached the turn limit on your task. Continue your in-progress work from where you left off; if finished, end your turn.`);
1819
- bus.emit({
1820
- type: 'status',
1821
- from: role.id,
1822
- reason: 'turn-limit-recover',
1823
- msg: `agent "${role.id}" hit turn limit error — continuing with fresh session`,
1824
- });
1825
- continue;
1826
- }
1827
- // Exit 143 = SIGTERM. If the mailbox is already closed, we
1828
- // sent the signal ourselves during stop — not a crash.
1829
- const killedByStop = mailbox.isClosed && /exit(?:ed)? with code 143/.test(message);
1830
- // #251: the org's own stop/complete (finishStop) removes this run
1831
- // from this.orgs, closes every mailbox and aborts every session —
1832
- // an idle one then rejects with the abort ("Operation aborted",
1833
- // "Claude Code process aborted by user"). That is a shutdown, not a
1834
- // crash. A non-abort error surfacing during the stop, or a crash
1835
- // already backing off when the stop landed, stays a crash.
1836
- const abortedByStop = mailbox.isClosed &&
1837
- this.orgs.get(name) !== running &&
1838
- (err?.name === 'AbortError' ||
1839
- /\baborted\b/i.test(message));
1840
- const crash = () => {
1841
- if (killedByStop) {
1842
- runtime.status = 'ended';
1843
- bus.emit({
1844
- type: 'status',
1845
- from: role.id,
1846
- msg: `agent "${role.id}" terminated by stop (was still working when drain window expired)`,
1847
- reason: 'terminated-by-stop',
1848
- });
1849
- return;
1850
- }
1851
- if (abortedByStop) {
1852
- runtime.status = 'ended';
1853
- // #304: `message` here is always an abort string (see abortedByStop
1854
- // above) — either "Operation aborted" or the SDK's "Claude Code
1855
- // process aborted by user". Echoing it made a planned stop read as
1856
- // a human interruption, and made roles of the same run read
1857
- // differently. Report WHY the org stopped instead; the raw string
1858
- // stays in `data` for debugging.
1859
- const why = running.closedBy === 'org-complete' ? 'org_complete' : 'stop requested';
1860
- bus.emit({
1861
- type: 'status',
1862
- from: role.id,
1863
- msg: `agent "${role.id}" stopped with the org (${why})`,
1864
- reason: 'agent-stopped',
1865
- data: { agentId: role.id, error: message },
1866
- });
1867
- return;
1868
- }
1869
- runtime.status = 'crashed';
1870
- runtime.error = message;
1871
- // Close the mailbox so deliver()/receiveRemote() report a real
1872
- // error instead of pushing into a queue no session will read
1873
- // (and returning a false "delivered" receipt to the sender).
1874
- mailbox.close();
1875
- const isContextLimit = OrgDaemon.CONTEXT_LIMIT_RE.test(message);
1876
- bus.emit({
1877
- type: 'audit',
1878
- from: role.id,
1879
- msg: `agent "${role.id}" crashed: ${message}`,
1880
- reason: isContextLimit ? 'agent-context-limit' : 'agent-session-crash',
1881
- data: {
1882
- agentId: role.id,
1883
- error: message,
1884
- restarts: attempt,
1885
- contextLimit: isContextLimit,
1886
- },
1887
- });
1888
- if (role.id !== running.bossRoleId) {
1889
- // #2/#3: a worker is gone for the rest of this run. Without this
1890
- // notice the coordinator keeps messaging a corpse (observed: four
1891
- // unanswered org_send calls to a developer that had crashed on a
1892
- // context-window limit). Tell the boss to reassign — and if the
1893
- // crash was a context overflow, tell it to chunk smaller, since
1894
- // re-dispatching the same task verbatim fails the same way.
1895
- const bossRt = running.agents.get(running.bossRoleId);
1896
- if (bossRt && !bossRt.mailbox.isClosed) {
1897
- const guidance = isContextLimit
1898
- ? ' This was a context-window overflow — re-dispatching the same task verbatim will fail identically. Break the work into smaller pieces (one file or section at a time) and do not paste large file contents in a single message.'
1899
- : '';
1900
- bossRt.mailbox.push(`[system] Worker "${role.id}" crashed and will not recover this run (${message}). It can no longer receive messages — stop messaging it. Reassign its outstanding work to another agent or take it on yourself.${guidance}`);
1901
- bus.emit({
1902
- type: 'audit',
1903
- from: running.bossRoleId,
1904
- reason: 'worker-crashed',
1905
- msg: `worker "${role.id}" crashed (contextLimit=${isContextLimit}); coordinator notified to reassign`,
1906
- });
1907
- }
1908
- }
1909
- else {
1910
- // #4: the coordinator itself died. Don't go silent and wait for a
1911
- // human — attempt a bounded whole-org restart with fresh sessions
1912
- // (which also sheds whatever bloated context caused the crash).
1913
- this.scheduleBossRestart(name);
1914
- }
1915
- };
1916
- // Fatal errors (provider auth/quota/billing — tagged with
1917
- // err.fatal by the runner) can NEVER be fixed by a restart: the
1918
- // same call fails identically or hangs. Skip the backoff loop
1919
- // and go straight to terminal crash handling instead of burning
1920
- // the retry budget and wall-clock on a guaranteed failure.
1921
- const fatal = err?.fatal === true;
1922
- if (fatal) {
1923
- bus.emit({
1924
- type: 'status',
1925
- from: role.id,
1926
- reason: 'agent-fatal',
1927
- msg: `agent "${role.id}" hit a fatal (non-retryable) error — not restarting`,
1928
- });
1929
- crash();
1930
- return;
1931
- }
1932
- if (mailbox.isClosed || attempt >= BACKOFFS_MS.length) {
1933
- crash();
1934
- return;
1935
- }
1936
- bus.emit({
1937
- type: 'status',
1938
- from: role.id,
1939
- reason: 'agent-restart',
1940
- msg: `agent "${role.id}" crashed (${message}) — restarting in ${BACKOFFS_MS[attempt]}ms (attempt ${attempt + 1}/${BACKOFFS_MS.length})`,
1941
- });
1942
- await new Promise((r) => {
1943
- const t = setTimeout(r, BACKOFFS_MS[attempt]);
1944
- t.unref?.();
1945
- // Org stop (finishStop) aborts every active slot's controller —
1946
- // without racing it here, this wait wouldn't notice for up to
1947
- // BACKOFFS_MS[attempt] (default up to 15s), well past finishStop's
1948
- // own bounded drain window. That let this loop's crash() —
1949
- // and the bus.emit() it triggers — fire AFTER finishStop had
1950
- // already declared the org stopped and returned, capable of
1951
- // recreating files in a run directory a caller was already
1952
- // deleting.
1953
- if (abort.signal.aborted) {
1954
- clearTimeout(t);
1955
- r();
1956
- return;
1957
- }
1958
- abort.signal.addEventListener('abort', () => {
1959
- clearTimeout(t);
1960
- r();
1961
- }, { once: true });
1962
- });
1963
- if (isStaleGeneration())
1964
- return; // superseded during the backoff wait
1965
- if (mailbox.isClosed) {
1966
- crash();
1967
- return;
1968
- } // org stopped during backoff — never recovered
1969
- // #247: continue the crashed conversation (briefing, task context,
1970
- // finished work) instead of starting cold. runAgentSession falls
1971
- // back to one fresh session if this id can't be resumed.
1972
- sessionOpts.resumeSessionId = runtime.sessionId;
1973
- }
1974
- }
1975
- })();
1976
- }
1977
- return { runtime, abort };
187
+ return roleIncarnation.spawnRoleIncarnation(this, name, running, role, generation, opts);
1978
188
  }
1979
189
  /** org_respawn_role's daemon-owned implementation. See the design doc's
1980
190
  * "Replacement algorithm" (13 steps) — this method's body follows those
1981
191
  * steps in order, numbered in comments. */
1982
192
  async respawnRole(name, callerId, rawInput) {
1983
- const running = this.orgs.get(name);
1984
- if (!running) {
1985
- return {
1986
- success: false,
1987
- roleId: '',
1988
- generation: 0,
1989
- respawnCount: 0,
1990
- respawnsRemaining: 0,
1991
- error: `org "${name}" is not running`,
1992
- };
1993
- }
1994
- // Step 1: authorize (defense in depth — buildOrgTools only ever wires
1995
- // onRespawnRole for the selected coordinator, but re-check here too).
1996
- if (callerId !== running.bossRoleId) {
1997
- return {
1998
- success: false,
1999
- roleId: '',
2000
- generation: 0,
2001
- respawnCount: 0,
2002
- respawnsRemaining: 0,
2003
- error: 'only the selected coordinator may call org_respawn_role',
2004
- };
2005
- }
2006
- if (this.stopping.has(name)) {
2007
- return {
2008
- success: false,
2009
- roleId: '',
2010
- generation: 0,
2011
- respawnCount: 0,
2012
- respawnsRemaining: 0,
2013
- error: `org "${name}" is stopping`,
2014
- };
2015
- }
2016
- const validated = validateRespawnInput(rawInput);
2017
- if (!validated.ok) {
2018
- return {
2019
- success: false,
2020
- roleId: '',
2021
- generation: 0,
2022
- respawnCount: 0,
2023
- respawnsRemaining: 0,
2024
- error: validated.error,
2025
- };
2026
- }
2027
- const input = validated.value;
2028
- if (input.roleId === running.bossRoleId) {
2029
- return {
2030
- success: false,
2031
- roleId: input.roleId,
2032
- generation: 0,
2033
- respawnCount: 0,
2034
- respawnsRemaining: 0,
2035
- error: 'cannot replace the selected coordinator',
2036
- };
2037
- }
2038
- const slot = running.roleSlots.get(input.roleId);
2039
- if (!slot) {
2040
- return {
2041
- success: false,
2042
- roleId: input.roleId,
2043
- generation: 0,
2044
- respawnCount: 0,
2045
- respawnsRemaining: 0,
2046
- error: `unknown or not-yet-started role "${input.roleId}"`,
2047
- };
2048
- }
2049
- const maxRespawns = running.def.run_config.max_role_respawns ?? 0;
2050
- if (slot.phase === 'removed') {
2051
- return buildRespawnReceipt(slot, maxRespawns, false, {
2052
- roleId: input.roleId,
2053
- error: `role "${input.roleId}" was removed from this org`,
2054
- });
2055
- }
2056
- // Step 2: acquire the role slot (reject a concurrent replacement).
2057
- if (running.respawning.has(input.roleId) || slot.respawnPromise) {
2058
- return buildRespawnReceipt(slot, maxRespawns, false, {
2059
- roleId: input.roleId,
2060
- error: `role "${input.roleId}" is already undergoing replacement`,
2061
- });
2062
- }
2063
- if (slot.respawnCount >= maxRespawns) {
2064
- return buildRespawnReceipt(slot, maxRespawns, false, {
2065
- roleId: input.roleId,
2066
- error: `role "${input.roleId}" has reached its respawn limit (${slot.respawnCount}/${maxRespawns}) for this run`,
2067
- });
2068
- }
2069
- // Step 3: resolve the candidate configuration.
2070
- if (input.providerName !== undefined && slot.effectiveRole.provider) {
2071
- return buildRespawnReceipt(slot, maxRespawns, false, {
2072
- roleId: input.roleId,
2073
- error: `role "${input.roleId}" has an inline provider, which always takes precedence over adapter_config.provider — replacing an inline provider is a separate design`,
2074
- });
2075
- }
2076
- const candidateRole = mergeEffectiveRoleConfig(slot.effectiveRole, {
2077
- runtime: input.runtime,
2078
- model: input.model,
2079
- providerName: input.providerName,
2080
- });
2081
- const budgetTokens = input.budgetTokens ?? computeReplacementBudget(running.def, input.roleId);
2082
- // Step 4: preflight — must not mutate the old runtime.
2083
- try {
2084
- resolveRoleProvider(candidateRole, this.root);
2085
- }
2086
- catch (err) {
2087
- return buildRespawnReceipt(slot, maxRespawns, false, {
2088
- roleId: input.roleId,
2089
- error: `preflight failed: ${err instanceof Error ? err.message : String(err)}`,
2090
- });
2091
- }
2092
- // resolveRoleRunner's undefined return is the valid Claude default, not
2093
- // an error — nothing further to validate for the runtime dimension here.
2094
- resolveRoleRunner(candidateRole.runtime, running.def.runtime, candidateRole.provider?.kind, undefined, candidateRole.provider);
2095
- // Step 5: consume one attempt — only after validation/preflight succeed.
2096
- running.respawning.add(input.roleId);
2097
- slot.respawnCount++;
2098
- running.bus.emit({
2099
- type: 'audit',
2100
- from: callerId,
2101
- reason: 'role-respawn-started',
2102
- msg: `replacing role "${input.roleId}": ${input.reason}`,
2103
- data: {
2104
- roleId: input.roleId,
2105
- from: redactRoleConfig(slot.effectiveRole),
2106
- to: redactRoleConfig(candidateRole),
2107
- generation: slot.generation,
2108
- caller: callerId,
2109
- },
2110
- });
2111
- // Step 6: quiesce the old incarnation. Bump the generation NOW, before
2112
- // draining starts — not at the final publish (step 11-13) — so the OLD
2113
- // generation's crash-retry loop (spawnRoleIncarnation's isStaleGeneration
2114
- // check) recognizes supersession immediately. Without this, a backoff
2115
- // timer firing during the drain/force-stop window, or the forced abort's
2116
- // own rejection, would still see itself as the current generation:
2117
- // the abort's rejection doesn't match killedByStop's SIGTERM-only regex,
2118
- // so it would run full terminal crash handling — a duplicate live runner
2119
- // (mid-backoff restart) or a false worker-crashed notification, exactly
2120
- // what the guard exists to prevent.
2121
- const newGeneration = slot.generation + 1;
2122
- slot.generation = newGeneration;
2123
- slot.phase = 'draining';
2124
- // Every await from here on can race a stop/restart of this org — verify
2125
- // ownership before EVERY subsequent step, not just once before the final
2126
- // publish, so a stale operation can never mutate accounting, force-stop
2127
- // a runtime, or spawn into an org that's no longer the live one.
2128
- const stillOwned = () => this.orgs.get(name) === running && running.roleSlots.get(input.roleId) === slot;
2129
- const abandonedReceipt = () => {
2130
- running.respawning.delete(input.roleId);
2131
- return buildRespawnReceipt(slot, maxRespawns, false, {
2132
- roleId: input.roleId,
2133
- error: `org "${name}" stopped or restarted during replacement`,
2134
- });
2135
- };
2136
- const oldRuntime = slot.runtime;
2137
- const sweptQueue = oldRuntime.mailbox.beginDrain();
2138
- slot.queuedDuringSwap.push(...sweptQueue);
2139
- const drainTimeoutMs = running.def.run_config.respawn_drain_timeout_ms ?? 30_000;
2140
- const drained = await Promise.race([
2141
- oldRuntime.done.then(() => true),
2142
- new Promise((r) => setTimeout(() => r(false), drainTimeoutMs)),
2143
- ]);
2144
- if (!stillOwned())
2145
- return abandonedReceipt();
2146
- let drainTimedOut = false;
2147
- if (!drained) {
2148
- drainTimedOut = true;
2149
- // Step 7: force stop.
2150
- slot.abort?.abort();
2151
- const forceStopMs = running.def.run_config.respawn_force_stop_timeout_ms ?? 5_000;
2152
- const stopped = await Promise.race([
2153
- oldRuntime.done.then(() => true).catch(() => true),
2154
- new Promise((r) => setTimeout(() => r(false), forceStopMs)),
2155
- ]);
2156
- if (!stillOwned())
2157
- return abandonedReceipt();
2158
- if (!stopped) {
2159
- slot.phase = 'stuck';
2160
- running.respawning.delete(input.roleId);
2161
- running.bus.emit({
2162
- type: 'audit',
2163
- from: callerId,
2164
- reason: 'role-respawn-failed',
2165
- msg: `role "${input.roleId}" forced stop did not confirm termination — refusing to spawn a replacement`,
2166
- });
2167
- return buildRespawnReceipt(slot, maxRespawns, false, {
2168
- roleId: input.roleId,
2169
- error: `role "${input.roleId}" could not be confirmed stopped; not replaced`,
2170
- });
2171
- }
2172
- }
2173
- // Step 8: preserve durable role state (worktree path, task ownership, and
2174
- // the DAG survive untouched — they live outside AgentRuntime/Mailbox
2175
- // entirely, keyed by role.id, which never changes). Reclaim any message
2176
- // abandoned mid-yield by a forced stop for at-least-once redelivery.
2177
- oldRuntime.mailbox.reclaimInFlight();
2178
- const reclaimedQueue = oldRuntime.mailbox.serialize().queue;
2179
- slot.queuedDuringSwap.push(...reclaimedQueue);
2180
- // Step 9: retire accounting BEFORE replacing the runtime.
2181
- slot.retiredUsage = {
2182
- // Budgeted basis: this total is summed with live policy.budgetedUsage
2183
- // against the org-wide budget_tokens ceiling (ADR-O001 D1), so the two
2184
- // terms must share a basis.
2185
- tokens: slot.retiredUsage.tokens + oldRuntime.policy.budgetedUsage,
2186
- costUsd: slot.retiredUsage.costUsd + oldRuntime.metrics.costUsd,
2187
- };
2188
- // Step 10: spawn generation N+1 (generation already bumped in step 6).
2189
- const { runtime: newRuntime, abort: newAbort } = this.spawnRoleIncarnation(name, running, candidateRole, newGeneration, { budgetTokensOverride: budgetTokens });
2190
- // Seed the new mailbox with everything swapped/reclaimed, delivered
2191
- // FIFO, plus a delimited coordinator briefing appended last so it reads
2192
- // as the newest context once the replacement starts its first turn.
2193
- for (const queued of slot.queuedDuringSwap)
2194
- newRuntime.mailbox.push(queued);
2195
- newRuntime.mailbox.push(`[system: role replacement briefing — not a system prompt] You are a fresh session replacing the previous incarnation of role "${input.roleId}". Reason: ${input.reason}\n\n${input.briefing}`);
2196
- // Seed USD accounting from retained totals so a respawn cannot reset
2197
- // role.budget_usd.
2198
- if (running.def.roles.find((r) => r.id === input.roleId)?.budget_usd !== undefined) {
2199
- newRuntime.policy.setUsageUsd(slot.retiredUsage.costUsd);
2200
- }
2201
- // "Ready" here means "did not crash within the startup window" — a
2202
- // silent-but-healthy runner (one that never emits a chat/tool/usage
2203
- // event, e.g. because it hasn't finished its first turn yet) must not be
2204
- // misreported as a startup failure, so this does NOT wait for a positive
2205
- // signal. It races the new incarnation's own crash-retry loop (which
2206
- // shares this generation, so it is NOT superseded and behaves normally)
2207
- // against the timeout: a config that fails immediately (bad model,
2208
- // missing runtime binary, auth failure) crashes fast and newRuntime.done
2209
- // resolves with status 'crashed' well before startTimeoutMs, correctly
2210
- // failing readiness and triggering rollback.
2211
- const startTimeoutMs = running.def.run_config.respawn_start_timeout_ms ?? 60_000;
2212
- const ready = await Promise.race([
2213
- newRuntime.done.then(() => newRuntime.status !== 'crashed'),
2214
- new Promise((r) => setTimeout(() => r(true), startTimeoutMs)),
2215
- ]);
2216
- // Step 11: publish atomically — verify ownership is still current.
2217
- if (!stillOwned()) {
2218
- newAbort.abort();
2219
- return abandonedReceipt();
2220
- }
2221
- if (!ready) {
2222
- // Step 12: rollback — one attempt with the prior effective config.
2223
- newAbort.abort();
2224
- running.bus.emit({
2225
- type: 'audit',
2226
- from: callerId,
2227
- reason: 'role-respawn-failed',
2228
- msg: `role "${input.roleId}" replacement did not become ready within ${startTimeoutMs}ms — attempting rollback`,
2229
- });
2230
- try {
2231
- const { runtime: rolledBack, abort: rolledBackAbort } = this.spawnRoleIncarnation(name, running, slot.effectiveRole, newGeneration + 1, {});
2232
- for (const queued of slot.queuedDuringSwap)
2233
- rolledBack.mailbox.push(queued);
2234
- running.agents.set(input.roleId, rolledBack);
2235
- slot.runtime = rolledBack;
2236
- slot.abort = rolledBackAbort;
2237
- slot.generation = newGeneration + 1;
2238
- slot.phase = 'running';
2239
- slot.queuedDuringSwap = [];
2240
- running.respawning.delete(input.roleId);
2241
- running.bus.emit({
2242
- type: 'audit',
2243
- from: callerId,
2244
- reason: 'role-respawn-failed',
2245
- msg: `role "${input.roleId}" replacement failed; rolled back to prior config`,
2246
- });
2247
- return buildRespawnReceipt(slot, maxRespawns, false, {
2248
- roleId: input.roleId,
2249
- drainTimedOut,
2250
- error: `replacement failed to start; rolled back to prior configuration`,
2251
- });
2252
- }
2253
- catch (rollbackErr) {
2254
- slot.phase = 'crashed';
2255
- running.respawning.delete(input.roleId);
2256
- running.bus.emit({
2257
- type: 'audit',
2258
- from: callerId,
2259
- reason: 'role-respawn-rollback-failed',
2260
- msg: `role "${input.roleId}" replacement AND rollback both failed: ${rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr)}`,
2261
- });
2262
- return buildRespawnReceipt(slot, maxRespawns, false, {
2263
- roleId: input.roleId,
2264
- drainTimedOut,
2265
- error: `replacement and rollback both failed; role "${input.roleId}" is unavailable`,
2266
- });
2267
- }
2268
- }
2269
- running.agents.set(input.roleId, newRuntime);
2270
- slot.runtime = newRuntime;
2271
- slot.abort = newAbort;
2272
- slot.generation = newGeneration;
2273
- slot.effectiveRole = candidateRole;
2274
- slot.phase = 'running';
2275
- slot.queuedDuringSwap = [];
2276
- running.respawning.delete(input.roleId);
2277
- // Step 13: audit and persist.
2278
- running.bus.emit({
2279
- type: 'audit',
2280
- from: callerId,
2281
- reason: 'role-respawned',
2282
- msg: `role "${input.roleId}" replaced (generation ${newGeneration})`,
2283
- data: {
2284
- roleId: input.roleId,
2285
- generation: newGeneration,
2286
- respawnCount: slot.respawnCount,
2287
- drainTimedOut,
2288
- },
2289
- });
2290
- this.persistState(name, 'running', running.run);
2291
- return buildRespawnReceipt(slot, maxRespawns, true, { roleId: input.roleId, drainTimedOut });
193
+ return roleRespawn.respawnRole(this, name, callerId, rawInput);
2292
194
  }
2293
195
  /** @internal */
2294
196
  hasOrgDef(name) {
@@ -2305,325 +207,7 @@ export class OrgDaemon {
2305
207
  * watchdog, boss-restart-exhausted, manual `org stop`) and exit non-zero
2306
208
  * for the latter. Only the org_complete auto-stop path passes this. */
2307
209
  async stopOrg(name, opts) {
2308
- // Join an in-flight stop instead of no-oping: the self-stop paths
2309
- // (org_complete, idle watchdog) run detached, and a caller like
2310
- // `org run`'s final stopAll() must not resolve — letting the process
2311
- // exit — while that stop is still flushing the bus and writing
2312
- // history/runtime.json.
2313
- const inflight = this.stopping.get(name);
2314
- if (inflight)
2315
- return inflight;
2316
- const org = this.orgs.get(name);
2317
- if (!org)
2318
- return; // already stopped
2319
- org.pendingRoles?.clear(); // prevent lazy spawns after stop
2320
- this.spawning.delete(name); // clean up spawning tracking for this org
2321
- // #304: set before the delete below, since that delete is what makes
2322
- // abortedByStop (role loop) true — the role loop reads it off this same
2323
- // object reference, not a fresh lookup (the org is gone from the map by then).
2324
- org.closedBy = opts?.closedBy;
2325
- // Remove immediately (not at the end) so a concurrent stopOrg(name) call —
2326
- // e.g. stopAll() racing a scheduler-triggered stop on SIGINT — joins this
2327
- // shutdown via `stopping` instead of re-running the whole sequence and
2328
- // double-emitting 'org stopped' (duplicate org:complete/session:complete).
2329
- this.orgs.delete(name);
2330
- const p = this.finishStop(name, org, opts?.drainMs, opts?.closedBy);
2331
- this.stopping.set(name, p);
2332
- try {
2333
- await p;
2334
- }
2335
- finally {
2336
- this.stopping.delete(name);
2337
- }
2338
- }
2339
- async finishStop(name, org, drainMs, closedBy) {
2340
- // Process- and daemon-level handles come off FIRST, before anything that
2341
- // can throw. These used to be removed after captureCheckpoint(), so a
2342
- // throw there — which a half-started org can provoke, since it may be
2343
- // missing state a checkpoint expects — aborted the whole stop and left a
2344
- // process 'exit' listener, an interval and a broker lease behind for a run
2345
- // that no longer exists. startOrg()'s teardown-on-failure path swallows a
2346
- // rejecting stopOrg (it has its own error to report), so the leak was
2347
- // silent.
2348
- const cleanup = org._crashCleanup;
2349
- if (cleanup)
2350
- process.removeListener('exit', cleanup);
2351
- const wd = this.watchdogs.get(name);
2352
- if (wd) {
2353
- clearInterval(wd);
2354
- this.watchdogs.delete(name);
2355
- }
2356
- clearIdleRecord(this.root, name);
2357
- // The run's gates are authoritative; put them back over whatever the file
2358
- // holds now (a role may have rewritten it).
2359
- if (org.gates) {
2360
- try {
2361
- decisionOps.writeGates(this.root, name, org.gates);
2362
- }
2363
- catch {
2364
- /* the next start reads the last write-through */
2365
- }
2366
- }
2367
- this.leases.get(name)?.stop();
2368
- this.leases.delete(name);
2369
- // Capture THIS run's forwarder now: an autoWake-restart of the same org
2370
- // during the long tail below (agent wait, flush, history write) would
2371
- // register a NEW forwarder under the same name — settling/unsubscribing
2372
- // that one would sever the new run's dashboard stream.
2373
- const forwarder = this.forwarders.get(name);
2374
- // Snapshot checkpoint BEFORE closing mailboxes / draining sessions — the
2375
- // queue is emptied during the drain, so capturing afterwards loses all
2376
- // unconsumed messages (the whole point of checkpoint-resume). Best-effort:
2377
- // a run that cannot be checkpointed must still be stopped and cleaned up.
2378
- let stopCheckpoint;
2379
- try {
2380
- stopCheckpoint = captureCheckpoint(org, 'stopped');
2381
- }
2382
- catch (err) {
2383
- console.error(`org ${name}: could not capture the stop checkpoint:`, err instanceof Error ? err.message : err);
2384
- }
2385
- // #275: drop task dispatches still inside their coalescing window — the
2386
- // mailboxes they target are closed on the next line anyway.
2387
- for (const held of org.pendingDispatch?.values() ?? [])
2388
- clearTimeout(held.timer);
2389
- org.pendingDispatch?.clear();
2390
- for (const a of org.agents.values())
2391
- a.mailbox.close();
2392
- // Closing the mailbox stops new work being handed to a session, but does
2393
- // NOT cancel a turn already in flight (e.g. mid provider call) — that
2394
- // session can keep running, and eventually crash/finish, well past this
2395
- // function's own bounded drain below. Abort each slot's live incarnation
2396
- // too, reusing respawnRole's existing force-stop handle, so in-flight
2397
- // work is told to stop now instead of merely being denied new input.
2398
- for (const slot of org.roleSlots.values())
2399
- slot.abort?.abort();
2400
- // M1: kill every tool-provider process of this org's sessions.
2401
- this.toolProviders.closeOrg(name);
2402
- // M2: stop endpoint retry timers (queued entries stay queued).
2403
- stopEndpointRetries(this, name);
2404
- // Bounded: a genuinely hung agent session (stuck mid-tool-call, not just
2405
- // idle) must not make stopOrg() hang forever — callers like the scheduler
2406
- // already race their own timeout around a run, and this wait re-blocking
2407
- // unboundedly on the same never-resolving promises defeated that bound.
2408
- // A planned completion is not an abort. The boss declaring the cycle done
2409
- // says nothing about its siblings: they are routinely mid-build or mid-edit
2410
- // when it fires, and a 15s window SIGTERM'd them (exit 143, reported as
2411
- // "crashed") and threw the work away. allSettled resolves as soon as every
2412
- // session ends, so a long drain is a ceiling, not a delay.
2413
- const stopWaitMs = drainMs ?? this.opts.stopWaitMs ?? 15_000;
2414
- const allDone = Promise.allSettled([...org.agents.values()].map((a) => a.done)).then(() => false);
2415
- // Clear the ceiling timer once the sessions win the race: left pending, a
2416
- // COMPLETE_DRAIN_MS stop kept `org run` (which returns without
2417
- // process.exit on a clean completion) alive for up to five minutes after
2418
- // every session had already ended. Deliberately NOT unref'd — on the
2419
- // timed-out path this timer may be the only thing keeping the loop alive
2420
- // long enough to write 'stopped' to runtime.json and flush the bus.
2421
- let drainTimer;
2422
- const timedOut = await Promise.race([
2423
- allDone,
2424
- new Promise((r) => {
2425
- drainTimer = setTimeout(() => r(true), stopWaitMs);
2426
- }),
2427
- ]);
2428
- clearTimeout(drainTimer);
2429
- if (timedOut) {
2430
- // #152: "proceeding anyway" alone didn't say WHO got cut off — a run
2431
- // reviewer had no way to tell whether real, in-progress work (a
2432
- // mid-build, a mid-write) was force-stopped, or the drain window
2433
- // simply outlived a handful of already-idle sessions. status is only
2434
- // 'ended'/'crashed' once a role's session promise has actually
2435
- // settled; still 'running' here means it was mid-turn when the
2436
- // ceiling hit, not merely idle-but-not-yet-reaped.
2437
- const stillActive = [...org.agents.entries()]
2438
- .filter(([, a]) => a.status === 'running')
2439
- .map(([roleId]) => roleId);
2440
- const rosterSuffix = stillActive.length ? ` — still active: ${stillActive.join(', ')}` : '';
2441
- org.bus.emit({
2442
- type: 'audit',
2443
- msg: `org stop timed out after ${stopWaitMs}ms waiting for agent sessions to finish — proceeding anyway${rosterSuffix}`,
2444
- reason: 'stop-timeout',
2445
- data: { stillActive },
2446
- });
2447
- // Reap only SDK processes spawned by THIS node process — ownerPid filter
2448
- // ensures other `monomind org run` daemons' agents are untouched.
2449
- try {
2450
- const reaped = reapOrphanedSdkProcesses(new Set(), process.pid);
2451
- if (reaped > 0)
2452
- org.bus.emit({
2453
- type: 'audit',
2454
- reason: 'orphan-reap',
2455
- msg: `reaped ${reaped} orphaned SDK process(es) after stop timeout`,
2456
- });
2457
- }
2458
- catch {
2459
- /* best-effort */
2460
- }
2461
- }
2462
- // The run's sessions are gone: take down the sandbox stubs it held.
2463
- sandboxStubs.release(`${name}:${org.run}`);
2464
- // #302 truth gate: every stop path funnels through here, so this is the
2465
- // one place that can record how the run ACTUALLY ended, regardless of
2466
- // which of the five paths triggered it. `closedBy` is undefined only for
2467
- // a bare manual `stopOrg(name)` (CLI `org stop`, shutdown) — every
2468
- // automated path above now tags its own real cause. reporting.ts reads
2469
- // this event (reason: 'org-stopped') to decide whether the run's outcome
2470
- // may be rendered as a boss-attributed 'partial'/'achieved' at all: only
2471
- // closedBy === 'org-complete' may be.
2472
- const runnableTasks = org.taskDag?.pendingTaskCount() ?? 0;
2473
- // Rendered, not just recorded (#302 AC6, same reasoning as the
2474
- // blockerSuffix above): `org logs` prints `msg` verbatim.
2475
- const stopSuffix = closedBy && closedBy !== 'org-complete'
2476
- ? ` (${closedBy}${runnableTasks > 0 ? `, ${runnableTasks} task(s) left` : ''})`
2477
- : '';
2478
- org.bus.emit({
2479
- type: 'status',
2480
- reason: 'org-stopped',
2481
- msg: `org stopped${stopSuffix}`,
2482
- data: { closedBy, runnableTasks },
2483
- });
2484
- await org.bus.flush();
2485
- // Append this run's summary to <org>/history.jsonl — read back from the
2486
- // flushed bus.jsonl (the full durable record) rather than the bounded
2487
- // in-memory buffer, so long runs summarize completely.
2488
- //
2489
- // This block runs BEFORE the seal below (#293): storeRunMemory emits an
2490
- // audit event when the run's memory could not be stored, and a sealed bus
2491
- // fans out to in-memory listeners without ever reaching bus.jsonl — an
2492
- // event the live view shows and the durable record does not, which is both
2493
- // the divergence test-loop's `persisted` check exists to catch and useless
2494
- // to whoever reads the run back later. Sealing after it keeps every emitted
2495
- // event durable. The seal still closes before this function returns, which
2496
- // is what its own contract (below) is about.
2497
- try {
2498
- const events = readRunEvents(this.root, name, org.run);
2499
- if (events.length) {
2500
- const summary = summarizeRun(events);
2501
- const { appendFileSync } = await import('node:fs');
2502
- appendFileSync(historyFile(this.root, name), `${JSON.stringify(summary)}\n`, 'utf8');
2503
- // Cross-run memory: make this run's outcome recallable by meaning.
2504
- // #293: the result is CHECKED — a store that silently did nothing used
2505
- // to be indistinguishable from one that worked, and the symptom
2506
- // (org_recall always empty) showed up runs later with no trail. The
2507
- // reason is stashed for persistState() below so runtime.json — and
2508
- // therefore `org status` — carries it after the bus event and the
2509
- // stderr warning have scrolled away.
2510
- const memory = await this.storeRunMemory(name, org.def, org.run, summary, org.bus);
2511
- if (memory.stored)
2512
- this.memoryErrors.delete(name);
2513
- else
2514
- this.memoryErrors.set(name, memory.reason ?? 'unknown');
2515
- }
2516
- }
2517
- catch (err) {
2518
- console.error(`org ${name}: could not write run history:`, err instanceof Error ? err.message : err);
2519
- }
2520
- finally {
2521
- this.recallUsage.delete(name);
2522
- this.orgLearnedRuns.delete(`${name}:${org.run}`);
2523
- }
2524
- // flush() only awaits a snapshot of writes queued at call time (see its
2525
- // own doc comment) — it has no visibility into a session that crashes
2526
- // after the abort signal above but before this function returns. Seal
2527
- // the bus now so any such late bus.emit() still reaches in-memory
2528
- // listeners but can never schedule a new disk write into a run
2529
- // directory a caller (e.g. a test's afterEach) may already be deleting.
2530
- // seal() awaits the pending writes first, so the audit event the block
2531
- // above may have emitted is on disk before the bus closes.
2532
- await org.bus.seal();
2533
- // the "org stopped" event above triggers the forwarder's final org:complete /
2534
- // session:complete POST — without waiting for it here, the CLI process can exit
2535
- // (and kill the in-flight fetch) before that last event reaches the dashboard,
2536
- // leaving the run stuck showing "running" forever. Bounded: a stalled
2537
- // dashboard must not hang org shutdown indefinitely.
2538
- if (forwarder) {
2539
- await Promise.race([
2540
- forwarder.settle(),
2541
- new Promise((r) => {
2542
- const t = setTimeout(r, 5_000);
2543
- t.unref?.();
2544
- }),
2545
- ]);
2546
- forwarder.unsubscribe();
2547
- // Only remove from the map if it's still OURS — an autoWake-restart may
2548
- // have registered the new run's forwarder under this name meanwhile.
2549
- if (this.forwarders.get(name) === forwarder)
2550
- this.forwarders.delete(name);
2551
- }
2552
- // Same guard for runtime.json: if a new run started during shutdown, its
2553
- // 'running' record must not be overwritten with this old run's 'stopped'.
2554
- // Pass the org directly since we already removed it from the map.
2555
- if (!this.orgs.has(name))
2556
- this.persistState(name, 'stopped', org.run, org, stopCheckpoint, closedBy);
2557
- // Clean up git worktrees — shared (workspace: 'worktree') and per-role.
2558
- try {
2559
- const { execFileSync } = await import('node:child_process');
2560
- if (org.worktreePath) {
2561
- try {
2562
- execFileSync('git', ['worktree', 'remove', '--force', org.worktreePath], {
2563
- cwd: this.root,
2564
- stdio: 'ignore',
2565
- timeout: 30_000,
2566
- });
2567
- }
2568
- catch {
2569
- /* best-effort */
2570
- }
2571
- }
2572
- for (const agent of org.agents.values()) {
2573
- if (agent.worktreePath) {
2574
- try {
2575
- execFileSync('git', ['worktree', 'remove', '--force', agent.worktreePath], {
2576
- cwd: this.root,
2577
- stdio: 'ignore',
2578
- timeout: 30_000,
2579
- });
2580
- }
2581
- catch {
2582
- /* best-effort */
2583
- }
2584
- }
2585
- }
2586
- // #301: roles create their own linked worktrees with Bash (paths the
2587
- // daemon never recorded — org.worktreePath/agent.worktreePath above are
2588
- // only ever set for workspace: 'worktree'/'worktree-per-role', empty
2589
- // for the common workspace: 'repo' shape), and deleting the working
2590
- // directory from inside a role sandbox leaves .git/worktrees/<name>
2591
- // behind — `git worktree list` then hides it, and it never gets
2592
- // cleaned up. Unconditional on purpose: gating this on
2593
- // org/agent.worktreePath would skip exactly the runs that hit the bug.
2594
- // prune only drops metadata whose worktree directory is already gone,
2595
- // so a live worktree — including the owner's — is never touched; it is
2596
- // idempotent; and the two removals just above already run `git
2597
- // worktree remove --force` against this same repo from this same cwd,
2598
- // so this is strictly less invasive than what already ships. Run after
2599
- // both removal loops so a worktree just removed is also pruned.
2600
- //
2601
- // Bounded race, measured rather than assumed (same treatment as the
2602
- // SIGKILL case above): an entry whose `gitdir` file is absent is
2603
- // pruned unconditionally, and `--expire` cannot protect it — measured
2604
- // across every window from `--expire=now` to `--expire=3.months.ago`,
2605
- // a fresh no-gitdir entry is removed regardless, while `--expire` also
2606
- // makes an already-deleted worktree SURVIVE, breaking the "a run
2607
- // always begins clean" guarantee this fix exists to provide. So a
2608
- // concurrent `git worktree add` by another process in this repo is
2609
- // vulnerable for the microseconds between its `mkdir` and its
2610
- // `gitdir` write. A mid-creation state cannot persist longer than
2611
- // that, so an entry found in that state is dead metadata, not a live
2612
- // worktree in progress.
2613
- try {
2614
- execFileSync('git', ['worktree', 'prune'], {
2615
- cwd: this.root,
2616
- stdio: 'ignore',
2617
- timeout: 30_000,
2618
- });
2619
- }
2620
- catch {
2621
- /* best-effort: not a git repo, git missing, or a wedged hook */
2622
- }
2623
- }
2624
- catch {
2625
- /* node:child_process unavailable — skip */
2626
- }
210
+ return orgStop.stopOrg(this, name, opts);
2627
211
  }
2628
212
  async stopAll() {
2629
213
  await Promise.all([
@@ -2638,52 +222,7 @@ export class OrgDaemon {
2638
222
  * stop`). Mirrors persistCrashStateAll()'s existing closedBy: 'crash-handler'
2639
223
  * for the process-crash path, which org.ts already reads. */
2640
224
  persistState(name, status, run, org, checkpointOverride, closedBy) {
2641
- const p = join(this.root, ORG_DIR, name, 'runtime.json');
2642
- const missing = [...(this.abandoned.get(name) ?? [])];
2643
- const memoryError = this.memoryErrors.get(name);
2644
- const running = org ?? this.orgs.get(name);
2645
- const validStatus = status === 'stopped' || status === 'crashed' ? status : 'running';
2646
- // Pattern 3: Capture full checkpoint state for resume. On stop, finishStop
2647
- // passes a snapshot captured BEFORE mailboxes close and sessions drain —
2648
- // otherwise the queue is always empty by persist time.
2649
- let checkpoint = checkpointOverride ?? null;
2650
- if (!checkpoint && running) {
2651
- // Best-effort, like the snapshot in finishStop: persisting the run's
2652
- // state matters more than the resume checkpoint inside it, and a stop
2653
- // must not fail because a checkpoint could not be built.
2654
- try {
2655
- checkpoint = captureCheckpoint(running, validStatus);
2656
- }
2657
- catch (err) {
2658
- console.error(`org ${name}: could not capture the ${validStatus} checkpoint:`, err instanceof Error ? err.message : err);
2659
- checkpoint = null;
2660
- }
2661
- }
2662
- else if (checkpoint && checkpoint.status !== validStatus) {
2663
- const { checksum: _, ...state } = checkpoint;
2664
- checkpoint = {
2665
- ...state,
2666
- status: validStatus,
2667
- checksum: generateChecksum({
2668
- ...state,
2669
- status: validStatus,
2670
- }),
2671
- };
2672
- }
2673
- // C4: writeJsonFileAtomic (tmp + rename) — a direct writeFileSync here
2674
- // could leave runtime.json truncated on Ctrl-C during `org stop`, which
2675
- // would brick every subsequent `org status` / isOrgRunning / scheduler
2676
- // call. The state files in 6 other daemon paths already use this helper.
2677
- writeJsonFileAtomic(p, {
2678
- status,
2679
- run,
2680
- pid: process.pid,
2681
- updated: new Date().toISOString(),
2682
- ...(missing.length ? { abandonedRoles: missing } : {}),
2683
- ...(memoryError ? { memoryError } : {}),
2684
- ...(checkpoint ? { checkpoint } : {}),
2685
- ...(closedBy ? { closedBy } : {}),
2686
- });
225
+ orgStateFile.persistState(this, name, status, run, org, checkpointOverride, closedBy);
2687
226
  }
2688
227
  /** Mark every currently-running org as crashed in runtime.json.
2689
228
  * Called from process-level crash handlers — must be synchronous and best-effort.
@@ -2691,68 +230,19 @@ export class OrgDaemon {
2691
230
  * this, `runOutcomeResult` (org.ts)'s "crashed: <error>" message always
2692
231
  * read "crashed: unknown error" regardless of what actually happened. */
2693
232
  persistCrashStateAll(error) {
2694
- for (const [name, org] of this.orgs) {
2695
- try {
2696
- const p = join(this.root, ORG_DIR, name, 'runtime.json');
2697
- // Capture separately from the write below: a throw here (e.g. a
2698
- // cyclic structure in roleState reaching generateChecksum) must not
2699
- // suppress the base crash record, which is the actually-important
2700
- // best-effort write this method exists for.
2701
- let checkpoint;
2702
- try {
2703
- checkpoint = captureCheckpoint(org, 'crashed');
2704
- }
2705
- catch {
2706
- /* best effort — proceed without a checkpoint */
2707
- }
2708
- // C4: atomic write — crash handler is the most likely place to hit
2709
- // a partial write since the process is mid-teardown.
2710
- writeJsonFileAtomic(p, {
2711
- status: 'crashed',
2712
- run: org.run,
2713
- pid: process.pid,
2714
- updated: new Date().toISOString(),
2715
- closedBy: 'crash-handler',
2716
- ...(checkpoint ? { checkpoint } : {}),
2717
- ...(error ? { error } : {}),
2718
- });
2719
- }
2720
- catch {
2721
- /* best effort — filesystem may be unavailable */
2722
- }
2723
- }
2724
- }
2725
- heartbeatPath() {
2726
- return join(this.root, '.monomind', 'serve-heartbeat.json');
233
+ orgStateFile.persistCrashStateAll(this, error);
2727
234
  }
2728
235
  /** Write a heartbeat file so `org status` can distinguish "daemon alive" from
2729
236
  * "daemon gone" even when runtime.json still says running. */
2730
237
  writeHeartbeat() {
2731
- try {
2732
- const p = this.heartbeatPath();
2733
- mkdirSync(join(this.root, '.monomind'), { recursive: true });
2734
- // C4: atomic write — heartbeat corruption is how `org status` reports
2735
- // a phantom daemon after a crash.
2736
- writeJsonFileAtomic(p, {
2737
- pid: process.pid,
2738
- updatedAt: new Date().toISOString(),
2739
- running: this.listRunning(),
2740
- });
2741
- }
2742
- catch {
2743
- /* best effort */
2744
- }
238
+ orgStateFile.writeHeartbeat(this);
2745
239
  }
2746
240
  clearHeartbeat() {
2747
- try {
2748
- unlinkSync(this.heartbeatPath());
2749
- }
2750
- catch {
2751
- /* already gone or never written */
2752
- }
241
+ orgStateFile.clearHeartbeat(this);
2753
242
  }
2754
243
  // ── Delegated methods — extracted to focused modules ──────────────────
2755
244
  // approvals.ts
245
+ /** @internal */
2756
246
  checkApproval(org, role, action, input) {
2757
247
  return approvalOps.checkApproval(this, org, role, action, input);
2758
248
  }
@@ -2767,6 +257,7 @@ export class OrgDaemon {
2767
257
  return questionOps.answerQuestion(this, org, role, questionId, answer, resolvedBy);
2768
258
  }
2769
259
  // decisions.ts
260
+ /** @internal */
2770
261
  readGates(org) {
2771
262
  return decisionOps.gatesFor(this, org);
2772
263
  }
@@ -2779,9 +270,11 @@ export class OrgDaemon {
2779
270
  listGates(org, status) {
2780
271
  return decisionOps.listGates(this, org, status);
2781
272
  }
273
+ /** @internal */
2782
274
  dagCreateTask(org, role, title, assignee, deps, loadout, brief, pick) {
2783
275
  return decisionOps.dagCreateTask(this, org, role, title, assignee, deps, loadout, brief, pick);
2784
276
  }
277
+ /** @internal */
2785
278
  dagCompleteTask(org, role, taskId, result, evidence) {
2786
279
  return decisionOps.dagCompleteTask(this, org, role, taskId, result, evidence);
2787
280
  }
@@ -2789,18 +282,23 @@ export class OrgDaemon {
2789
282
  dagRequestReview(org, role, taskId, reviewer, base) {
2790
283
  return decisionOps.dagRequestReview(this, org, role, taskId, reviewer, base);
2791
284
  }
285
+ /** @internal */
2792
286
  dagSplitTask(org, role, parentId, children) {
2793
287
  return decisionOps.dagSplitTask(this, org, role, parentId, children);
2794
288
  }
289
+ /** @internal */
2795
290
  dagMergeTask(org, role, sourceId, targetId) {
2796
291
  return decisionOps.dagMergeTask(this, org, role, sourceId, targetId);
2797
292
  }
293
+ /** @internal */
2798
294
  dagCancelTask(org, role, taskId, reason) {
2799
295
  return decisionOps.dagCancelTask(this, org, role, taskId, reason);
2800
296
  }
297
+ /** @internal */
2801
298
  dagBlockTask(org, role, taskId, untilIso, reason, recheckAfterMinutes) {
2802
299
  return decisionOps.dagBlockTask(this, org, role, taskId, untilIso, reason, recheckAfterMinutes);
2803
300
  }
301
+ /** @internal */
2804
302
  dagPlanGraph(org, role, specs) {
2805
303
  return decisionOps.dagPlanGraph(this, org, role, specs);
2806
304
  }
@@ -2823,6 +321,7 @@ export class OrgDaemon {
2823
321
  autoWake(name) {
2824
322
  scheduler.autoWake(this, name);
2825
323
  }
324
+ /** @internal */
2826
325
  scheduleBossRestart(name) {
2827
326
  scheduler.scheduleBossRestart(this, name);
2828
327
  }
@@ -2841,24 +340,30 @@ export class OrgDaemon {
2841
340
  orgMemoryNamespace(name, def) {
2842
341
  return orgMemory.orgMemoryNamespace(name, def);
2843
342
  }
343
+ /** @internal */
2844
344
  orgMemoryDbPath() {
2845
345
  return orgMemory.orgMemoryDbPath(this.root);
2846
346
  }
347
+ /** @internal */
2847
348
  orgMemoryUsable() {
2848
349
  return orgMemory.orgMemoryUsable(this.root);
2849
350
  }
351
+ /** @internal */
2850
352
  async rememberOrgMemory(name, def, role, content, scope, run) {
2851
353
  return orgMemory.rememberOrgMemory(this.root, name, def, role, content, scope, run);
2852
354
  }
355
+ /** @internal */
2853
356
  async recallOrgMemory(name, def, query, role) {
2854
357
  return orgMemory.recallOrgMemory(this, name, def, query, role);
2855
358
  }
2856
359
  async searchProjectKnowledge(query) {
2857
360
  return orgMemory.searchProjectKnowledge(this.root, query);
2858
361
  }
362
+ /** @internal */
2859
363
  async learnOrgKnowledge(name, run, payload) {
2860
364
  return orgMemory.learnOrgKnowledge(this, name, run, payload);
2861
365
  }
366
+ /** @internal */
2862
367
  async storeRunMemory(name, def, run, summary, bus) {
2863
368
  return orgMemory.storeRunMemory(this, name, def, run, summary, bus);
2864
369
  }