@adhdev/daemon-core 0.9.82-rc.50 → 0.9.82-rc.501

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (478) hide show
  1. package/dist/boot/daemon-lifecycle.d.ts +9 -0
  2. package/dist/boot/process-hardening.d.ts +50 -0
  3. package/dist/build-info.d.ts +37 -0
  4. package/dist/chat/source-machine.d.ts +166 -0
  5. package/dist/chat/source-resolver.d.ts +104 -0
  6. package/dist/chat/subscription-updates.d.ts +1 -0
  7. package/dist/cli-adapter-types.d.ts +85 -2
  8. package/dist/cli-adapters/cli-script-runner.d.ts +117 -0
  9. package/dist/cli-adapters/cli-state-engine.d.ts +246 -0
  10. package/dist/cli-adapters/provider-cli-adapter.d.ts +139 -64
  11. package/dist/cli-adapters/provider-cli-parse.d.ts +5 -0
  12. package/dist/cli-adapters/provider-cli-runtime.d.ts +34 -0
  13. package/dist/cli-adapters/provider-cli-shared.d.ts +74 -0
  14. package/dist/cli-adapters/pty-write-chunking.d.ts +34 -0
  15. package/dist/cli-adapters/raw-terminal-io.d.ts +37 -0
  16. package/dist/cli-adapters/resolve-executable.d.ts +69 -0
  17. package/dist/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +6 -3
  18. package/dist/cli-adapters/terminal-backends/types.d.ts +10 -2
  19. package/dist/cli-adapters/terminal-screen.d.ts +4 -11
  20. package/dist/commands/chat-commands-debug-bundle.d.ts +14 -0
  21. package/dist/commands/chat-commands-read.d.ts +23 -0
  22. package/dist/commands/chat-commands-scope.d.ts +39 -0
  23. package/dist/commands/chat-commands-shared.d.ts +33 -0
  24. package/dist/commands/chat-commands-write.d.ts +14 -0
  25. package/dist/commands/chat-commands.d.ts +10 -23
  26. package/dist/commands/cli-manager.d.ts +71 -1
  27. package/dist/commands/handler.d.ts +125 -0
  28. package/dist/commands/high-family/index.d.ts +3 -0
  29. package/dist/commands/high-family/mesh-coordinator-launch.d.ts +2 -0
  30. package/dist/commands/high-family/mesh-events.d.ts +2 -0
  31. package/dist/commands/high-family/mesh-status.d.ts +2 -0
  32. package/dist/commands/high-family/types.d.ts +64 -0
  33. package/dist/commands/low-family/coordinator-prompt.d.ts +9 -0
  34. package/dist/commands/low-family/daemon-lifecycle.d.ts +2 -0
  35. package/dist/commands/low-family/diagnostics.d.ts +2 -0
  36. package/dist/commands/low-family/index.d.ts +3 -0
  37. package/dist/commands/low-family/mesh-ledger.d.ts +10 -0
  38. package/dist/commands/low-family/mesh-node-logs.d.ts +2 -0
  39. package/dist/commands/low-family/notification.d.ts +2 -0
  40. package/dist/commands/low-family/refine-config.d.ts +2 -0
  41. package/dist/commands/low-family/session-host.d.ts +2 -0
  42. package/dist/commands/low-family/spec-providerdev.d.ts +11 -0
  43. package/dist/commands/low-family/status-meta.d.ts +2 -0
  44. package/dist/commands/low-family/types.d.ts +33 -0
  45. package/dist/commands/med-family/cli-agent.d.ts +2 -0
  46. package/dist/commands/med-family/fast-forward.d.ts +2 -0
  47. package/dist/commands/med-family/ide.d.ts +10 -0
  48. package/dist/commands/med-family/index.d.ts +3 -0
  49. package/dist/commands/med-family/mesh-crud.d.ts +48 -0
  50. package/dist/commands/med-family/mesh-host-pairing.d.ts +2 -0
  51. package/dist/commands/med-family/mesh-queue.d.ts +2 -0
  52. package/dist/commands/med-family/mesh-restart.d.ts +2 -0
  53. package/dist/commands/med-family/types.d.ts +152 -0
  54. package/dist/commands/mesh-coordinator.d.ts +85 -1
  55. package/dist/commands/router-aggregate-status.d.ts +24 -0
  56. package/dist/commands/router-mesh-session-owner.d.ts +59 -0
  57. package/dist/commands/router-refine.d.ts +151 -0
  58. package/dist/commands/router-worktree-cleanup.d.ts +133 -0
  59. package/dist/commands/router.d.ts +203 -15
  60. package/dist/commands/upgrade-helper.d.ts +41 -1
  61. package/dist/config/chat-history.d.ts +11 -0
  62. package/dist/config/config.d.ts +19 -0
  63. package/dist/config/mesh-config.d.ts +66 -0
  64. package/dist/config/mesh-json-config.d.ts +199 -0
  65. package/dist/config/registry-resolver.d.ts +54 -0
  66. package/dist/config/repo-settings.d.ts +77 -0
  67. package/dist/config/state-store.d.ts +30 -0
  68. package/dist/daemon/dev-server.d.ts +0 -2
  69. package/dist/detection/cli-detector.d.ts +17 -0
  70. package/dist/detection/ide-detector.d.ts +13 -0
  71. package/dist/detection/win32-ide-version.d.ts +37 -0
  72. package/dist/git/change-impact-config.d.ts +159 -0
  73. package/dist/git/git-commands.d.ts +26 -2
  74. package/dist/git/git-diff.d.ts +6 -0
  75. package/dist/git/git-executor.d.ts +11 -0
  76. package/dist/git/git-status.d.ts +80 -0
  77. package/dist/git/git-types.d.ts +2 -50
  78. package/dist/git/git-worktree.d.ts +71 -1
  79. package/dist/git/index.d.ts +3 -1
  80. package/dist/index.d.ts +71 -17
  81. package/dist/index.js +56455 -21586
  82. package/dist/index.js.map +1 -1
  83. package/dist/index.mjs +53763 -19064
  84. package/dist/index.mjs.map +1 -1
  85. package/dist/installer.d.ts +1 -4
  86. package/dist/ipc/local-ipc-server.d.ts +91 -0
  87. package/dist/launch.d.ts +1 -1
  88. package/dist/logging/async-batch-writer.d.ts +10 -0
  89. package/dist/logging/debug-config.d.ts +16 -0
  90. package/dist/logging/log-redactor.d.ts +24 -0
  91. package/dist/logging/log-tail-reader.d.ts +81 -0
  92. package/dist/logging/logger.d.ts +1 -1
  93. package/dist/mesh/contracts.d.ts +256 -0
  94. package/dist/mesh/coordinator-prompt.d.ts +76 -0
  95. package/dist/mesh/coordinator-registry.d.ts +59 -0
  96. package/dist/mesh/mesh-active-work.d.ts +203 -0
  97. package/dist/mesh/mesh-clone-grace.d.ts +8 -0
  98. package/dist/mesh/mesh-completion-synthesis.d.ts +4 -0
  99. package/dist/mesh/mesh-coordinator-config.d.ts +21 -0
  100. package/dist/mesh/mesh-delivery-policy.d.ts +104 -0
  101. package/dist/mesh/mesh-event-classify.d.ts +7 -0
  102. package/dist/mesh/mesh-event-forwarding.d.ts +58 -0
  103. package/dist/mesh/mesh-event-trace.d.ts +21 -0
  104. package/dist/mesh/mesh-events-coordinator.d.ts +4 -0
  105. package/dist/mesh/mesh-events-pending.d.ts +258 -0
  106. package/dist/mesh/mesh-events-stale.d.ts +61 -0
  107. package/dist/mesh/mesh-events-utils.d.ts +75 -0
  108. package/dist/mesh/mesh-events.d.ts +6 -49
  109. package/dist/mesh/mesh-fast-forward.d.ts +81 -0
  110. package/dist/mesh/mesh-host-ownership.d.ts +21 -1
  111. package/dist/mesh/mesh-idle-reminder.d.ts +68 -0
  112. package/dist/mesh/mesh-init.d.ts +128 -0
  113. package/dist/mesh/mesh-ledger-reconciliation.d.ts +23 -4
  114. package/dist/mesh/mesh-ledger.d.ts +139 -1
  115. package/dist/mesh/mesh-magi-status.d.ts +63 -0
  116. package/dist/mesh/mesh-missions.d.ts +281 -0
  117. package/dist/mesh/mesh-node-identity.d.ts +310 -0
  118. package/dist/mesh/mesh-queue-assignment.d.ts +162 -0
  119. package/dist/mesh/mesh-reconcile-acked-hold.d.ts +17 -0
  120. package/dist/mesh/mesh-reconcile-config.d.ts +6 -0
  121. package/dist/mesh/mesh-reconcile-identity.d.ts +6 -0
  122. package/dist/mesh/mesh-reconcile-loop.d.ts +84 -0
  123. package/dist/mesh/mesh-reconcile-v2-backstop.d.ts +16 -0
  124. package/dist/mesh/mesh-refine-batch.d.ts +68 -0
  125. package/dist/mesh/mesh-refine-gates.d.ts +457 -0
  126. package/dist/mesh/mesh-refine-status.d.ts +62 -0
  127. package/dist/mesh/mesh-remote-event-pull.d.ts +15 -0
  128. package/dist/mesh/mesh-review-inbox.d.ts +90 -0
  129. package/dist/mesh/mesh-routing.d.ts +70 -0
  130. package/dist/mesh/mesh-runtime-store.d.ts +584 -0
  131. package/dist/mesh/mesh-scheduling-runtime.d.ts +78 -0
  132. package/dist/mesh/mesh-task-inflight.d.ts +46 -0
  133. package/dist/mesh/mesh-task-stats.d.ts +49 -0
  134. package/dist/mesh/mesh-unresolved-forward-outbox.d.ts +34 -0
  135. package/dist/mesh/mesh-warmup-deadline.d.ts +68 -0
  136. package/dist/mesh/mesh-work-queue.d.ts +392 -1
  137. package/dist/mesh/preview-freshness.d.ts +28 -0
  138. package/dist/mesh/refine-config.d.ts +97 -0
  139. package/dist/mesh/worktree-bootstrap-config.d.ts +219 -0
  140. package/dist/providers/acp-provider-instance.d.ts +5 -0
  141. package/dist/providers/approval-utils.d.ts +46 -0
  142. package/dist/providers/chat-message-normalization.d.ts +58 -1
  143. package/dist/providers/cli-provider-effect-format.d.ts +30 -0
  144. package/dist/providers/cli-provider-history-dedup.d.ts +17 -0
  145. package/dist/providers/cli-provider-input-prompt.d.ts +12 -0
  146. package/dist/providers/cli-provider-instance-types.d.ts +47 -0
  147. package/dist/providers/cli-provider-instance.d.ts +461 -30
  148. package/dist/providers/cli-provider-status-helpers.d.ts +46 -0
  149. package/dist/providers/cli-provider-transcript-merge.d.ts +7 -0
  150. package/dist/providers/contracts.d.ts +196 -6
  151. package/dist/providers/external-sources.d.ts +71 -0
  152. package/dist/providers/manual-attendance.d.ts +79 -0
  153. package/dist/providers/native-history/antigravity-claim-registry.d.ts +39 -0
  154. package/dist/providers/native-history/antigravity-cli-transcript.d.ts +137 -0
  155. package/dist/providers/native-history/claude-cli-transcript.d.ts +70 -0
  156. package/dist/providers/native-history/codex-cli-transcript.d.ts +73 -0
  157. package/dist/providers/native-history/constants.d.ts +12 -0
  158. package/dist/providers/native-history/dispatcher.d.ts +44 -0
  159. package/dist/providers/native-history/hermes-cli-transcript.d.ts +30 -0
  160. package/dist/providers/native-history/index.d.ts +13 -0
  161. package/dist/providers/provider-instance-manager.d.ts +30 -0
  162. package/dist/providers/provider-instance.d.ts +31 -1
  163. package/dist/providers/provider-loader.d.ts +42 -5
  164. package/dist/providers/provider-trust.d.ts +31 -0
  165. package/dist/providers/read-chat-contract.d.ts +29 -0
  166. package/dist/providers/sdk/v1/builders/acp/detect-status.d.ts +68 -0
  167. package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +86 -0
  168. package/dist/providers/sdk/v1/builders/cli/parse-approval-squash.d.ts +59 -0
  169. package/dist/providers/sdk/v1/builders/cli/parse-approval.d.ts +65 -0
  170. package/dist/providers/sdk/v1/builders/cli/parse-session.d.ts +91 -0
  171. package/dist/providers/sdk/v1/builders/cli/visible-region.d.ts +42 -0
  172. package/dist/providers/sdk/v1/fixture-tooling/format.d.ts +126 -0
  173. package/dist/providers/sdk/v1/fixture-tooling/index.d.ts +8 -0
  174. package/dist/providers/sdk/v1/fixture-tooling/replay.d.ts +38 -0
  175. package/dist/providers/sdk/v1/index.d.ts +30 -0
  176. package/dist/providers/sdk/v1/sandbox/README-design.d.ts +193 -0
  177. package/dist/providers/sdk/v1/sandbox/require-whitelist.d.ts +74 -0
  178. package/dist/providers/sdk/v1/sandbox/script-runner.d.ts +98 -0
  179. package/dist/providers/sdk/v1/types/cli/index.d.ts +277 -0
  180. package/dist/providers/sdk/v1/types/common/index.d.ts +203 -0
  181. package/dist/providers/sdk/v1/validators/index.d.ts +5 -0
  182. package/dist/providers/sdk/v1/validators/manifest.d.ts +40 -0
  183. package/dist/providers/sdk/v1/validators/taint.d.ts +52 -0
  184. package/dist/providers/spec/adapter.d.ts +98 -0
  185. package/dist/providers/spec/background-task-detector.d.ts +29 -0
  186. package/dist/providers/spec/cli-adapter.d.ts +224 -0
  187. package/dist/providers/spec/evaluator.d.ts +45 -0
  188. package/dist/providers/spec/fsm-driver.d.ts +449 -0
  189. package/dist/providers/spec/fsm-evaluator.d.ts +93 -0
  190. package/dist/providers/spec/fsm-loader.d.ts +14 -0
  191. package/dist/providers/spec/fsm-types.d.ts +202 -0
  192. package/dist/providers/spec/native-history-executor.d.ts +50 -0
  193. package/dist/providers/spec/pre-launch-trust.d.ts +16 -0
  194. package/dist/providers/spec/route.d.ts +4 -0
  195. package/dist/providers/spec/types.d.ts +244 -0
  196. package/dist/providers/status-monitor.d.ts +7 -7
  197. package/dist/providers/transcript-v2.d.ts +176 -0
  198. package/dist/providers/types/interactive-prompt.d.ts +84 -0
  199. package/dist/providers/working-dir.d.ts +17 -0
  200. package/dist/repo-mesh-types.d.ts +629 -14
  201. package/dist/runtime-defaults.d.ts +2 -0
  202. package/dist/session-host/managed-host.d.ts +64 -0
  203. package/dist/session-host/runtime-surface.d.ts +10 -16
  204. package/dist/sessions/registry.d.ts +26 -0
  205. package/dist/shared-types-extra.d.ts +2 -4
  206. package/dist/shared-types.d.ts +110 -55
  207. package/dist/status/chat-tail-hot-sessions.d.ts +40 -0
  208. package/dist/status/normalize.d.ts +1 -1
  209. package/dist/status/normalize.js +1 -0
  210. package/dist/status/normalize.js.map +1 -1
  211. package/dist/status/normalize.mjs +1 -0
  212. package/dist/status/normalize.mjs.map +1 -1
  213. package/dist/status/reporter.d.ts +2 -0
  214. package/dist/status/snapshot.d.ts +27 -0
  215. package/dist/system/hash.d.ts +8 -0
  216. package/dist/system/load-better-sqlite3.d.ts +21 -0
  217. package/dist/types.d.ts +20 -6
  218. package/package.json +11 -4
  219. package/src/agent-stream/poller.ts +2 -3
  220. package/src/agent-stream/provider-adapter.ts +1 -1
  221. package/src/boot/daemon-lifecycle.ts +79 -14
  222. package/src/boot/process-hardening.ts +89 -0
  223. package/src/build-info.ts +73 -0
  224. package/src/chat/source-machine.ts +534 -0
  225. package/src/chat/source-resolver.ts +0 -0
  226. package/src/chat/subscription-updates.ts +20 -1
  227. package/src/cli-adapter-types.d.ts +3 -1
  228. package/src/cli-adapter-types.ts +85 -2
  229. package/src/cli-adapters/cli-script-runner.ts +421 -0
  230. package/src/cli-adapters/cli-state-engine.ts +1379 -0
  231. package/src/cli-adapters/provider-cli-adapter.d.ts +1 -1
  232. package/src/cli-adapters/provider-cli-adapter.ts +1059 -1143
  233. package/src/cli-adapters/provider-cli-parse.d.ts +2 -0
  234. package/src/cli-adapters/provider-cli-parse.ts +19 -0
  235. package/src/cli-adapters/provider-cli-runtime.ts +56 -7
  236. package/src/cli-adapters/provider-cli-shared.d.ts +4 -0
  237. package/src/cli-adapters/provider-cli-shared.ts +121 -11
  238. package/src/cli-adapters/pty-transport.ts +2 -1
  239. package/src/cli-adapters/pty-write-chunking.ts +106 -0
  240. package/src/cli-adapters/raw-terminal-io.ts +252 -0
  241. package/src/cli-adapters/resolve-executable.ts +204 -0
  242. package/src/cli-adapters/session-host-transport.ts +2 -1
  243. package/src/cli-adapters/terminal-backends/ghostty-vt-backend.ts +63 -29
  244. package/src/cli-adapters/terminal-backends/types.ts +10 -3
  245. package/src/cli-adapters/terminal-screen.ts +16 -81
  246. package/src/commands/chat-commands-debug-bundle.ts +398 -0
  247. package/src/commands/chat-commands-read.ts +2788 -0
  248. package/src/commands/chat-commands-scope.ts +54 -0
  249. package/src/commands/chat-commands-shared.ts +114 -0
  250. package/src/commands/chat-commands-write.ts +891 -0
  251. package/src/commands/chat-commands.ts +19 -1841
  252. package/src/commands/cli-manager.d.ts +2 -0
  253. package/src/commands/cli-manager.ts +724 -27
  254. package/src/commands/handler.ts +865 -2
  255. package/src/commands/high-family/index.ts +28 -0
  256. package/src/commands/high-family/mesh-coordinator-launch.ts +696 -0
  257. package/src/commands/high-family/mesh-events.ts +102 -0
  258. package/src/commands/high-family/mesh-status.ts +857 -0
  259. package/src/commands/high-family/types.ts +80 -0
  260. package/src/commands/low-family/coordinator-prompt.ts +125 -0
  261. package/src/commands/low-family/daemon-lifecycle.ts +120 -0
  262. package/src/commands/low-family/diagnostics.ts +57 -0
  263. package/src/commands/low-family/index.ts +37 -0
  264. package/src/commands/low-family/mesh-ledger.ts +62 -0
  265. package/src/commands/low-family/mesh-node-logs.ts +87 -0
  266. package/src/commands/low-family/notification.ts +116 -0
  267. package/src/commands/low-family/refine-config.ts +106 -0
  268. package/src/commands/low-family/session-host.ts +282 -0
  269. package/src/commands/low-family/spec-providerdev.ts +217 -0
  270. package/src/commands/low-family/status-meta.ts +112 -0
  271. package/src/commands/low-family/types.ts +39 -0
  272. package/src/commands/med-family/cli-agent.ts +270 -0
  273. package/src/commands/med-family/fast-forward.ts +230 -0
  274. package/src/commands/med-family/ide.ts +163 -0
  275. package/src/commands/med-family/index.ts +37 -0
  276. package/src/commands/med-family/mesh-crud.ts +1309 -0
  277. package/src/commands/med-family/mesh-host-pairing.ts +234 -0
  278. package/src/commands/med-family/mesh-queue.ts +167 -0
  279. package/src/commands/med-family/mesh-restart.ts +92 -0
  280. package/src/commands/med-family/types.ts +153 -0
  281. package/src/commands/mesh-coordinator.ts +334 -124
  282. package/src/commands/router-aggregate-status.ts +209 -0
  283. package/src/commands/router-mesh-session-owner.ts +114 -0
  284. package/src/commands/router-refine.ts +1794 -0
  285. package/src/commands/router-worktree-cleanup.ts +870 -0
  286. package/src/commands/router.ts +567 -4149
  287. package/src/commands/stream-commands.ts +8 -0
  288. package/src/commands/upgrade-helper.ts +310 -45
  289. package/src/config/chat-history.ts +517 -24
  290. package/src/config/config.ts +30 -0
  291. package/src/config/mesh-config.ts +300 -24
  292. package/src/config/mesh-json-config.ts +376 -0
  293. package/src/config/recent-activity.ts +8 -2
  294. package/src/config/registry-resolver.ts +100 -0
  295. package/src/config/repo-settings.ts +111 -0
  296. package/src/config/state-store.ts +55 -0
  297. package/src/daemon/dev-auto-implement.ts +3 -2
  298. package/src/daemon/dev-cli-debug.ts +10 -1
  299. package/src/daemon/dev-server.ts +0 -541
  300. package/src/detection/cli-detector.ts +94 -9
  301. package/src/detection/ide-detector.ts +55 -16
  302. package/src/detection/win32-ide-version.ts +106 -0
  303. package/src/git/change-impact-config.ts +354 -0
  304. package/src/git/git-commands.ts +112 -16
  305. package/src/git/git-diff.ts +81 -11
  306. package/src/git/git-executor.ts +12 -0
  307. package/src/git/git-status.ts +822 -48
  308. package/src/git/git-types.ts +14 -62
  309. package/src/git/git-worktree.ts +261 -4
  310. package/src/git/index.ts +17 -0
  311. package/src/index.ts +191 -13
  312. package/src/installer.d.ts +1 -1
  313. package/src/installer.ts +8 -6
  314. package/src/ipc/local-ipc-server.ts +278 -0
  315. package/src/launch.d.ts +1 -1
  316. package/src/launch.ts +37 -28
  317. package/src/logging/async-batch-writer.ts +55 -0
  318. package/src/logging/command-log.ts +7 -5
  319. package/src/logging/debug-config.ts +25 -0
  320. package/src/logging/debug-trace.ts +7 -2
  321. package/src/logging/log-redactor.ts +100 -0
  322. package/src/logging/log-tail-reader.ts +341 -0
  323. package/src/logging/logger.ts +14 -7
  324. package/src/mesh/contracts.ts +510 -0
  325. package/src/mesh/coordinator-prompt.ts +626 -34
  326. package/src/mesh/coordinator-registry.ts +121 -0
  327. package/src/mesh/mesh-active-work.ts +706 -0
  328. package/src/mesh/mesh-clone-grace.ts +68 -0
  329. package/src/mesh/mesh-completion-synthesis.ts +416 -0
  330. package/src/mesh/mesh-coordinator-config.ts +97 -0
  331. package/src/mesh/mesh-delivery-policy.ts +284 -0
  332. package/src/mesh/mesh-event-classify.ts +78 -0
  333. package/src/mesh/mesh-event-forwarding.ts +2042 -0
  334. package/src/mesh/mesh-event-trace.ts +67 -0
  335. package/src/mesh/mesh-events-coordinator.ts +32 -0
  336. package/src/mesh/mesh-events-pending.ts +1548 -0
  337. package/src/mesh/mesh-events-stale.ts +452 -0
  338. package/src/mesh/mesh-events-utils.ts +443 -0
  339. package/src/mesh/mesh-events.ts +43 -1035
  340. package/src/mesh/mesh-fast-forward.ts +856 -0
  341. package/src/mesh/mesh-host-ownership.ts +41 -3
  342. package/src/mesh/mesh-idle-reminder.ts +153 -0
  343. package/src/mesh/mesh-init.ts +350 -0
  344. package/src/mesh/mesh-ledger-reconciliation.ts +12 -7
  345. package/src/mesh/mesh-ledger.ts +897 -102
  346. package/src/mesh/mesh-magi-status.ts +223 -0
  347. package/src/mesh/mesh-missions.ts +695 -0
  348. package/src/mesh/mesh-node-identity.ts +2065 -0
  349. package/src/mesh/mesh-queue-assignment.ts +2498 -0
  350. package/src/mesh/mesh-reconcile-acked-hold.ts +230 -0
  351. package/src/mesh/mesh-reconcile-config.ts +66 -0
  352. package/src/mesh/mesh-reconcile-identity.ts +103 -0
  353. package/src/mesh/mesh-reconcile-loop.ts +1709 -0
  354. package/src/mesh/mesh-reconcile-v2-backstop.ts +62 -0
  355. package/src/mesh/mesh-refine-batch.ts +205 -0
  356. package/src/mesh/mesh-refine-gates.ts +1792 -0
  357. package/src/mesh/mesh-refine-status.ts +231 -0
  358. package/src/mesh/mesh-remote-event-pull.ts +286 -0
  359. package/src/mesh/mesh-review-inbox.ts +308 -0
  360. package/src/mesh/mesh-routing.ts +291 -0
  361. package/src/mesh/mesh-runtime-store.ts +2318 -0
  362. package/src/mesh/mesh-scheduling-runtime.ts +198 -0
  363. package/src/mesh/mesh-task-inflight.ts +70 -0
  364. package/src/mesh/mesh-task-stats.ts +161 -0
  365. package/src/mesh/mesh-unresolved-forward-outbox.ts +215 -0
  366. package/src/mesh/mesh-warmup-deadline.ts +152 -0
  367. package/src/mesh/mesh-work-queue.ts +1489 -145
  368. package/src/mesh/p2p-relay-failure.ts +2 -2
  369. package/src/mesh/preview-freshness.ts +163 -0
  370. package/src/mesh/refine-config.ts +129 -12
  371. package/src/mesh/worktree-bootstrap-config.ts +599 -0
  372. package/src/providers/acp-provider-instance.ts +43 -10
  373. package/src/providers/approval-utils.d.ts +5 -0
  374. package/src/providers/approval-utils.ts +100 -6
  375. package/src/providers/chat-message-normalization.ts +126 -4
  376. package/src/providers/cli-provider-effect-format.ts +53 -0
  377. package/src/providers/cli-provider-history-dedup.ts +75 -0
  378. package/src/providers/cli-provider-input-prompt.ts +133 -0
  379. package/src/providers/cli-provider-instance-types.ts +150 -0
  380. package/src/providers/cli-provider-instance.ts +2914 -498
  381. package/src/providers/cli-provider-status-helpers.ts +123 -0
  382. package/src/providers/cli-provider-transcript-merge.ts +114 -0
  383. package/src/providers/contracts.d.ts +55 -0
  384. package/src/providers/contracts.ts +207 -6
  385. package/src/providers/extension-provider-instance.ts +12 -7
  386. package/src/providers/external-sources.ts +218 -0
  387. package/src/providers/ide-provider-instance.ts +35 -12
  388. package/src/providers/manual-attendance.ts +105 -0
  389. package/src/providers/native-history/antigravity-claim-registry.ts +138 -0
  390. package/src/providers/native-history/antigravity-cli-transcript.ts +1275 -0
  391. package/src/providers/native-history/claude-cli-transcript.ts +396 -0
  392. package/src/providers/native-history/codex-cli-transcript.ts +479 -0
  393. package/src/providers/native-history/constants.ts +19 -0
  394. package/src/providers/native-history/dispatcher.ts +617 -0
  395. package/src/providers/native-history/hermes-cli-transcript.ts +312 -0
  396. package/src/providers/native-history/index.ts +30 -0
  397. package/src/providers/provider-instance-manager.ts +71 -0
  398. package/src/providers/provider-instance.ts +24 -1
  399. package/src/providers/provider-loader.ts +700 -55
  400. package/src/providers/provider-schema.ts +93 -14
  401. package/src/providers/provider-trust.ts +114 -0
  402. package/src/providers/read-chat-contract.ts +76 -16
  403. package/src/providers/sdk/README.md +49 -0
  404. package/src/providers/sdk/v1/builders/acp/detect-status.ts +144 -0
  405. package/src/providers/sdk/v1/builders/cli/detect-status.ts +313 -0
  406. package/src/providers/sdk/v1/builders/cli/parse-approval-squash.ts +158 -0
  407. package/src/providers/sdk/v1/builders/cli/parse-approval.ts +261 -0
  408. package/src/providers/sdk/v1/builders/cli/parse-session.ts +276 -0
  409. package/src/providers/sdk/v1/builders/cli/visible-region.ts +143 -0
  410. package/src/providers/sdk/v1/fixture-tooling/format.ts +130 -0
  411. package/src/providers/sdk/v1/fixture-tooling/index.ts +22 -0
  412. package/src/providers/sdk/v1/fixture-tooling/replay.ts +352 -0
  413. package/src/providers/sdk/v1/index.ts +152 -0
  414. package/src/providers/sdk/v1/sandbox/README-design.ts +195 -0
  415. package/src/providers/sdk/v1/sandbox/require-whitelist.ts +472 -0
  416. package/src/providers/sdk/v1/sandbox/script-runner.ts +150 -0
  417. package/src/providers/sdk/v1/schemas/cli/provider.schema.json +538 -0
  418. package/src/providers/sdk/v1/schemas/primitives/acp-session-protocol-v1.json +131 -0
  419. package/src/providers/sdk/v1/schemas/primitives/native-history-codex-rollout-v1.json +66 -0
  420. package/src/providers/sdk/v1/schemas/primitives/tui-approval-squash-v1.json +91 -0
  421. package/src/providers/sdk/v1/schemas/primitives/tui-assistant-block-v1.json +91 -0
  422. package/src/providers/sdk/v1/schemas/primitives/tui-cue-ordering-v1.json +47 -0
  423. package/src/providers/sdk/v1/schemas/primitives/tui-dispatch-order-v1.json +32 -0
  424. package/src/providers/sdk/v1/schemas/primitives/tui-footer-chrome-v1.json +42 -0
  425. package/src/providers/sdk/v1/schemas/primitives/tui-index-finder-v1.json +27 -0
  426. package/src/providers/sdk/v1/schemas/primitives/tui-modal-v1.json +125 -0
  427. package/src/providers/sdk/v1/schemas/primitives/tui-prompt-marker-v1.json +45 -0
  428. package/src/providers/sdk/v1/schemas/primitives/tui-session-id-extraction-v1.json +46 -0
  429. package/src/providers/sdk/v1/schemas/primitives/tui-settled-prompt-v1.json +71 -0
  430. package/src/providers/sdk/v1/schemas/primitives/tui-spinner-v1.json +83 -0
  431. package/src/providers/sdk/v1/schemas/primitives/tui-transcript-pty-v1.json +83 -0
  432. package/src/providers/sdk/v1/schemas/primitives/tui-visible-region-v1.json +57 -0
  433. package/src/providers/sdk/v1/schemas/primitives/tui-welcome-screen-v1.json +35 -0
  434. package/src/providers/sdk/v1/types/cli/index.ts +365 -0
  435. package/src/providers/sdk/v1/types/common/index.ts +229 -0
  436. package/src/providers/sdk/v1/validators/index.ts +19 -0
  437. package/src/providers/sdk/v1/validators/manifest.ts +110 -0
  438. package/src/providers/sdk/v1/validators/taint.ts +309 -0
  439. package/src/providers/spec/adapter.ts +246 -0
  440. package/src/providers/spec/background-task-detector.ts +178 -0
  441. package/src/providers/spec/cli-adapter.ts +1090 -0
  442. package/src/providers/spec/evaluator.ts +407 -0
  443. package/src/providers/spec/fsm-driver.ts +1528 -0
  444. package/src/providers/spec/fsm-evaluator.ts +290 -0
  445. package/src/providers/spec/fsm-loader.ts +128 -0
  446. package/src/providers/spec/fsm-types.ts +301 -0
  447. package/src/providers/spec/native-history-executor.ts +1256 -0
  448. package/src/providers/spec/pre-launch-trust.ts +104 -0
  449. package/src/providers/spec/route.ts +51 -0
  450. package/src/providers/spec/types.ts +284 -0
  451. package/src/providers/status-monitor.d.ts +7 -7
  452. package/src/providers/status-monitor.ts +37 -22
  453. package/src/providers/transcript-v2.ts +567 -0
  454. package/src/providers/types/interactive-prompt.ts +536 -0
  455. package/src/providers/version-archive.ts +64 -24
  456. package/src/providers/working-dir.ts +23 -0
  457. package/src/repo-mesh-types.ts +877 -16
  458. package/src/runtime-defaults.ts +39 -0
  459. package/src/session-host/managed-host.ts +218 -0
  460. package/src/session-host/runtime-surface.ts +20 -80
  461. package/src/sessions/registry.ts +44 -0
  462. package/src/shared-types-extra.ts +2 -4
  463. package/src/shared-types.d.ts +8 -0
  464. package/src/shared-types.ts +148 -55
  465. package/src/status/builders.ts +69 -6
  466. package/src/status/chat-tail-hot-sessions.ts +117 -2
  467. package/src/status/normalize.ts +2 -0
  468. package/src/status/reporter.ts +19 -1
  469. package/src/status/snapshot.ts +100 -27
  470. package/src/system/hash.ts +23 -0
  471. package/src/system/host-memory.ts +29 -12
  472. package/src/system/load-better-sqlite3.ts +68 -0
  473. package/src/types.ts +26 -6
  474. package/dist/cli-adapters/terminal-backends/xterm-backend.d.ts +0 -17
  475. package/dist/mesh/mesh-sync.d.ts +0 -53
  476. package/src/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +0 -16
  477. package/src/cli-adapters/terminal-backends/xterm-backend.ts +0 -97
  478. package/src/mesh/mesh-sync.ts +0 -111
@@ -0,0 +1,1709 @@
1
+ // ---------------------------------------------------------------------------
2
+ // mesh-reconcile-loop — periodic queue → live coordinator reconciliation
3
+ // ---------------------------------------------------------------------------
4
+ // Single-model replacement for the old event-based "spontaneous forward" paths
5
+ // (remote P2P mesh_forward_event dispatch + live-CLI PTY fire-and-forget inject).
6
+ // Those pushed events at the moment a worker transitioned state, and silently
7
+ // dropped on the network (P2P) or when the coordinator was generating.
8
+ //
9
+ // The reliable backbone has always been the pending-events queue (SQLite +
10
+ // JSONL): every mesh coordinator event is persisted there before anything else
11
+ // (see injectMeshSystemMessage). What was missing was an *active* drainer that
12
+ // runs on a schedule rather than only when the coordinator (an LLM) happens to
13
+ // call a mesh tool.
14
+ //
15
+ // This loop is that drainer. On a fixed interval it:
16
+ // 1. Finds live CLI coordinator sessions on THIS daemon (meshCoordinatorFor
17
+ // stamp). For each mesh, drains the local queue scoped to this daemon and
18
+ // injects pending events into the coordinator. When a coordinator is idle it
19
+ // receives every queued event. When ONLY generating coordinators exist (the
20
+ // common case while the coordinator is blocked awaiting a worker result), the
21
+ // loop force-drains ONLY the force-inject events (completion / approval / stop /
22
+ // refine·bootstrap terminal) and force-writes them into the generating PTY —
23
+ // the same busy-bypass send-guard escape the live-CLI inject used to use.
24
+ // Non-force progress events stay queued for the next idle tick (injecting them
25
+ // mid-generation would be noise). This is what makes a coordinator parked in
26
+ // `generating` while awaiting a worker's completion actually receive it.
27
+ // 2. In cloud mode (dispatchMeshCommand present), pulls each remote worker
28
+ // node daemon's queue over P2P (get_pending_mesh_events) and re-injects via
29
+ // handleMeshForwardEvent — the same pull the MCP drainCoordinatorPendingEvents
30
+ // already does, now driven by the daemon timer instead of an LLM tool call.
31
+ //
32
+ // IMPORTANT — limits of this loop:
33
+ // - It only delivers to *live CLI coordinator instances* on this daemon. A
34
+ // pure stdio MCP coordinator (an LLM with no live CLI session to inject
35
+ // into) has no inject target here; that case stays pull-driven — the LLM
36
+ // drains the queue when it calls mesh_status / mesh_read_chat. We do NOT try
37
+ // to "wake" an LLM from the daemon; that is structurally impossible over a
38
+ // stdio request/response transport. See docs/refactoring/2026-06-15-mesh-event-to-queue-polling.md §4.7.
39
+ // - Queue persistence (queuePendingMeshCoordinatorEvent) and the SQLite
40
+ // drained=1 idempotency are the trust backbone and are untouched by this loop.
41
+ // ---------------------------------------------------------------------------
42
+
43
+ import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
44
+ import { loadConfig } from '../config/config.js';
45
+ import { listMeshes, getMesh } from '../config/mesh-config.js';
46
+ import { maybeInjectIdleActiveMissionReminder } from './mesh-idle-reminder.js';
47
+ import { LOG, getLogLevel } from '../logging/logger.js';
48
+ import { drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, buildPendingEventFingerprint, queuePendingMeshCoordinatorEvent } from './mesh-events-pending.js';
49
+ import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
50
+ import { appendLedgerEntry } from './mesh-ledger.js';
51
+ import { MeshRuntimeStore } from './mesh-runtime-store.js';
52
+ import { handleMeshForwardEvent, shouldForceInjectMeshEvent, triggerMeshQueue, resolveForwardEventMeshId } from './mesh-events-coordinator.js';
53
+ import { isMeshApprovalEvent, MESH_APPROVAL_EVENTS } from './mesh-event-classify.js';
54
+ import {
55
+ peekUnresolvedDelegateForwards,
56
+ ackUnresolvedDelegateForward,
57
+ expireStaleUnresolvedDelegateForwards,
58
+ registerUnresolvedForwardRetryNudge,
59
+ } from './mesh-unresolved-forward-outbox.js';
60
+ import { readNonEmptyString, readMeshCompletionSummary, buildMeshSystemMessage } from './mesh-events-utils.js';
61
+ import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
62
+ import { expandDaemonIdForms, daemonIdsEquivalent, sessionIdsEquivalent, meshNodeIdMatches } from '@adhdev/mesh-shared';
63
+ import { getQueue, reclaimStrandedAssignedTask, updateTaskStatus } from './mesh-work-queue.js';
64
+ import { resolveSessionBusyVerdict } from './mesh-queue-assignment.js';
65
+ import { readLedgerEntries } from './mesh-ledger.js';
66
+ import type { MeshLedgerEntry } from './mesh-ledger.js';
67
+ import { findTerminalLedgerEvidenceForTask } from './mesh-events-stale.js';
68
+ import {
69
+ resolveCoordinatorDaemonIds,
70
+ daemonHostsMesh,
71
+ resolveCoordinatorSelfIds,
72
+ } from './mesh-reconcile-identity.js';
73
+ import {
74
+ resolveAutoPruneMinAgeMs,
75
+ resolvePendingHeldDrainEscalateMs,
76
+ resolveReconcileIntervalMs,
77
+ } from './mesh-reconcile-config.js';
78
+ import { pullRemoteNodeQueues } from './mesh-remote-event-pull.js';
79
+ import {
80
+ reconcileUnterminatedDirectDispatches,
81
+ autoPruneStaleDirectDispatches,
82
+ } from './mesh-completion-synthesis.js';
83
+
84
+ // Re-export the extracted public API so existing importers (mesh-events.ts barrel;
85
+ // the reconcile-loop test suite) keep their `from './mesh-reconcile-loop.js'` paths.
86
+ export { getMeshV2BackstopCounters, __resetMeshV2BackstopCountersForTests } from './mesh-reconcile-v2-backstop.js';
87
+ export { __resetReconcileInFlightSynthDebounceForTests } from './mesh-reconcile-acked-hold.js';
88
+
89
+ // Reconcile-loop timing tunables + their env-override resolvers
90
+ // (DEFAULT_RECONCILE_INTERVAL_MS, DEFAULT_AUTO_PRUNE_MIN_AGE_MS,
91
+ // DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS, resolveReconcileIntervalMs,
92
+ // resolveAutoPruneMinAgeMs, resolvePendingHeldDrainEscalateMs) live in
93
+ // ./mesh-reconcile-config.ts (A-3 extraction) and are imported above.
94
+
95
+ interface LiveCoordinator {
96
+ meshId: string;
97
+ instance: ReturnType<DaemonComponents['instanceManager']['getInstance']>;
98
+ // Runtime session id of this coordinator instance (getState().instanceId). PHASE 2
99
+ // strict-matches an event's targetCoordinatorSessionId against this so a completion
100
+ // routes back to the exact originating coordinator session, not a sibling on the same
101
+ // daemon (the multi-coordinator misroute).
102
+ sessionId: string;
103
+ // PTY-OVERTRUST-DRAIN (Defect B): drain-eligibility, decided on the RAW adapter
104
+ // turn-state (mask-stripped) — NOT on getState().status, which overlays the
105
+ // auto-approve "hold-idle" visual mask that paints a genuinely-idle coordinator
106
+ // `generating` and so used to strand its worker's completion. True only when the
107
+ // raw adapter is at a real turn end AND the session is not modal-parked. When the
108
+ // instance does not expose getDrainStatus() (non-CLI / older), this falls back to
109
+ // the masked `status === 'idle'` (the pre-fix behaviour) so nothing regresses.
110
+ idle: boolean;
111
+ // True when the coordinator session is parked on a harness modal awaiting a
112
+ // human answer — claude-cli AskUserQuestion (waiting_choice) or a tool-consent
113
+ // prompt (waiting_approval). A force-inject into such a session would write raw
114
+ // keystrokes the modal key handler consumes, silently selecting a choice the
115
+ // user never made (data corruption). PHASE 2 excludes these from force-inject
116
+ // and leaves the event queued for a later (modal-resolved) tick.
117
+ modalParked: boolean;
118
+ }
119
+
120
+ // Observability: last-seen modal-park state per coordinator session, so we LOG.info
121
+ // only on a TRANSITION (clear → parked, parked → cleared) instead of every 4s tick.
122
+ // Per-process; a restart re-logs the first observation, which is desirable — it
123
+ // re-confirms a coordinator that is still parked after the restart (the exact
124
+ // "restart does not clear it" symptom the operator needs visibility into).
125
+ const coordinatorModalParkState = new Map<string, boolean>();
126
+
127
+ // Find live CLI coordinator instances on THIS daemon, keyed by mesh.
128
+ function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
129
+ const out: LiveCoordinator[] = [];
130
+ for (const inst of components.instanceManager.getByCategory('cli')) {
131
+ const state = inst.getState();
132
+ const settings = state.settings && typeof state.settings === 'object'
133
+ ? state.settings as Record<string, unknown>
134
+ : {};
135
+ const meshId = readNonEmptyString(settings.meshCoordinatorFor);
136
+ if (!meshId) continue;
137
+ const status = readNonEmptyString(state.status).toLowerCase();
138
+ // getState() overlays the modal-park statuses: an active AskUserQuestion
139
+ // prompt surfaces as waiting_choice, a tool-consent prompt as waiting_approval.
140
+ // NOTIF-HELD-DRAIN (Fix 1): consult the instance's own isModalParked() rather than the
141
+ // raw status literal so the corrected classification flows here — a busy mesh
142
+ // coordinator's routine, in-flight tool-consent (auto-approve off) is NOT a human-await
143
+ // modal and must NOT wedge the mesh's pending completion events under `modal_parked`.
144
+ // resolveModalParkStatus() (which isModalParked wraps) already encodes that distinction
145
+ // and the waiting_choice/stalled-auto-approve genuine-modal cases. Fall back to the
146
+ // status literal for any instance that does not expose the method. Lowercase compare —
147
+ // the SessionStatus enum is forked across modules and waiting_choice is absent from some.
148
+ const modalParked = typeof (inst as any).isModalParked === 'function'
149
+ ? (inst as any).isModalParked() === true
150
+ : (status === 'waiting_choice' || status === 'waiting_approval');
151
+ // PTY-OVERTRUST-DRAIN (Defect B, fix A): drain-eligible idle is decided on the
152
+ // RAW adapter turn-state, not getState().status. getState() overlays the
153
+ // auto-approve hold-idle mask that paints a genuinely-idle coordinator
154
+ // `generating` (a UI-flicker suppressant), and the reconcile loop used to trust
155
+ // that mask and HOLD the worker's completion (generating_no_idle_coordinator)
156
+ // even though the PTY was at a real turn end. getDrainStatus() strips the mask
157
+ // (raw adapter idle, modal-park preserved). Fall back to the masked literal for
158
+ // any instance that does not expose it (non-CLI / older) — regression-0.
159
+ const drainStatus: string | null = typeof (inst as any).getDrainStatus === 'function'
160
+ ? (inst as any).getDrainStatus()
161
+ : null;
162
+ const idle = drainStatus !== null ? drainStatus === 'idle' : (status === 'idle');
163
+ const sessionId = readNonEmptyString(state.instanceId);
164
+ // ── NOTIF (B) desync diagnostic (read-only, no behavior change) ───────────
165
+ // The confirmed (B) defect: a coordinator whose FSM is idle (status above ===
166
+ // 'idle' for minutes) is nonetheless classified busy here, so the generating/
167
+ // modal-park hold never drains and a worker completion is stranded until the
168
+ // user's next turn edge. Static analysis found no code path where getState()
169
+ // returns generating while lastStatus and the adapter raw are both idle — so the
170
+ // divergence is a runtime desync between the three status sources. Capture all
171
+ // three (plus the auto-approve mask state that getState() overlays at :803) for
172
+ // EVERY mesh-coordinator candidate on this tick, so the source that diverges from
173
+ // the others can be read directly against the same-tick "skip → generating"/
174
+ // "skip → modal-parked" hold logs below (pair by sessionId + timestamp).
175
+ //
176
+ // CRITICAL: reuse the `state` already fetched above (line ~301) — do NOT call
177
+ // getState() again. getState() runs maybeAutoApproveStatus() as a side effect,
178
+ // which would mutate the very auto-approve mask we are trying to observe. The
179
+ // adapter raw read uses allowParse:false, which only reads engine.activeModal and
180
+ // is side-effect-free.
181
+ if (getLogLevel() === 'debug') {
182
+ let adapterRaw = '?';
183
+ try {
184
+ const a = (inst as any).adapter;
185
+ if (a && typeof a.getStatus === 'function') {
186
+ adapterRaw = readNonEmptyString(a.getStatus({ allowParse: false })?.status) || '?';
187
+ }
188
+ } catch (e: any) {
189
+ adapterRaw = `err:${e?.message || e}`;
190
+ }
191
+ const lastStatus = readNonEmptyString((inst as any).lastStatus) || '?';
192
+ const autoApproveBusy = (inst as any).autoApproveBusy;
193
+ const maskSince = (inst as any).autoApproveMaskSince;
194
+ // PTY-OVERTRUST-DRAIN: include the mask-stripped drainStatus next to the three
195
+ // legacy sources so the divergence (getState=generating while adapterRaw=idle =
196
+ // the mask) is directly readable, and confirm drain now follows adapterRaw.
197
+ LOG.debug('MeshReconcile', `coordDiag sess=${sessionId || '?'} mesh=${meshId} getState=${status || '?'} drainStatus=${drainStatus || 'n/a'} lastStatus=${lastStatus} adapterRaw=${adapterRaw} autoApproveBusy=${autoApproveBusy === true} maskSince=${maskSince || 0}`);
198
+ }
199
+ // Modal-park transition observability: a coordinator entering modal-park is what
200
+ // begins holding completion events under `modal_parked`; one leaving it is what
201
+ // drains them. Both transitions were previously SILENT (the operator had no log
202
+ // to diagnose a stuck/held completion), so emit a single line per edge.
203
+ const stateKey = `${meshId}::${sessionId || '?'}`;
204
+ const prevParked = coordinatorModalParkState.get(stateKey);
205
+ if (prevParked !== modalParked) {
206
+ coordinatorModalParkState.set(stateKey, modalParked);
207
+ if (modalParked) {
208
+ LOG.info('MeshReconcile', `Coordinator ${sessionId || '?'} (mesh ${meshId}) entered modal-park (status=${status}) — terminal events for it will be held until the modal is answered`);
209
+ } else if (prevParked === true) {
210
+ LOG.info('MeshReconcile', `Coordinator ${sessionId || '?'} (mesh ${meshId}) left modal-park (status=${status}) — held events will drain on this/next tick`);
211
+ }
212
+ }
213
+ out.push({ meshId, instance: inst, sessionId, idle, modalParked });
214
+ }
215
+ return out;
216
+ }
217
+
218
+ /**
219
+ * DRAIN-WITHOUT-INJECT guard. Classify, for a mesh on THIS daemon, whether a
220
+ * queue-drain caller (the MCP `get_pending_mesh_events` poll) may safely consume
221
+ * pending coordinator events — i.e. whether there is a surface that will actually
222
+ * deliver them.
223
+ *
224
+ * Root cause being guarded: `get_pending_mesh_events` marks rows drained=1
225
+ * atomically and unconditionally. When the live CLI coordinator for the mesh is
226
+ * GENERATING (or modal-parked), the reconcile loop correctly HOLDS its terminal
227
+ * events (drained=0) for the coordinator's next idle tick — but a concurrent MCP
228
+ * poll draining the SAME queue consumes those held rows into a tool result that
229
+ * the busy coordinator never surfaces as a turn, so the completion is lost
230
+ * forever (drained=1, never re-queued). The reconcile loop is the authoritative
231
+ * delivery path for a live CLI coordinator; the MCP poll must defer to it.
232
+ *
233
+ * Returns:
234
+ * - hasLiveCliCoordinator: a CLI session with meshCoordinatorFor === meshId
235
+ * exists on this daemon (the reconcile loop owns its delivery).
236
+ * - deliverableNow: there is an IDLE live CLI coordinator (reconcile would
237
+ * full-drain into it) — draining now is safe and equivalent.
238
+ * - holdForReconcile: a live CLI coordinator exists but is non-idle
239
+ * (generating / modal-parked). The MCP poll MUST NOT drain; the reconcile
240
+ * loop holds the events undrained and injects them on the next idle tick.
241
+ *
242
+ * A mesh with NO live CLI coordinator on this daemon is a pure stdio MCP / LLM
243
+ * coordinator: the MCP tool result IS the only surface, so the poll legitimately
244
+ * drains (holdForReconcile=false). No regression to that path.
245
+ */
246
+ export function resolveCoordinatorDrainDeliverability(
247
+ components: Pick<DaemonComponents, 'instanceManager'>,
248
+ meshId: string,
249
+ ): { hasLiveCliCoordinator: boolean; deliverableNow: boolean; holdForReconcile: boolean } {
250
+ const coordinators = findLiveCoordinators(components as DaemonComponents).filter(c => c.meshId === meshId);
251
+ if (coordinators.length === 0) {
252
+ return { hasLiveCliCoordinator: false, deliverableNow: false, holdForReconcile: false };
253
+ }
254
+ const hasIdle = coordinators.some(c => c.idle);
255
+ return {
256
+ hasLiveCliCoordinator: true,
257
+ deliverableNow: hasIdle,
258
+ // A live CLI coordinator exists but none is idle → the reconcile loop is
259
+ // holding the events; the poll must not steal them.
260
+ holdForReconcile: !hasIdle,
261
+ };
262
+ }
263
+
264
+ /**
265
+ * DRAIN-WITHOUT-INJECT guard for the `get_pending_mesh_events` daemon handler.
266
+ *
267
+ * Decides whether an incoming pending-events DRAIN must be held (return nothing,
268
+ * leave rows drained=0) because the only surface for those events is a LOCAL live
269
+ * CLI coordinator that is currently busy (generating / modal-parked) — in which
270
+ * case the reconcile loop owns delivery on the coordinator's next idle tick, and
271
+ * the poll draining them now would lose them.
272
+ *
273
+ * The hold applies ONLY when BOTH:
274
+ * 1) a live CLI coordinator for this mesh on THIS daemon is non-idle, AND
275
+ * 2) the drain is targeted at THIS daemon (the requested coordinatorDaemonId is
276
+ * empty/broadcast, or matches one of this daemon's id forms).
277
+ *
278
+ * A REMOTE coordinator pulling our worker's events passes its own (remote)
279
+ * coordinatorDaemonId — condition (2) is false — so the drain proceeds and the
280
+ * remote pull is never blocked by our local coordinator's busy state. A pure
281
+ * stdio MCP coordinator (no live CLI session) never satisfies (1), so its tool
282
+ * result remains the surface and the drain proceeds. No regression to either.
283
+ *
284
+ * SELF-COORDINATOR INBOX LEVEL-DRAIN (Defect 2): the hold above assumes the ONLY
285
+ * surface for a busy local coordinator's events is a future PTY inject on its idle
286
+ * edge, so it defers to the reconcile loop. But when the drain caller IS the local
287
+ * coordinator reading its OWN inbox (the `get_pending_mesh_events` call whose events
288
+ * are returned in the caller's tool RESULT — a data queue the self-coordinating LLM
289
+ * consumes directly), the events ARE surfaced losslessly the moment the tool returns,
290
+ * with NO PTY write. A busy self-coordinating LLM that calls a mesh tool mid-turn would
291
+ * otherwise get an empty inbox (held) and only see the completion on its NEXT busy→idle
292
+ * edge — the measured ~59s strand. `callerIsSelfCoordinatorInboxRead` marks that safe
293
+ * caller: the hold is relaxed for it (return the events), while every OTHER drain (a
294
+ * backfill relay, a broadcast poll, a DIFFERENT coordinator that genuinely needs its PTY)
295
+ * still defers to the reconcile loop. This relaxes delivery INTO the coordinator's own
296
+ * inbox only — it never changes how events are injected into a live PTY prompt.
297
+ */
298
+ export function shouldHoldPendingDrainForBusyLocalCoordinator(
299
+ components: Pick<DaemonComponents, 'instanceManager'> & { statusInstanceId?: string },
300
+ meshId: string,
301
+ requestedCoordinatorDaemonId?: string | null,
302
+ callerIsSelfCoordinatorInboxRead?: boolean,
303
+ ): boolean {
304
+ if (!meshId) return false;
305
+ const deliverability = resolveCoordinatorDrainDeliverability(components, meshId);
306
+ if (!deliverability.holdForReconcile) return false;
307
+ // The local CLI coordinator is busy. Hold only when the drain is for THIS daemon.
308
+ const requested = readNonEmptyString(requestedCoordinatorDaemonId);
309
+ if (!requested) return true; // broadcast drain → would consume the held local events
310
+ const localIds = expandDaemonIdForms([
311
+ readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId),
312
+ readNonEmptyString(loadConfig().machineId),
313
+ ]);
314
+ const targetsLocalCoordinator = localIds.some(id => daemonIdsEquivalent(id, requested));
315
+ if (!targetsLocalCoordinator) return false;
316
+ // SELF-COORDINATOR INBOX LEVEL-DRAIN: the busy local coordinator is itself the caller,
317
+ // reading its own inbox — the drained events return in ITS tool result (lossless data-queue
318
+ // surface, no PTY inject). Do NOT hold; let the self-coordinator see its completions now.
319
+ if (callerIsSelfCoordinatorInboxRead) return false;
320
+ return true;
321
+ }
322
+
323
+ // Inject a drained pending event into a live coordinator session. Force-inject
324
+ // events carry force:true so they bypass the busy send-guard and land in the PTY
325
+ // even while the coordinator is generating (see shouldForceInjectMeshEvent).
326
+ function injectPendingIntoCoordinator(
327
+ coordinator: LiveCoordinator['instance'],
328
+ pending: PendingMeshCoordinatorEvent,
329
+ opts?: { forceOverride?: boolean },
330
+ ): void {
331
+ if (!coordinator) return;
332
+ // NOTIF-DROP-SYNTH-NO-MESSAGE (defence-in-depth): a queued event with no coordinatorMessage
333
+ // used to be dropped here (drain-without-inject) — the row had already been consumed
334
+ // (drained=1) by the caller's drain, so silently returning lost it forever. The primary fix
335
+ // makes the transcript-reconcile synth always carry a coordinatorMessage, but as a backstop,
336
+ // lazily synthesize the [System] text for any force-inject (terminal: completion / approval /
337
+ // stop / refine·bootstrap) event that still arrives message-less, so it surfaces instead of
338
+ // vanishing. A NON-force lifecycle event (agent:ready / generating_started) legitimately
339
+ // carries no message and must NOT be injected (it is queued only to re-drive the claim state
340
+ // machine on pull) — for it we still return without injecting.
341
+ let coordinatorMessage = pending.coordinatorMessage;
342
+ if (!coordinatorMessage) {
343
+ if (!shouldForceInjectMeshEvent(pending.event)) return;
344
+ const metadataEvent = pending.metadataEvent && typeof pending.metadataEvent === 'object'
345
+ ? pending.metadataEvent
346
+ : {};
347
+ coordinatorMessage = buildMeshSystemMessage({
348
+ event: pending.event,
349
+ nodeLabel: pending.nodeLabel,
350
+ metadataEvent,
351
+ });
352
+ if (!coordinatorMessage) return; // builder produced nothing — nothing to surface
353
+ LOG.warn('MeshReconcile', `Lazily synthesized missing coordinatorMessage for ${pending.event} (mesh ${pending.meshId}) at inject time — a queued terminal event arrived message-less`);
354
+ }
355
+ // forceOverride lets the APPROVAL-Q1-REALTIME nudge path deliver into a busy
356
+ // coordinator WITHOUT a raw PTY force-write (force-inject-into-generating stays
357
+ // intentionally removed): a non-force send_message enters the adapter's
358
+ // pendingOutboundQueue and is surfaced at the coordinator's next turn boundary.
359
+ const force = opts?.forceOverride ?? shouldForceInjectMeshEvent(pending.event);
360
+ // EVTTRACE: event surfaced to the coordinator (injected into its live CLI session).
361
+ // This is the terminal happy-path stage. Observation only.
362
+ traceMeshEventStage('surfaced', {
363
+ taskId: pending.metadataEvent?.taskId,
364
+ sessionId: pending.metadataEvent?.targetSessionId ?? pending.targetCoordinatorSessionId,
365
+ nodeId: pending.nodeId,
366
+ meshId: pending.meshId,
367
+ event: pending.event,
368
+ }, force ? 'force-inject' : 'inject');
369
+ coordinator.onEvent('send_message', {
370
+ input: { text: coordinatorMessage, textFallback: coordinatorMessage },
371
+ ...(force ? { force: true } : {}),
372
+ });
373
+ }
374
+
375
+ // Held-event ledger dedup: fingerprints of held terminal events already written as an
376
+ // `event_held` ledger audit record in THIS process. Prevents the 4s reconcile tick from
377
+ // re-logging the same held event every interval while a coordinator stays modal-parked.
378
+ // Per-process only (not persisted) — if the daemon restarts while an event is still held
379
+ // it is re-logged once, which is desirable: it re-confirms the event is still undelivered.
380
+ const heldEventLedgerRecorded = new Set<string>();
381
+
382
+ // C1 (data safety): when a terminal completion/approval/bootstrap event cannot be
383
+ // delivered because the only coordinators are modal-parked, the event is held at
384
+ // drained=0 in the pending queue (SQLite + JSONL) for a later tick. That queue is
385
+ // disk-persisted but carries no operator-visible audit trail and can be silently
386
+ // dropped by the pending-file trim (100 KB / 50-entry cap). To guarantee a held
387
+ // completion's worker summary is never silently lost, mirror each held terminal event
388
+ // into the coordinator's mesh ledger as an `event_held` entry — auditable and
389
+ // recoverable (the finalSummary survives even if the pending copy is later trimmed or
390
+ // the coordinator session is force-resolved before re-drain). Idempotent per process
391
+ // via heldEventLedgerRecorded so a long modal park does not spam the ledger.
392
+ function recordHeldTerminalEventsToLedger(
393
+ meshId: string,
394
+ drainDaemonIds: string[],
395
+ reason: string,
396
+ heldForCoordinatorCount: number,
397
+ ): void {
398
+ let pending: readonly PendingMeshCoordinatorEvent[];
399
+ try {
400
+ pending = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
401
+ } catch {
402
+ return; // best-effort audit — never let a peek failure break the tick
403
+ }
404
+ for (const event of pending) {
405
+ // Only audit terminal/force-inject events (completion / approval / stop / refine·
406
+ // bootstrap). Silent lifecycle events (agent:ready / generating_started) carry no
407
+ // worker output to preserve and re-drain harmlessly, so they need no audit trail.
408
+ if (!shouldForceInjectMeshEvent(event.event)) continue;
409
+ const fingerprint = buildPendingEventFingerprint(event);
410
+ const key = `${meshId}::${fingerprint || `${event.event}::${event.nodeId || ''}::${event.queuedAt}`}`;
411
+ if (heldEventLedgerRecorded.has(key)) continue;
412
+ heldEventLedgerRecorded.add(key);
413
+ const finalSummary = readMeshCompletionSummary(event.metadataEvent);
414
+ try {
415
+ appendLedgerEntry(meshId, {
416
+ kind: 'event_held',
417
+ ...(event.nodeId ? { nodeId: event.nodeId } : {}),
418
+ payload: {
419
+ event: event.event,
420
+ reason,
421
+ recoverable: true,
422
+ heldForCoordinators: heldForCoordinatorCount,
423
+ nodeLabel: event.nodeLabel,
424
+ ...(event.workspace ? { workspace: event.workspace } : {}),
425
+ targetCoordinatorDaemonId: event.targetCoordinatorDaemonId ?? null,
426
+ queuedAt: event.queuedAt,
427
+ ...(fingerprint ? { fingerprint } : {}),
428
+ ...(finalSummary ? { finalSummary } : {}),
429
+ },
430
+ });
431
+ LOG.info('MeshReconcile', `Ledger-recorded held ${event.event} for mesh ${meshId} (reason ${reason}) — recoverable from ledger`);
432
+ } catch (e: any) {
433
+ // Failed to persist — drop the dedup marker so the next tick retries.
434
+ heldEventLedgerRecorded.delete(key);
435
+ LOG.warn('MeshReconcile', `Failed to ledger-record held ${event.event} for mesh ${meshId}: ${e?.message || e}`);
436
+ }
437
+ }
438
+ }
439
+
440
+ // PTY-OVERTRUST-DRAIN (Defect B, fix B). Age of the OLDEST queued terminal/force-inject
441
+ // event for a mesh, in ms — the signal the generating-hold age-escape gates on. Returns 0
442
+ // when there is no held terminal event (no escape needed). Best-effort: a peek failure
443
+ // returns 0 (no escape this tick), never throws into the tick.
444
+ function oldestHeldTerminalEventAgeMs(meshId: string, drainDaemonIds: string[]): number {
445
+ let pending: readonly PendingMeshCoordinatorEvent[];
446
+ try {
447
+ pending = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
448
+ } catch {
449
+ return 0;
450
+ }
451
+ const now = Date.now();
452
+ let maxAge = 0;
453
+ for (const event of pending) {
454
+ if (!shouldForceInjectMeshEvent(event.event)) continue; // only terminal events matter
455
+ const queuedAt = typeof event.queuedAt === 'number' ? event.queuedAt : now;
456
+ const age = now - queuedAt;
457
+ if (age > maxAge) maxAge = age;
458
+ }
459
+ return maxAge;
460
+ }
461
+
462
+ // PTY-OVERTRUST-DRAIN (Defect B, fix B). Re-confirm, on the RAW adapter (mask-stripped),
463
+ // which of the held-as-generating coordinators is GENUINELY idle right now. A coordinator
464
+ // whose getDrainStatus() reads 'idle' is a real drain target the time-based escape may
465
+ // deliver into. One that still reads 'generating'/'modal_parked'/'other' stays held — the
466
+ // escape NEVER injects into a genuinely-busy PTY (that is the data-loss force-inject path
467
+ // intentionally removed; re-confirmation is what keeps this safe). Falls back to the
468
+ // coordinator's already-computed `idle` flag when the instance does not expose
469
+ // getDrainStatus() (non-CLI / older) — that flag is itself raw-adapter-derived post-fix-A.
470
+ function reconfirmGenuinelyIdleCoordinators(generating: LiveCoordinator[]): LiveCoordinator[] {
471
+ const out: LiveCoordinator[] = [];
472
+ for (const c of generating) {
473
+ const inst = c.instance as any;
474
+ const drainStatus: string | null = typeof inst?.getDrainStatus === 'function'
475
+ ? inst.getDrainStatus()
476
+ : null;
477
+ const genuinelyIdle = drainStatus !== null ? drainStatus === 'idle' : c.idle;
478
+ if (genuinelyIdle) out.push({ ...c, idle: true });
479
+ }
480
+ return out;
481
+ }
482
+
483
+ // Full-drain the local pending queue for a mesh and inject every event into the given
484
+ // IDLE target coordinators, honouring strict session routing. Shared by the normal idle
485
+ // delivery path and the Defect-B age-escape so both deliver identically (one drain, one
486
+ // inject-per-event, strict-route hold for an unmatched session). Returns the number of
487
+ // events drained (0 when the queue was empty / drain failed). Callers must have already
488
+ // confirmed the targets are genuinely idle.
489
+ function drainAndInjectIntoTargets(
490
+ meshId: string,
491
+ drainDaemonIds: string[],
492
+ localDaemonId: string | undefined,
493
+ targetCoordinators: LiveCoordinator[],
494
+ logLabel: string,
495
+ ): number {
496
+ let pendingEvents: PendingMeshCoordinatorEvent[] = [];
497
+ try {
498
+ pendingEvents = drainPendingMeshCoordinatorEvents(
499
+ meshId,
500
+ drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
501
+ );
502
+ } catch (e: any) {
503
+ LOG.warn('MeshReconcile', `Drain failed for mesh ${meshId}: ${e?.message || e}`);
504
+ return 0;
505
+ }
506
+ if (pendingEvents.length === 0) return 0;
507
+
508
+ LOG.info('MeshReconcile', `Reconcile inject → ${logLabel}: ${pendingEvents.length} pending event(s) → ${targetCoordinators.length} coordinator(s) for mesh ${meshId}`);
509
+ for (const pending of pendingEvents) {
510
+ // Strict session routing (multi-coordinator): when the event names an
511
+ // originating coordinator session, deliver ONLY to the live coordinator whose
512
+ // session id matches — a sibling coordinator on the same daemon must NOT receive
513
+ // another coordinator's completion. When the event carries no session id (legacy /
514
+ // version-skewed / single-coordinator), fall back to the daemon-level set
515
+ // (unchanged behaviour — regression-0 for the common case).
516
+ const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
517
+ if (wantSession) {
518
+ // Session ids are single-form; sessionIdsEquivalent is the one canonical
519
+ // exact-match predicate — unlike the daemon-level set below it needs no
520
+ // form expansion.
521
+ const matched = targetCoordinators.filter(c => sessionIdsEquivalent(c.sessionId, wantSession));
522
+ if (matched.length === 0) {
523
+ // The originating coordinator session is not deliverable on this daemon
524
+ // right now (gone, or modal-parked and excluded from targets). Strict mode
525
+ // does NOT broadcast to siblings — hold the event for a later tick, and
526
+ // ledger-expire it past a TTL so it can never wedge forever.
527
+ holdOrExpireStrictUnmatchedEvent(pending, wantSession, meshId);
528
+ continue;
529
+ }
530
+ for (const c of matched) injectPendingIntoCoordinator(c.instance, pending);
531
+ continue;
532
+ }
533
+ for (const c of targetCoordinators) {
534
+ injectPendingIntoCoordinator(c.instance, pending);
535
+ }
536
+ }
537
+ return pendingEvents.length;
538
+ }
539
+
540
+ // APPROVAL-Q1-REALTIME stale guard. An approval nudge is RESOLVED once a real terminal
541
+ // ledger entry (task_completed / task_failed) for the same node/session landed at or
542
+ // after the nudge was queued — the worker either finished or died, so it is no longer
543
+ // waiting on that approval. Delivering the nudge then would falsely tell the coordinator
544
+ // the worker is still blocked (the exact UX inversion this fix must avoid), so a resolved
545
+ // nudge is dropped rather than delivered. Ledger-based so the check is daemon-local and
546
+ // deterministic (no dependence on a possibly-remote worker instance's live state).
547
+ function isApprovalNudgeResolved(meshId: string, pending: PendingMeshCoordinatorEvent): boolean {
548
+ const metadataEvent = (pending.metadataEvent && typeof pending.metadataEvent === 'object')
549
+ ? pending.metadataEvent as Record<string, unknown>
550
+ : {};
551
+ const nodeId = readNonEmptyString(pending.nodeId) || readNonEmptyString(metadataEvent.meshNodeId);
552
+ const sessionId = readNonEmptyString(metadataEvent.targetSessionId) || readNonEmptyString(metadataEvent.sessionId);
553
+ if (!nodeId && !sessionId) return false; // nothing to correlate a terminal against
554
+ const queuedAt = typeof pending.queuedAt === 'number' && Number.isFinite(pending.queuedAt) ? pending.queuedAt : 0;
555
+ let entries: MeshLedgerEntry[];
556
+ try {
557
+ entries = readLedgerEntries(meshId);
558
+ } catch {
559
+ return false; // best-effort — a read failure never blocks delivery
560
+ }
561
+ return entries.some(e => {
562
+ if (e.kind !== 'task_completed' && e.kind !== 'task_failed') return false;
563
+ if (queuedAt > 0) {
564
+ const t = new Date(e.timestamp).getTime();
565
+ if (Number.isFinite(t) && t < queuedAt) return false; // terminal predates the nudge
566
+ }
567
+ const nodeMatch = !!nodeId && !!e.nodeId && daemonIdsEquivalent(e.nodeId, nodeId);
568
+ const sessionMatch = !!sessionId && !!e.sessionId && sessionIdsEquivalent(e.sessionId, sessionId);
569
+ return nodeMatch || sessionMatch;
570
+ });
571
+ }
572
+
573
+ // APPROVAL-Q1-REALTIME. Deliver queued approval nudges to a mesh's coordinators every
574
+ // reconcile tick, EVEN when the only coordinators are busy (generating / modal-parked)
575
+ // and there is no idle drain target. This is the crux of the fix: a completion rides the
576
+ // idle-edge hold below (its payload lives only in the pending event), but an approval is
577
+ // LEVEL-backed (task_approval_needed ledger → mesh_status awaiting_approval) so it must
578
+ // NOT wait for an idle edge — during orchestration a coordinator can stay `generating`
579
+ // awaiting the very worker that is blocked on the approval, so the idle edge (the flush
580
+ // point) may never come, and the coordinator's mesh_approve arrives only after a human
581
+ // resolves it ('Not in approval state'). We drain ONLY approval events (leaving every
582
+ // other event for the unchanged hold), drop any already-resolved (stale) nudge, and
583
+ // deliver the rest into each coordinator's inbox WITHOUT a raw PTY force-write (non-force
584
+ // send_message → adapter pendingOutboundQueue → surfaced at the coordinator's next turn
585
+ // boundary). Dropping the pending copy after delivery is safe and prevents re-nudging
586
+ // every 4s — the level ledger state remains the durable, re-derivable source of truth.
587
+ // Returns the number of nudges delivered (0 when none were queued/deliverable).
588
+ function drainAndDeliverApprovalNudges(
589
+ meshId: string,
590
+ drainDaemonIds: string[],
591
+ localDaemonId: string | undefined,
592
+ meshCoordinators: LiveCoordinator[],
593
+ ): number {
594
+ // O(1) guard: only touch the queue when an approval event is actually present.
595
+ let peeked: readonly PendingMeshCoordinatorEvent[];
596
+ try {
597
+ peeked = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
598
+ } catch {
599
+ return 0;
600
+ }
601
+ if (!peeked.some(e => isMeshApprovalEvent(e.event))) return 0;
602
+
603
+ let drained: PendingMeshCoordinatorEvent[];
604
+ try {
605
+ drained = drainPendingMeshCoordinatorEvents(
606
+ meshId,
607
+ drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
608
+ { onlyEvents: MESH_APPROVAL_EVENTS },
609
+ );
610
+ } catch (e: any) {
611
+ LOG.warn('MeshReconcile', `Approval-nudge drain failed for mesh ${meshId}: ${e?.message || e}`);
612
+ return 0;
613
+ }
614
+
615
+ let delivered = 0;
616
+ for (const pending of drained) {
617
+ if (isApprovalNudgeResolved(meshId, pending)) {
618
+ // Stale: already resolved. Drop without delivery — re-surfacing it would
619
+ // mislead the coordinator into believing the worker is still awaiting approval.
620
+ traceMeshEventDrop('approval_nudge_stale_resolved', {
621
+ taskId: readNonEmptyString((pending.metadataEvent as Record<string, unknown>)?.taskId),
622
+ sessionId: readNonEmptyString((pending.metadataEvent as Record<string, unknown>)?.targetSessionId) ?? pending.targetCoordinatorSessionId,
623
+ nodeId: pending.nodeId,
624
+ meshId,
625
+ event: pending.event,
626
+ }, 'approval already resolved (terminal ledger entry present)');
627
+ LOG.info('MeshReconcile', `Dropped stale approval nudge for mesh ${meshId} (${pending.nodeLabel}) — approval already resolved`);
628
+ continue;
629
+ }
630
+ // Strict session routing (multi-coordinator): deliver only to the originating
631
+ // coordinator session when the nudge names one; otherwise broadcast to every
632
+ // coordinator for this mesh. Absent a live matching coordinator we drop the nudge —
633
+ // the level state (awaiting_approval) still surfaces via mesh_status, so nothing is lost.
634
+ const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
635
+ const targets = wantSession
636
+ ? meshCoordinators.filter(c => sessionIdsEquivalent(c.sessionId, wantSession))
637
+ : meshCoordinators;
638
+ if (targets.length === 0) continue;
639
+ for (const c of targets) injectPendingIntoCoordinator(c.instance, pending, { forceOverride: false });
640
+ delivered++;
641
+ LOG.info('MeshReconcile', `Delivered approval nudge (level) for mesh ${meshId} (${pending.nodeLabel}) → ${targets.length} coordinator(s) without waiting for an idle edge`);
642
+ }
643
+ return delivered;
644
+ }
645
+
646
+ // One reconcile tick. Two independent phases:
647
+ //
648
+ // PHASE 1 — Remote queue pull (the fix for remote worktree completions never
649
+ // reaching an MCP/LLM coordinator). For EVERY mesh this daemon hosts/
650
+ // coordinates, pull each remote worker node's pending-events queue over P2P
651
+ // into THIS daemon's local queue. This runs *regardless of whether a live CLI
652
+ // coordinator exists* — the coordinator is usually a pure stdio MCP LLM with
653
+ // no live CLI session, and it can only observe a remote worker's completion
654
+ // once that event has been pulled into the local queue (which it then drains
655
+ // on its next mesh tool call). Previously this pull was gated behind a live
656
+ // CLI coordinator and so never ran for MCP/LLM coordinators — remote
657
+ // completions sat on the remote node's queue until the LLM happened to call
658
+ // mesh_read_chat, which triggered the MCP-side pull. The daemon now does it
659
+ // autonomously on the timer. Standalone (no dispatchMeshCommand) skips this
660
+ // phase entirely — there are no remote nodes to pull from.
661
+ //
662
+ // PHASE 2 — Live CLI inject. For each mesh that has a live CLI coordinator on
663
+ // THIS daemon, drain the local queue and inject pending events into the PTY.
664
+ // Unchanged from before.
665
+ // Bug B: how long a row may sit 'assigned' with an unconfirmed dispatch before the
666
+ // watchdog reclaims it. Must be comfortably larger than the per-dispatch confirm
667
+ // timeout (DISPATCH_CONFIRM_TIMEOUT_MS in mesh-events-coordinator) so a slow-but-live
668
+ // dispatch still inside its normal confirm window is never reclaimed early — this is
669
+ // the durable backstop for the case the in-process confirm timer can't cover (a timer
670
+ // lost to a daemon restart between claim and confirm).
671
+ const ASSIGNED_STRANDED_DEADLINE_MS = 5 * 60_000;
672
+
673
+ // COMPLETION-PROPAGATION F3: how long a row may sit 'assigned' with a CONFIRMED delivery
674
+ // (delivered/acked) but no terminal completion before the watchdog reclaims it as a
675
+ // delivered-but-lost completion. Distinct from — and deliberately larger than —
676
+ // ASSIGNED_STRANDED_DEADLINE_MS: a confirmed-delivered dispatch was genuinely handed to a
677
+ // worker, so the deadline must comfortably exceed any realistic single worker turn (a large
678
+ // generation) before we treat the missing completion as lost and re-open the task. Paired with
679
+ // the non-generating + no-terminal-ledger guards below so a worker still mid-turn is never
680
+ // reclaimed out from under itself.
681
+ const DELIVERED_NO_TURN_DEADLINE_MS = 15 * 60_000;
682
+
683
+ // DELIVERED-NOT-CONSUMED (remote autoLaunch delivered≠consumed gap): how long a row may sit
684
+ // 'assigned' with a CONFIRMED delivery ('delivered') that was never CONSUMED ('acked' — the
685
+ // worker's agent:generating_started never arrived) before the watchdog re-drives it. Far shorter
686
+ // than DELIVERED_NO_TURN_DEADLINE_MS (15min): a remote autoLaunch marks markAutoLaunch(completed)
687
+ // and returns immediately, relying on agent:ready/reconcile to inject; if the launch→ready→claim
688
+ // window (widened on win32 by the 3–4s git spawn latency) drops the inject, the row sits 'assigned'
689
+ // but the delivery never flips past 'delivered' to 'acked'. The delivered-not-acked state is the
690
+ // cross-daemon consumption signal — positive evidence the worker never started the turn — so we can
691
+ // safely re-open the task after a SHORT grace (well above a normal generating_started round-trip so
692
+ // a merely-slow start is never torn off) instead of waiting the full 15min turn budget. Floored
693
+ // comfortably above the auto-launch cooldown so a legitimate late inject still has room to land.
694
+ const ASSIGNED_DELIVERED_UNCONSUMED_REDRIVE_MS = 25_000;
695
+
696
+ // RECLAIM-FALSEPOS: how many CONSECUTIVE UNKNOWN busy-verdict ticks (past the delivered-no-turn
697
+ // deadline) must accumulate before a delivered row whose worker session cannot be positively
698
+ // observed is reclaimed. An UNKNOWN verdict means the assigned session is not present in THIS
699
+ // daemon's local instance map (remote / gone / id-form skew) — so it may be a REMOTE session that
700
+ // is genuinely mid-turn. Reclaiming it on a single UNKNOWN tick tears a live remote worker off its
701
+ // task and re-launches a near-duplicate (observed live 2026-07-04, session 21e34616 / task
702
+ // a26806c1). We therefore DEFER on UNKNOWN and only reclaim after this bounded grace, so a
703
+ // transient/remote absence never triggers a false reclaim while a genuinely-lost completion is
704
+ // still eventually recovered. A GENERATING or IDLE_CONFIRMED verdict (locally-present positive
705
+ // evidence) resets/bypasses the grace — see recoverStrandedAssignedDispatches.
706
+ const RECLAIM_UNKNOWN_GRACE_TICKS = 3;
707
+
708
+ // Per-row consecutive-UNKNOWN streak for delivered-no-turn reclaim, keyed `${meshId}::${taskId}`.
709
+ // In-memory (per process); pruned each pass to the set of currently-assigned rows so a
710
+ // completed/reclaimed/claimed-elsewhere row's counter is dropped (no unbounded growth).
711
+ const deliveredNoTurnUnknownStreak = new Map<string, number>();
712
+
713
+ // Test hook: clear the delivered-no-turn UNKNOWN streak between cases.
714
+ export function __resetReclaimUnknownStreakForTests(): void {
715
+ deliveredNoTurnUnknownStreak.clear();
716
+ }
717
+
718
+ // PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
719
+ // flips a row to 'assigned' BEFORE the fire-and-forget dispatch runs. If that dispatch
720
+ // neither rejects (→ no .catch requeue) nor is confirmed delivered — a relay that hangs
721
+ // without acking, or a confirm timer lost across a restart — the row stays 'assigned'
722
+ // forever: it contributes 0 pending, so PHASE 3 (gated on pendingQueueTaskCount>0) never
723
+ // re-examines it, and nothing but a manual requeue clears it. This is that missing net.
724
+ //
725
+ // Regression guard: a row whose delivery IS confirmed (delivered/acked/completed) is a
726
+ // genuinely in-flight (or completion-lost) task — left to PHASE 4's completion reconcile,
727
+ // never reclaimed here. And the deadline is generous so a slow-but-live dispatch still in
728
+ // its normal confirm window is never reclaimed early. Reclaimed rows return to 'pending'
729
+ // with ownership cleared, so the PHASE 3 trigger below re-dispatches them this same tick.
730
+ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId: string, store: MeshRuntimeStore): void {
731
+ const assigned = getQueue(meshId, { status: ['assigned'] });
732
+ if (!assigned.length) return;
733
+ const nowMs = Date.now();
734
+ // RECLAIM-FALSEPOS: prune UNKNOWN streaks for rows of THIS mesh that are no longer
735
+ // 'assigned' (completed / reclaimed / claimed elsewhere) so the counter map cannot grow
736
+ // unbounded and a re-used task id starts its grace fresh.
737
+ const assignedKeys = new Set(assigned.map(r => `${meshId}::${r.id}`));
738
+ const meshKeyPrefix = `${meshId}::`;
739
+ for (const key of [...deliveredNoTurnUnknownStreak.keys()]) {
740
+ if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) deliveredNoTurnUnknownStreak.delete(key);
741
+ }
742
+ for (const row of assigned) {
743
+ const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
744
+ if (!Number.isFinite(dispatchedAtMs)) continue; // no dispatch ts → can't age it
745
+ const ageMs = nowMs - dispatchedAtMs;
746
+ // DELIVERED-NOT-CONSUMED short-grace re-drive (remote autoLaunch delivered≠consumed gap).
747
+ // Runs BEFORE the ASSIGNED_STRANDED_DEADLINE_MS confirm-window gate below because its whole
748
+ // point is to recover a delivered-but-unconsumed row well inside that window. A remote
749
+ // autoLaunch marks the dispatch delivered (transport acked) but the worker may never emit
750
+ // agent:generating_started — the delivery then sits 'delivered' and never flips to 'acked',
751
+ // so the task is stranded 'assigned' with no live turn. This branch re-opens exactly that
752
+ // row after a short grace:
753
+ // - the delivery IS confirmed handed off (taskHasConfirmedDelivery) but was NEVER consumed
754
+ // (!taskDeliveryConsumed → no 'acked'/'completed' delivery) — the cross-daemon "worker
755
+ // never started the turn" signal, valid even for a REMOTE session whose local busy
756
+ // verdict is UNKNOWN;
757
+ // - AND the busy verdict is NOT GENERATING — a locally-present generating session IS
758
+ // consuming (ack lost/late), so never touch it (regression guard against tearing a live
759
+ // worker off its turn);
760
+ // - AND no terminal ledger evidence exists (the completion already landed → leave it).
761
+ // reclaimStrandedAssignedTask returns the row to 'pending' (bounded by MAX_STRANDED_RECLAIMS)
762
+ // so PHASE 3 re-dispatches it this same tick onto a fresh idle session — idempotent: it only
763
+ // mutates a still-'assigned' row, so a completion/ack that raced in already moved the row off
764
+ // 'assigned' and this is a no-op.
765
+ if (
766
+ ageMs >= ASSIGNED_DELIVERED_UNCONSUMED_REDRIVE_MS
767
+ && ageMs < ASSIGNED_STRANDED_DEADLINE_MS
768
+ && store.taskHasConfirmedDelivery(meshId, row.id)
769
+ && !store.taskDeliveryConsumed(meshId, row.id)
770
+ ) {
771
+ const terminal = findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id });
772
+ if (terminal) {
773
+ const status = terminal.kind === 'task_completed' ? 'completed' : 'failed';
774
+ updateTaskStatus(meshId, row.id, status);
775
+ continue;
776
+ }
777
+ const verdict = row.assignedSessionId
778
+ ? resolveSessionBusyVerdict(components, row.assignedSessionId)
779
+ : 'IDLE_CONFIRMED'; // no session bound → nothing live generating to protect
780
+ if (verdict !== 'GENERATING') {
781
+ const redriven = reclaimStrandedAssignedTask(meshId, row.id, {
782
+ reason: 'delivered_not_consumed_redrive',
783
+ ageMs,
784
+ });
785
+ if (redriven) {
786
+ LOG.warn('MeshReconcile', `Re-drove delivered-but-unconsumed task ${row.id} on mesh ${meshId} `
787
+ + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, delivered but no `
788
+ + `generating_started in ${Math.round(ageMs / 1000)}s, verdict ${verdict} → ${redriven.status})`);
789
+ traceMeshEventDrop('assigned_delivered_not_consumed_redrive', {
790
+ taskId: row.id,
791
+ sessionId: row.assignedSessionId,
792
+ nodeId: row.assignedNodeId,
793
+ meshId,
794
+ event: 'agent:generating_started',
795
+ }, `delivered_not_consumed ${Math.round(ageMs / 1000)}s → ${redriven.status}`);
796
+ continue;
797
+ }
798
+ }
799
+ }
800
+ if (ageMs < ASSIGNED_STRANDED_DEADLINE_MS) continue; // still in confirm window
801
+ const terminal = findTerminalLedgerEvidenceForTask({
802
+ meshId,
803
+ taskId: row.id,
804
+ });
805
+ if (terminal) {
806
+ const status = terminal.kind === 'task_completed' ? 'completed' : 'failed';
807
+ updateTaskStatus(meshId, row.id, status);
808
+ LOG.warn('MeshReconcile', `Skipped stranded reclaim redispatch for terminal task ${row.id} on mesh ${meshId}; ${terminal.kind} ledger evidence already exists`);
809
+ traceMeshEventDrop('assigned_stranded_terminal_ledger', {
810
+ taskId: row.id,
811
+ sessionId: row.assignedSessionId,
812
+ nodeId: row.assignedNodeId,
813
+ meshId,
814
+ event: 'agent:generating_completed',
815
+ }, terminal.kind);
816
+ continue;
817
+ }
818
+ if (store.taskHasConfirmedDelivery(meshId, row.id)) {
819
+ // COMPLETION-PROPAGATION F3 (delivered-but-lost completion): the dispatch WAS
820
+ // confirmed handed to a worker (delivered/acked) but no terminal completion ever
821
+ // landed and none is in the ledger (checked just above). Normally this is PHASE 4's
822
+ // job, but PHASE 4 only covers direct-dispatch rows / a live re-read; a claim-path
823
+ // queue row whose completion event was lost (the manual-launch flip-miss signature)
824
+ // sits 'assigned' forever.
825
+ //
826
+ // RECLAIM-FALSEPOS tri-state verdict: the reclaim used to gate ONLY on
827
+ // isSessionActivelyGenerating(), whose local instance lookup returns "not generating"
828
+ // for a REMOTE (or id-form-skewed) session that is genuinely mid-turn — so such a
829
+ // worker was reclaimed at the deadline and re-launched same tick (near-duplicate
830
+ // execution; observed live 2026-07-04, session 21e34616 / task a26806c1). Resolve an
831
+ // explicit GENERATING / IDLE_CONFIRMED / UNKNOWN verdict instead:
832
+ // - GENERATING → worker demonstrably alive; never reclaim, reset grace.
833
+ // - IDLE_CONFIRMED → positive LOCAL evidence (present instance, inactive) → reclaim
834
+ // now (past deadline) with the delivered-no-turn reason.
835
+ // - UNKNOWN → session not locally observable (remote / gone / id-skew). Do
836
+ // NOT fold into a definitive idle. DEFER: count consecutive
837
+ // UNKNOWN ticks and only reclaim after RECLAIM_UNKNOWN_GRACE_TICKS
838
+ // so a live remote worker is never torn off its task on a single
839
+ // absent observation; a genuinely-lost completion is still
840
+ // recovered after the bounded grace.
841
+ // reclaimStrandedAssignedTask ends the single-flight window (F4), so a subsequent
842
+ // re-dispatch/requeue is unblocked.
843
+ if (nowMs - dispatchedAtMs < DELIVERED_NO_TURN_DEADLINE_MS) continue; // still within turn budget
844
+ const streakKey = `${meshId}::${row.id}`;
845
+ const verdict = row.assignedSessionId
846
+ ? resolveSessionBusyVerdict(components, row.assignedSessionId)
847
+ : 'IDLE_CONFIRMED'; // no session bound → nothing live to protect
848
+ if (verdict === 'GENERATING') {
849
+ deliveredNoTurnUnknownStreak.delete(streakKey); // demonstrably alive → reset grace
850
+ continue; // worker still working
851
+ }
852
+ let reclaimReason: 'delivered_no_turn_deadline' | 'reclaim_after_unknown_grace';
853
+ if (verdict === 'IDLE_CONFIRMED') {
854
+ deliveredNoTurnUnknownStreak.delete(streakKey);
855
+ reclaimReason = 'delivered_no_turn_deadline';
856
+ } else {
857
+ // UNKNOWN — defer and accumulate the consecutive-UNKNOWN streak.
858
+ const streak = (deliveredNoTurnUnknownStreak.get(streakKey) ?? 0) + 1;
859
+ deliveredNoTurnUnknownStreak.set(streakKey, streak);
860
+ if (streak < RECLAIM_UNKNOWN_GRACE_TICKS) {
861
+ // Still within grace — hold this tick. Content-free trace (ids + streak only).
862
+ traceMeshEventDrop('reclaim_deferred_unknown_verdict', {
863
+ taskId: row.id,
864
+ sessionId: row.assignedSessionId,
865
+ nodeId: row.assignedNodeId,
866
+ meshId,
867
+ event: 'agent:generating_completed',
868
+ }, `unknown ${streak}/${RECLAIM_UNKNOWN_GRACE_TICKS}`);
869
+ continue;
870
+ }
871
+ reclaimReason = 'reclaim_after_unknown_grace';
872
+ }
873
+ const reclaimedLost = reclaimStrandedAssignedTask(meshId, row.id, {
874
+ reason: reclaimReason,
875
+ ageMs: nowMs - dispatchedAtMs,
876
+ });
877
+ if (reclaimedLost) {
878
+ deliveredNoTurnUnknownStreak.delete(streakKey);
879
+ LOG.warn('MeshReconcile', `Reclaimed delivered-but-lost task ${row.id} on mesh ${meshId} `
880
+ + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, delivered but no `
881
+ + `completion in ${Math.round((nowMs - dispatchedAtMs) / 1000)}s, verdict ${verdict} → ${reclaimReason} → ${reclaimedLost.status})`);
882
+ traceMeshEventDrop('assigned_stranded_delivered_no_turn', {
883
+ taskId: row.id,
884
+ sessionId: row.assignedSessionId,
885
+ nodeId: row.assignedNodeId,
886
+ meshId,
887
+ event: 'agent:generating_completed',
888
+ }, `delivered ${Math.round((nowMs - dispatchedAtMs) / 1000)}s ${reclaimReason} → ${reclaimedLost.status}`);
889
+ }
890
+ continue;
891
+ }
892
+ const reclaimed = reclaimStrandedAssignedTask(meshId, row.id, {
893
+ reason: 'assigned_stranded_dispatch_unconfirmed',
894
+ ageMs: nowMs - dispatchedAtMs,
895
+ });
896
+ if (reclaimed) {
897
+ LOG.warn('MeshReconcile', `Reclaimed stranded assigned task ${row.id} on mesh ${meshId} `
898
+ + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, dispatched `
899
+ + `${Math.round((nowMs - dispatchedAtMs) / 1000)}s ago, never confirmed delivered → ${reclaimed.status})`);
900
+ // EVTTRACE: the dispatch for this task was stranded (assigned, never confirmed
901
+ // delivered) and reclaimed (CANON-B) — its expected completion event never
902
+ // arrived. Observation only; the reclaim decision above is unchanged.
903
+ traceMeshEventDrop('assigned_stranded_reclaim', {
904
+ taskId: row.id,
905
+ sessionId: row.assignedSessionId,
906
+ nodeId: row.assignedNodeId,
907
+ meshId,
908
+ event: 'agent:generating_completed',
909
+ }, `unconfirmed ${Math.round((nowMs - dispatchedAtMs) / 1000)}s → ${reclaimed.status}`);
910
+ }
911
+ }
912
+ }
913
+
914
+ // ── PHASE 2.6: assigned-zombie sweep (runtime-store GC, SoT 1-11 (a)) ─────────
915
+ // recoverStrandedAssignedDispatches (PHASE 2.5) can only age a row by its
916
+ // dispatchTimestamp — a row that never got one (a legacy claim, a crashed claim
917
+ // path, a row whose payload drifted) is invisible to it FOREVER: it contributes 0
918
+ // pending (PHASE 3 skips), holds the node-busy gate (hasActiveNodeAssignment), and
919
+ // nothing ever transitions it. This sweep is that missing terminal net, scoped
920
+ // PRECISELY to the rows PHASE 2.5 can never touch (no parseable dispatchTimestamp)
921
+ // so the two nets never race each other over the same row.
922
+ //
923
+ // Conservative by construction:
924
+ // - age-gated on updatedAt/createdAt (>= ZOMBIE_ASSIGNED_MIN_AGE_MS) so a freshly
925
+ // claimed row mid-launch is never touched;
926
+ // - terminal ledger evidence wins first (row flips to the evidenced terminal,
927
+ // mirroring PHASE 2.5's terminal branch);
928
+ // - only fails a row whose assigned session is POSITIVELY absent on the daemon
929
+ // that owns the assigned node — a locally-present session (idle or generating)
930
+ // is skipped, and a REMOTE node's session (not locally observable) is skipped
931
+ // entirely rather than guessed dead;
932
+ // - the failure reason is explicit in both the queue mutation trace and a
933
+ // task_failed ledger entry, so the transition is auditable, never silent.
934
+ const ZOMBIE_ASSIGNED_MIN_AGE_MS = 30 * 60 * 1000; // 30 min — generous vs. session launch/restart races
935
+
936
+ export function reconcileZombieAssignedTasks(
937
+ components: DaemonComponents,
938
+ mesh: { id: string; nodes?: unknown[] },
939
+ selfIds: string[],
940
+ ): void {
941
+ const meshId = mesh.id;
942
+ const assigned = getQueue(meshId, { status: ['assigned'] });
943
+ if (!assigned.length) return;
944
+ const nowMs = Date.now();
945
+
946
+ // True when THIS daemon is authoritative for the row's assigned node — the only
947
+ // case where "no local instance" positively means "session no longer exists".
948
+ // Accepts a daemon-id form match against selfIds, or a mesh-node whose daemonId
949
+ // resolves to this daemon. Absent assignedNodeId → local (nothing remote to defer to).
950
+ const assignedNodeIsLocal = (assignedNodeId?: string): boolean => {
951
+ if (!assignedNodeId) return true;
952
+ if (selfIds.some(id => daemonIdsEquivalent(id, assignedNodeId))) return true;
953
+ const nodes = Array.isArray(mesh.nodes) ? mesh.nodes : [];
954
+ const node = nodes.find(n => meshNodeIdMatches(n as never, assignedNodeId)) as { daemonId?: unknown } | undefined;
955
+ const nodeDaemonId = readNonEmptyString(node?.daemonId);
956
+ return !!nodeDaemonId && selfIds.some(id => daemonIdsEquivalent(id, nodeDaemonId));
957
+ };
958
+
959
+ for (const row of assigned) {
960
+ // Rows WITH a parseable dispatchTimestamp belong to PHASE 2.5 — never double-handle.
961
+ if (Number.isFinite(Date.parse(row.dispatchTimestamp ?? ''))) continue;
962
+ const updatedMs = Date.parse(row.updatedAt ?? '');
963
+ const createdMs = Date.parse(row.createdAt ?? '');
964
+ const anchorMs = Number.isFinite(updatedMs) ? updatedMs : createdMs;
965
+ if (!Number.isFinite(anchorMs)) continue; // cannot age it → leave untouched
966
+ if (nowMs - anchorMs < ZOMBIE_ASSIGNED_MIN_AGE_MS) continue;
967
+
968
+ // A terminal already evidenced in the ledger → flip the row to that terminal
969
+ // (the completion arrived but the queue flip was lost), same as PHASE 2.5.
970
+ const terminal = findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id });
971
+ if (terminal) {
972
+ const status = terminal.kind === 'task_completed' ? 'completed' : 'failed';
973
+ updateTaskStatus(meshId, row.id, status);
974
+ LOG.warn('MeshReconcile', `Zombie assigned task ${row.id} on mesh ${meshId} had ${terminal.kind} ledger evidence — flipped to ${status}`);
975
+ continue;
976
+ }
977
+
978
+ if (!assignedNodeIsLocal(row.assignedNodeId)) continue; // remote session not locally observable — never guess
979
+ if (row.assignedSessionId) {
980
+ const verdict = resolveSessionBusyVerdict(components, row.assignedSessionId);
981
+ if (verdict !== 'UNKNOWN') continue; // session exists locally (idle or busy) → not a zombie
982
+ }
983
+
984
+ const reason = row.assignedSessionId
985
+ ? 'assigned_zombie_session_missing'
986
+ : 'assigned_zombie_no_session_bound';
987
+ const failed = updateTaskStatus(meshId, row.id, 'failed');
988
+ if (!failed) continue;
989
+ try {
990
+ appendLedgerEntry(meshId, {
991
+ kind: 'task_failed',
992
+ nodeId: row.assignedNodeId,
993
+ sessionId: row.assignedSessionId,
994
+ payload: {
995
+ taskId: row.id,
996
+ reason,
997
+ source: 'reconcile_zombie_assigned_sweep',
998
+ ageMs: nowMs - anchorMs,
999
+ },
1000
+ });
1001
+ } catch { /* ledger write is best-effort */ }
1002
+ LOG.warn('MeshReconcile', `Failed zombie assigned task ${row.id} on mesh ${meshId} `
1003
+ + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, no dispatchTimestamp, `
1004
+ + `stale ${Math.round((nowMs - anchorMs) / 60000)}m, ${reason})`);
1005
+ traceMeshEventDrop('assigned_zombie_failed', {
1006
+ taskId: row.id,
1007
+ sessionId: row.assignedSessionId,
1008
+ nodeId: row.assignedNodeId,
1009
+ meshId,
1010
+ event: 'agent:generating_completed',
1011
+ }, `${reason} stale=${Math.round((nowMs - anchorMs) / 60000)}m`);
1012
+ }
1013
+ }
1014
+
1015
+ export async function runMeshReconcileTick(components: DaemonComponents): Promise<void> {
1016
+ const localDaemonId = readNonEmptyString(loadConfig().machineId) || undefined;
1017
+ // The id-set used to scope the local queue drain (status id + machineId). See
1018
+ // resolveCoordinatorDaemonIds — the status id is what the MCP layer stamps and
1019
+ // is mandatory here for a generating CLI coordinator to self-receive completions.
1020
+ const drainDaemonIds = resolveCoordinatorDaemonIds(components);
1021
+ const dispatchMeshCommand = components.dispatchMeshCommand;
1022
+ const store = (() => {
1023
+ try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
1024
+ })();
1025
+
1026
+ // ── PHASE 0: retry the worker-side unresolved-delegate forward outbox ──────
1027
+ // Cloud-only (needs dispatchMeshCommand). A worker that is NOT a member of the
1028
+ // coordinator's mesh cannot be reached by the coordinator's PHASE 1 pull (it is
1029
+ // in no mesh.node), so its completion must be PUSHED to the coordinator. This
1030
+ // drains the durable outbox enqueued by forwardUnresolvedDelegateEvent and retries
1031
+ // any push that has not yet been acked. Since the spontaneous immediate push was
1032
+ // removed (polling single-model §2.1), this PHASE 0 retry is the ONLY delivery
1033
+ // path for unresolved-delegate events; the enqueue site nudges an early run of it
1034
+ // (scheduleUnresolvedForwardNudge) so happy-path latency stays sub-interval.
1035
+ // See mesh-unresolved-forward-outbox.ts.
1036
+ if (dispatchMeshCommand) {
1037
+ try {
1038
+ await retryUnresolvedDelegateForwards(components);
1039
+ } catch (e: any) {
1040
+ LOG.warn('MeshReconcile', `Unresolved-delegate forward retry failed: ${e?.message || e}`);
1041
+ }
1042
+ }
1043
+
1044
+ // ── PHASE 1: pull remote node queues for every mesh this daemon hosts ──────
1045
+ // Cloud-only (dispatchMeshCommand present). Runs whether or not a live CLI
1046
+ // coordinator exists — this is what lets an MCP/LLM coordinator ever see a
1047
+ // remote worker's completion.
1048
+ if (dispatchMeshCommand) {
1049
+ for (const mesh of listMeshes()) {
1050
+ // Expand to every id-form this daemon answers to for this mesh (runtime
1051
+ // drain ids ∪ config-form node/host ids) and use it for BOTH the host gate
1052
+ // and the remote pull filter, so a worker stamp in any form is recovered.
1053
+ const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1054
+ if (!daemonHostsMesh(mesh, selfIds)) continue;
1055
+ try {
1056
+ await pullRemoteNodeQueues(components, mesh, localDaemonId, selfIds);
1057
+ } catch (e: any) {
1058
+ LOG.warn('MeshReconcile', `Remote node pull failed for mesh ${mesh.id}: ${e?.message || e}`);
1059
+ }
1060
+ }
1061
+ }
1062
+
1063
+ // ── PHASE 2.5: assigned-stranded dispatch watchdog (Bug B) ─────────────────
1064
+ // Runs before PHASE 3 so any row it returns to 'pending' is re-dispatched by the
1065
+ // PHASE 3 trigger in this same tick. See recoverStrandedAssignedDispatches.
1066
+ if (store) {
1067
+ for (const mesh of listMeshes()) {
1068
+ const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1069
+ if (!daemonHostsMesh(mesh, selfIds)) continue;
1070
+ try {
1071
+ recoverStrandedAssignedDispatches(components, mesh.id, store);
1072
+ } catch (e: any) {
1073
+ LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
1074
+ }
1075
+ // PHASE 2.6 — assigned-zombie sweep: terminal-fails the rows PHASE 2.5
1076
+ // can never age (no dispatchTimestamp) whose session is positively gone.
1077
+ try {
1078
+ reconcileZombieAssignedTasks(components, mesh, selfIds);
1079
+ } catch (e: any) {
1080
+ LOG.warn('MeshReconcile', `Assigned-zombie sweep failed for mesh ${mesh.id}: ${e?.message || e}`);
1081
+ }
1082
+ }
1083
+ }
1084
+
1085
+ // ── PHASE 3: recover pending queue claims for newly-idle sessions ──────────
1086
+ // The event-driven claim paths (agent:ready / agent:generating_completed in
1087
+ // mesh-events-coordinator) re-claim the queue the moment a session goes idle,
1088
+ // but that depends on a single event being emitted AND (for a remote node)
1089
+ // successfully forwarded to this coordinator. If that event is missed/dropped,
1090
+ // a pending task targeting a now-idle session would sit unclaimed forever —
1091
+ // there was no periodic safety net. This phase is that net: for every mesh this
1092
+ // daemon hosts that has at least one pending task, run one triggerMeshQueue so a
1093
+ // session that became idle without a delivered ready-event still gets its work.
1094
+ //
1095
+ // O(1) guard: skip the (relatively expensive) full idle-session + remote-idle
1096
+ // scan entirely when the queue has no pending tasks — a COUNT(*) over the
1097
+ // indexed status column, so an idle mesh costs one cheap query per tick.
1098
+ // claimNextQueueTask is atomic, so racing the event-driven path can only have
1099
+ // one winner; double-claiming is impossible.
1100
+ for (const mesh of listMeshes()) {
1101
+ const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1102
+ if (!daemonHostsMesh(mesh, selfIds)) continue;
1103
+ if (store) {
1104
+ try {
1105
+ if (store.pendingQueueTaskCount(mesh.id) === 0) continue;
1106
+ } catch { /* fall through and let triggerMeshQueue decide */ }
1107
+ }
1108
+ try {
1109
+ await triggerMeshQueue(components, mesh.id);
1110
+ } catch (e: any) {
1111
+ LOG.warn('MeshReconcile', `Pending-claim recovery trigger failed for mesh ${mesh.id}: ${e?.message || e}`);
1112
+ }
1113
+ }
1114
+
1115
+ // ── PHASE 4: synthesize lost completions for unterminated direct dispatches ─
1116
+ // Symmetric to PHASE 3 (which recovers a *lost claim* for a newly-idle session)
1117
+ // but for the opposite gap: a worker that ALREADY completed, went idle, and
1118
+ // whose terminal completion event was never persisted (dropped before reaching
1119
+ // the queue/outbox, or its forward was lost). PHASE 1/2/3 can only deliver an
1120
+ // event that exists in a queue — they cannot recover a completion that was
1121
+ // never recorded, so the coordinator keeps believing the worker is generating.
1122
+ //
1123
+ // reconcileDirectDispatchCompletionFromTranscript already synthesizes the
1124
+ // missing terminal event from the worker's transcript, but until now it ran
1125
+ // ONLY when an LLM coordinator polled mesh_status (mcp_mesh_status_transcript_
1126
+ // reconciliation). This phase pulls that same correction onto the daemon timer
1127
+ // so it no longer depends on the LLM polling. The reconcile is idempotent
1128
+ // (hasTerminalLedgerAfterDispatch guards against re-synthesis), so attempting it
1129
+ // every tick for the same dispatch is safe — once a terminal exists it no-ops.
1130
+ for (const mesh of listMeshes()) {
1131
+ const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1132
+ if (!daemonHostsMesh(mesh, selfIds)) continue;
1133
+ try {
1134
+ await reconcileUnterminatedDirectDispatches(components, mesh, selfIds, localDaemonId);
1135
+ } catch (e: any) {
1136
+ LOG.warn('MeshReconcile', `Completion reconcile failed for mesh ${mesh.id}: ${e?.message || e}`);
1137
+ }
1138
+ }
1139
+
1140
+ // ── PHASE 5: auto-prune orphaned direct dispatch records ───────────────────
1141
+ // staleDirectWork (orphaned direct-dispatch rows whose node/session is no longer in the
1142
+ // live mesh) otherwise accumulates indefinitely: a removed worktree node or a cleanly
1143
+ // terminated session leaves its direct-dispatch row behind, stuck in a non-terminal status
1144
+ // (e.g. generating) for days. This is NOT a false-idle bug — it is the separate problem of
1145
+ // orphaned records that the only existing cleanup path (manual MCP mesh_prune_stale_direct)
1146
+ // never reaches unless an operator runs it by hand.
1147
+ //
1148
+ // This phase runs the SAME prune core the manual tool calls (pruneStaleDirectDispatches),
1149
+ // in execute mode, on the daemon timer. The only difference from the manual path is a
1150
+ // conservative age gate (DEFAULT_AUTO_PRUNE_MIN_AGE_MS): a freshly-orphaned record is held
1151
+ // back until it is provably stale, so a transient probe miss never auto-prunes live work.
1152
+ // Every other safety rule is inherited unchanged from the core — active/pending/generating
1153
+ // work and fresh unacknowledged dispatch failures are never pruned, ledger-only audit entries
1154
+ // are preserved, and the prune itself is recorded with a direct_dispatch_pruned ledger entry.
1155
+ // Idempotent: a pruned row is gone from getActiveDirectDispatches, so the next tick finds
1156
+ // nothing to re-prune. Isolated in its own try/catch per mesh so it can never kill the tick.
1157
+ {
1158
+ const minAgeMs = resolveAutoPruneMinAgeMs();
1159
+ for (const mesh of listMeshes()) {
1160
+ const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1161
+ if (!daemonHostsMesh(mesh, selfIds)) continue;
1162
+ try {
1163
+ await autoPruneStaleDirectDispatches(components, mesh, selfIds, localDaemonId, minAgeMs);
1164
+ } catch (e: any) {
1165
+ LOG.warn('MeshReconcile', `Auto-prune stale direct failed for mesh ${mesh.id}: ${e?.message || e}`);
1166
+ }
1167
+ }
1168
+ }
1169
+
1170
+ // ── PHASE 2: inject into live CLI coordinators on this daemon ──────────────
1171
+ const coordinators = findLiveCoordinators(components);
1172
+ if (coordinators.length === 0) {
1173
+ // No live CLI coordinator on this daemon — nothing to inject into.
1174
+ // (MCP-only LLM coordinators drain the local queue via their own tool
1175
+ // calls; PHASE 1 above has already populated it from remote nodes.)
1176
+ return;
1177
+ }
1178
+
1179
+ // Group coordinators by mesh; multiple coordinator instances for one mesh is
1180
+ // unusual but supported (each gets the same drained events).
1181
+ const byMesh = new Map<string, LiveCoordinator[]>();
1182
+ for (const c of coordinators) {
1183
+ const list = byMesh.get(c.meshId);
1184
+ if (list) list.push(c);
1185
+ else byMesh.set(c.meshId, [c]);
1186
+ }
1187
+
1188
+ for (const [meshId, meshCoordinators] of byMesh) {
1189
+ // Drain the local queue scoped to this coordinator daemon and inject.
1190
+ // - If an idle coordinator exists, FULL-drain and deliver every event to it
1191
+ // (the idle input box accepts the prompt as a real next turn). The drain
1192
+ // marks consumed rows drained=1 atomically, so the pull path can't re-deliver.
1193
+ // - If only GENERATING coordinators exist (no idle target), we HOLD: leave the
1194
+ // events queued (drained=0) for the coordinator's next idle/turn-end tick.
1195
+ //
1196
+ // NOTIF-SURFACE-LOCAL (false-idle hold): we used to force-inject terminal events
1197
+ // (completion/approval/stop/refine·bootstrap) straight into a *generating*
1198
+ // coordinator's PTY (forceSendMessage → atomic content+\r write), on the theory it
1199
+ // bypassed the busy send-guard and broke the await-result deadlock. But a raw PTY
1200
+ // write into a claude-cli that is mid-generation is NOT consumed as a new turn — the
1201
+ // bytes land in the terminal input buffer and the LLM never reads them on its next
1202
+ // turn. The `surfaced/force-inject` trace fired, the row was marked drained=1, and the
1203
+ // genuine completion was lost forever (the exact same-daemon local-worktree miss: the
1204
+ // coordinator's OWN session is generating at the moment its worker completes). The
1205
+ // deadlock the force path guarded against does not actually require force: a
1206
+ // coordinator that dispatched a task via mesh_send_task returns to idle when that
1207
+ // tool call resolves (dispatch is fire-and-forget; the worker runs for minutes while
1208
+ // the coordinator is idle/between turns), so the completion lands on the very next
1209
+ // idle tick (≤ one reconcile interval). Holding the event undrained for that idle
1210
+ // tick is therefore the single, reliable delivery — and it is the SAME skip-and-hold
1211
+ // the modal-park branch below already uses. This also makes double-injection
1212
+ // structurally impossible: there is exactly one delivery path (the idle full-drain),
1213
+ // so we never need a surface-time fingerprint to dedup a force-write against a re-drain.
1214
+ const idleCoordinators = meshCoordinators.filter(c => c.idle);
1215
+ // A coordinator parked on a harness modal (waiting_choice / waiting_approval) is
1216
+ // non-idle; it is held under the modal-park branch (a force-inject into a modal would
1217
+ // write raw keystrokes the modal key handler eats, silently selecting a choice the
1218
+ // user never made). A plainly-generating coordinator (non-idle, non-modal-parked) is
1219
+ // ALSO held now — for the false-idle reason above — but separately, so the C1 ledger
1220
+ // audit and the operator-facing skip log can name the right hold reason.
1221
+ const generatingCoordinators = meshCoordinators.filter(c => !c.idle && !c.modalParked);
1222
+ const modalParkedCoordinators = meshCoordinators.filter(c => !c.idle && c.modalParked);
1223
+ // Only an IDLE coordinator is a deliverable target. A generating coordinator's PTY
1224
+ // does not consume an injected prompt as a turn, so it is held (not a target).
1225
+ const targetCoordinators = idleCoordinators;
1226
+
1227
+ // ── no-idle-target short-circuit (MUST precede the drain) ─────────────────
1228
+ // When there is no IDLE coordinator for this mesh — only generating and/or
1229
+ // modal-parked ones — there is nowhere a queued event can land as a real turn.
1230
+ // We skip-and-hold: by NOT draining we leave the events at drained=0 in the queue,
1231
+ // so a later tick (once a coordinator returns to idle) delivers them. This
1232
+ // short-circuit MUST run BEFORE drainPendingMeshCoordinatorEvents — the drain marks
1233
+ // rows drained=1 atomically, which would lose the events for a coordinator that is
1234
+ // only transiently busy (the false-idle local-worktree miss). Both the generating
1235
+ // hold and the modal-park hold record a C1 ledger audit copy so a held completion's
1236
+ // worker summary is recoverable even if the coordinator never returns or the pending
1237
+ // file is later trimmed.
1238
+ if (targetCoordinators.length === 0) {
1239
+ // ── APPROVAL-Q1-REALTIME: level-deliver approval nudges BEFORE the hold ──
1240
+ // Approval events are LEVEL-backed (task_approval_needed ledger →
1241
+ // mesh_status awaiting_approval), so they must not be edge-held like a
1242
+ // completion (whose payload lives only in the pending event). Drain and
1243
+ // deliver them to the busy coordinator's inbox (non-force, next-turn-boundary)
1244
+ // this tick, dropping any already-resolved (stale) nudge — and leave ONLY the
1245
+ // completion/other events in the queue for the existing hold semantics below
1246
+ // (their behaviour is unchanged: shouldForceInjectMeshEvent no longer sees the
1247
+ // approval rows because this drained them). MUST run first so the modal-park
1248
+ // orphan-escape and the generating-hold audit only ever see non-approval events.
1249
+ drainAndDeliverApprovalNudges(meshId, drainDaemonIds, localDaemonId, meshCoordinators);
1250
+ // If approval nudges were the only queued events, nothing remains to hold — skip
1251
+ // the hold branches (and their "holding pending event(s)" log) entirely.
1252
+ if (store) {
1253
+ try { if (store.pendingEventCount(meshId) === 0) continue; } catch { /* fall through */ }
1254
+ }
1255
+ if (modalParkedCoordinators.length > 0) {
1256
+ // ── orphan escape (MUST precede the blanket modal-park hold) ──────────
1257
+ // A modal-parked coordinator with no idle/generating sibling otherwise
1258
+ // wedges EVERY pending event under `modal_parked` until that modal resolves
1259
+ // — including a STRICT-routed completion whose originating coordinator
1260
+ // session is GONE (an orphan: the worktree/session that produced it was
1261
+ // removed, or that coordinator session died). Such an event will never be
1262
+ // deliverable to its target session no matter what the modal-parked sibling
1263
+ // does, so holding it under modal_parked is a permanent-held leak (the very
1264
+ // "data restart re-reproduces it" symptom — the gate is reconstructed live
1265
+ // from the still-parked modal, so a restart does not clear it). Route those
1266
+ // orphan events through the strict-route hold/expire path so the bounded
1267
+ // STRICT_SESSION_MATCH_TTL eventually expires them (recoverable, ledgered)
1268
+ // instead of leaving them held forever. A strict event whose target session
1269
+ // IS live but merely modal-parked is left to the blanket hold below (it is
1270
+ // genuinely transiently blocked, not orphaned).
1271
+ const liveSessionIds = new Set(
1272
+ meshCoordinators.map(c => readNonEmptyString(c.sessionId)).filter(Boolean),
1273
+ );
1274
+ let orphanEscaped = 0;
1275
+ const hasPendingForOrphanPeek = !store
1276
+ || (() => { try { return store.pendingEventCount(meshId) > 0; } catch { return true; } })();
1277
+ if (hasPendingForOrphanPeek) {
1278
+ // Identify which pending event NAMES correspond to orphan-targeted events
1279
+ // (a strict targetCoordinatorSessionId that matches no live coordinator).
1280
+ let peeked: readonly PendingMeshCoordinatorEvent[] = [];
1281
+ try {
1282
+ peeked = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
1283
+ } catch { peeked = []; }
1284
+ const isOrphan = (e: PendingMeshCoordinatorEvent): boolean => {
1285
+ const want = readNonEmptyString(e.targetCoordinatorSessionId);
1286
+ return !!want && !liveSessionIds.has(want);
1287
+ };
1288
+ const orphanEventNames = new Set(peeked.filter(isOrphan).map(e => e.event));
1289
+ if (orphanEventNames.size > 0) {
1290
+ // The drain filter is event-NAME scoped (not per-row), so draining by the
1291
+ // orphan event names also pulls any non-orphan event sharing that name. Drain
1292
+ // them all, then re-route: orphan-targeted events go through the strict-route
1293
+ // hold/expire path (bounded TTL → eventually ledger-expired, recoverable);
1294
+ // non-orphan events of the same name are re-queued unchanged (queuedAt
1295
+ // preserved) so they remain genuinely held for their still-live, modal-parked
1296
+ // target. This is the same per-event strict routing PHASE 2 does below — just
1297
+ // reached here because the blanket modal-park short-circuit would otherwise
1298
+ // wedge the orphans forever.
1299
+ let drained: PendingMeshCoordinatorEvent[] = [];
1300
+ try {
1301
+ drained = drainPendingMeshCoordinatorEvents(
1302
+ meshId,
1303
+ drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
1304
+ { onlyEvents: orphanEventNames },
1305
+ );
1306
+ } catch (e: any) {
1307
+ LOG.warn('MeshReconcile', `Orphan-escape drain failed for mesh ${meshId}: ${e?.message || e}`);
1308
+ drained = [];
1309
+ }
1310
+ for (const pending of drained) {
1311
+ if (isOrphan(pending)) {
1312
+ holdOrExpireStrictUnmatchedEvent(pending, readNonEmptyString(pending.targetCoordinatorSessionId), meshId);
1313
+ orphanEscaped++;
1314
+ } else {
1315
+ // Still-live (modal-parked) target — re-queue unchanged so it is held
1316
+ // for the next modal-resolved tick, exactly like the blanket hold would.
1317
+ try { queuePendingMeshCoordinatorEvent(pending); } catch { /* best-effort re-queue */ }
1318
+ }
1319
+ }
1320
+ }
1321
+ }
1322
+ LOG.info('MeshReconcile', `Reconcile skip → modal-parked: holding pending event(s) for mesh ${meshId} (${modalParkedCoordinators.length} coordinator(s) awaiting a modal answer; events left queued${orphanEscaped > 0 ? `; ${orphanEscaped} orphan-targeted event(s) routed to strict-route TTL` : ''})`);
1323
+ // NOTIF (B) diagnostic: name the session(s) classified modal-parked so the
1324
+ // same-tick coordDiag line (paired by sessionId) shows whether the modal-park
1325
+ // overlay is a real human-await or an unreleased mask (the getState_overlay origin).
1326
+ if (getLogLevel() === 'debug') {
1327
+ LOG.debug('MeshReconcile', `coordHoldModalParked mesh=${meshId} heldFor=[${modalParkedCoordinators.map(c => c.sessionId || '?').join(',')}] (these were classified modal-parked; cross-ref same-tick coordDiag by sessionId)`);
1328
+ }
1329
+ // C1: mirror held terminal events into the ledger so a held completion's
1330
+ // worker summary is auditable/recoverable even if the modal is never
1331
+ // resolved, the coordinator restarts, or the pending file is later trimmed.
1332
+ // The events stay queued (drained=0) for re-drain on a later tick; this only
1333
+ // adds the durable audit copy. Idempotent per process — only newly-held
1334
+ // events are logged. O(1)-gated: skip the peek when the queue is empty.
1335
+ let hasPending = true;
1336
+ if (store) {
1337
+ try { hasPending = store.pendingEventCount(meshId) > 0; } catch { /* peek below */ }
1338
+ }
1339
+ if (hasPending) {
1340
+ recordHeldTerminalEventsToLedger(
1341
+ meshId,
1342
+ drainDaemonIds.length > 0 ? drainDaemonIds : (localDaemonId ? [localDaemonId] : []),
1343
+ 'modal_parked',
1344
+ modalParkedCoordinators.length,
1345
+ );
1346
+ }
1347
+ } else if (generatingCoordinators.length > 0) {
1348
+ // ── generating hold (NOTIF-SURFACE-LOCAL false-idle fix) ─────────────
1349
+ // The only coordinator(s) for this mesh are plainly generating (no idle, no
1350
+ // modal). A raw force-write into a generating claude-cli PTY is not consumed
1351
+ // as a turn, so we do NOT inject and do NOT drain — the events stay queued
1352
+ // (drained=0) and the next tick that finds the coordinator idle full-drains
1353
+ // them as real turns (the coordinator returns to idle when its current
1354
+ // tool-call/turn resolves; a dispatched worker runs for minutes while the
1355
+ // coordinator is idle, so this lands within one reconcile interval). C1: mirror
1356
+ // any held terminal events into the ledger so a completion's worker summary is
1357
+ // recoverable even before that idle tick. Idempotent per process; O(1)-gated.
1358
+ let hasPending = true;
1359
+ if (store) {
1360
+ try { hasPending = store.pendingEventCount(meshId) > 0; } catch { /* peek below */ }
1361
+ }
1362
+ if (hasPending) {
1363
+ // ── PTY-OVERTRUST-DRAIN (Defect B, fix B): age-based escape ───────────
1364
+ // Fix A already routes the common mask-driven false-busy to the idle path,
1365
+ // so reaching here means the coordinator's RAW adapter reads generating.
1366
+ // That is almost always genuine — but a status-source desync fix A does not
1367
+ // reach can momentarily make the raw adapter read generating while the PTY
1368
+ // is actually at a turn end, stranding the completion across many ticks. As a
1369
+ // TIME-BASED BACKSTOP, once the oldest held terminal event has aged past the
1370
+ // escalate threshold, RE-CONFIRM each held coordinator's raw adapter idle and,
1371
+ // if genuinely idle, drain ONCE into it. The re-confirmation gate is what makes
1372
+ // this safe: it NEVER injects into a genuinely-generating PTY (that is the
1373
+ // data-loss force-inject path intentionally removed). A coordinator still
1374
+ // genuinely generating stays held.
1375
+ const escalateMs = resolvePendingHeldDrainEscalateMs();
1376
+ const heldAgeMs = oldestHeldTerminalEventAgeMs(
1377
+ meshId,
1378
+ drainDaemonIds.length > 0 ? drainDaemonIds : (localDaemonId ? [localDaemonId] : []),
1379
+ );
1380
+ if (heldAgeMs >= escalateMs) {
1381
+ const escapeTargets = reconfirmGenuinelyIdleCoordinators(generatingCoordinators);
1382
+ if (escapeTargets.length > 0) {
1383
+ LOG.info('MeshReconcile', `Reconcile age-escape → generating-hold: held terminal event(s) for mesh ${meshId} aged ${Math.round(heldAgeMs / 1000)}s (≥ ${Math.round(escalateMs / 1000)}s) and ${escapeTargets.length} coordinator(s) re-confirmed genuinely idle on the raw adapter — draining once`);
1384
+ const drained = drainAndInjectIntoTargets(meshId, drainDaemonIds, localDaemonId, escapeTargets, 'age-escape');
1385
+ if (drained > 0) continue; // delivered → no hold this tick
1386
+ }
1387
+ }
1388
+ LOG.info('MeshReconcile', `Reconcile skip → generating: holding pending event(s) for mesh ${meshId} (${generatingCoordinators.length} coordinator(s) busy; events left queued for the next idle tick)`);
1389
+ // NOTIF (B) diagnostic: this is the hold that strands the completion. Name
1390
+ // the sessionId(s) the loop just classified non-idle/non-modal so the
1391
+ // same-tick coordDiag line above (paired by sessionId) reveals which status
1392
+ // source diverged. If a coordDiag for one of these sessions shows getState
1393
+ // (or lastStatus/adapterRaw) === idle, that is the runtime desync origin.
1394
+ if (getLogLevel() === 'debug') {
1395
+ LOG.debug('MeshReconcile', `coordHoldGenerating mesh=${meshId} heldFor=[${generatingCoordinators.map(c => c.sessionId || '?').join(',')}] (these were classified busy; cross-ref same-tick coordDiag by sessionId)`);
1396
+ }
1397
+ recordHeldTerminalEventsToLedger(
1398
+ meshId,
1399
+ drainDaemonIds.length > 0 ? drainDaemonIds : (localDaemonId ? [localDaemonId] : []),
1400
+ 'generating_no_idle_coordinator',
1401
+ generatingCoordinators.length,
1402
+ );
1403
+ }
1404
+ }
1405
+ continue;
1406
+ }
1407
+
1408
+ // O(1) guard: skip the drain entirely when the queue is empty.
1409
+ if (store) {
1410
+ try {
1411
+ if (store.pendingEventCount(meshId) === 0) {
1412
+ // An idle coordinator is present (the no-idle short-circuit already
1413
+ // `continue`d above) and the pending queue is empty → this mesh is at a
1414
+ // fully-idle edge with nothing to inject. Nudge (once/debounced) if it
1415
+ // still has active missions but no work in flight, so a lingering mission
1416
+ // is not left drifting in 'active'. Best-effort; never blocks the loop.
1417
+ maybeInjectIdleActiveMissionReminder(
1418
+ meshId,
1419
+ targetCoordinators[0].instance,
1420
+ getMesh(meshId)?.policy,
1421
+ );
1422
+ continue;
1423
+ }
1424
+ } catch { /* fall through to drain */ }
1425
+ }
1426
+
1427
+ // An idle coordinator is present (targetCoordinators.length > 0): FULL-drain every
1428
+ // queued event and deliver it to the idle input box as a real turn. The no-idle case
1429
+ // (generating/modal-only) was already held above and never reaches here, so there is
1430
+ // no force-drain-into-generating path left — the single delivery is the idle drain.
1431
+ drainAndInjectIntoTargets(meshId, drainDaemonIds, localDaemonId, targetCoordinators, 'idle');
1432
+ }
1433
+ }
1434
+
1435
+ // Strict-routing TTL: how long a drained completion whose originating coordinator session
1436
+ // is not currently deliverable is held (re-queued for re-drain) before it is ledger-
1437
+ // expired. Bounded so a coordinator session that never returns cannot wedge the event
1438
+ // forever; broad enough to ride out a transient modal-park / brief restart.
1439
+ const STRICT_SESSION_MATCH_TTL_MS = 60_000;
1440
+
1441
+ // Re-queue (hold) a strict-routed event whose coordinator session is not live, or — once it
1442
+ // has aged past STRICT_SESSION_MATCH_TTL_MS — ledger-expire it (recoverable) and drop it.
1443
+ // We deliberately do NOT broadcast an aged-out event to sibling coordinators: that is the
1444
+ // very misroute strict routing exists to prevent. The drain already marked the row drained=1,
1445
+ // so re-queuing re-persists a fresh undrained copy (dedup keys on drained=0 only); queuedAt is
1446
+ // preserved so the TTL measures the event's true age across re-queues.
1447
+ function holdOrExpireStrictUnmatchedEvent(
1448
+ pending: PendingMeshCoordinatorEvent,
1449
+ wantSession: string,
1450
+ meshId: string,
1451
+ ): void {
1452
+ const queuedAt = typeof pending.queuedAt === 'number' ? pending.queuedAt : Date.now();
1453
+ if (Date.now() - queuedAt <= STRICT_SESSION_MATCH_TTL_MS) {
1454
+ try {
1455
+ queuePendingMeshCoordinatorEvent(pending); // preserves queuedAt → true age retained
1456
+ LOG.info('MeshReconcile', `Strict route hold: coordinator session ${wantSession} not live on mesh ${meshId} — re-queued (${pending.event})`);
1457
+ // EVTTRACE: event held (re-queued) — its originating coordinator session is not
1458
+ // currently deliverable. Held, not dropped; surfaces later or expires past TTL.
1459
+ traceMeshEventDrop('strict_route_hold', {
1460
+ taskId: pending.metadataEvent?.taskId,
1461
+ sessionId: pending.metadataEvent?.targetSessionId ?? wantSession,
1462
+ nodeId: pending.nodeId,
1463
+ meshId,
1464
+ event: pending.event,
1465
+ }, `coordinatorSession=${wantSession} not live`);
1466
+ } catch (e: any) {
1467
+ LOG.warn('MeshReconcile', `Strict route re-queue failed for ${pending.event} on mesh ${meshId}: ${e?.message || e}`);
1468
+ }
1469
+ return;
1470
+ }
1471
+ const finalSummary = readMeshCompletionSummary(pending.metadataEvent || {});
1472
+ try {
1473
+ appendLedgerEntry(meshId, {
1474
+ kind: 'event_held',
1475
+ ...(pending.nodeId ? { nodeId: pending.nodeId } : {}),
1476
+ payload: {
1477
+ event: pending.event,
1478
+ reason: 'strict_route_expired',
1479
+ recoverable: true,
1480
+ targetCoordinatorSessionId: wantSession,
1481
+ targetCoordinatorDaemonId: pending.targetCoordinatorDaemonId ?? null,
1482
+ nodeLabel: pending.nodeLabel,
1483
+ ...(pending.workspace ? { workspace: pending.workspace } : {}),
1484
+ queuedAt,
1485
+ ...(finalSummary ? { finalSummary } : {}),
1486
+ },
1487
+ });
1488
+ LOG.warn('MeshReconcile', `Strict route expire: coordinator session ${wantSession} never returned for mesh ${meshId} — recorded to ledger (recoverable), dropped (${pending.event})`);
1489
+ // EVTTRACE: event expired past the strict-route TTL — dropped (recoverable, ledgered).
1490
+ traceMeshEventDrop('strict_route_expired', {
1491
+ taskId: pending.metadataEvent?.taskId,
1492
+ sessionId: pending.metadataEvent?.targetSessionId ?? wantSession,
1493
+ nodeId: pending.nodeId,
1494
+ meshId,
1495
+ event: pending.event,
1496
+ }, `coordinatorSession=${wantSession} never returned`);
1497
+ } catch (e: any) {
1498
+ LOG.warn('MeshReconcile', `Failed to ledger-expire strict-unmatched ${pending.event} for mesh ${meshId}: ${e?.message || e}`);
1499
+ }
1500
+ }
1501
+
1502
+ // Cloud-only: retry the worker-side unresolved-delegate forward outbox. For each
1503
+ // durably-queued entry, push it to its coordinator daemon over P2P (mesh_forward_event)
1504
+ // and ack (mark drained) ONLY on a successful, non-rejected response. A failed or
1505
+ // rejected push leaves the entry queued for the next tick — at-least-once delivery.
1506
+ // Stale entries (coordinator unreachable past the max age) are expired first so the
1507
+ // outbox can't grow without bound. The coordinator dedups duplicate deliveries on its
1508
+ // own fingerprint, so a retry that races the original immediate push is harmless.
1509
+ // RECONCILE-MESHID-DROP: per-entry count of consecutive HARD rejections (the coordinator
1510
+ // returned success:false, e.g. "meshId required"). A rejection means the push was delivered
1511
+ // and deterministically refused — retrying the identical payload every 4s can never succeed,
1512
+ // so it would loop until the 30-minute age expiry, spamming the log the whole time. After
1513
+ // MAX_FORWARD_REJECTIONS such rejections we drop the entry (drain it) with ONE fail-loud
1514
+ // warning. Transient transport failures (the dispatch throws — coordinator momentarily
1515
+ // unreachable) do NOT count here; those legitimately retry until the age expiry. In-memory
1516
+ // (keyed by the durable outbox row id) is sufficient: a daemon restart re-arms the loop, and
1517
+ // the age expiry remains the durable backstop. Cleared whenever an entry is delivered/drained.
1518
+ const unresolvedForwardRejectionCounts = new Map<string, number>();
1519
+ const MAX_FORWARD_REJECTIONS = 5;
1520
+
1521
+ export function __resetUnresolvedForwardRejectionCountsForTests(): void {
1522
+ unresolvedForwardRejectionCounts.clear();
1523
+ }
1524
+
1525
+ // ── Unresolved-forward reconcile nudge (polling single-model §2.1 (B)) ────────
1526
+ // forwardUnresolvedDelegateEvent no longer pushes the event itself — it only
1527
+ // persists to the durable outbox and fires a data-free nudge asking THIS loop to
1528
+ // run the PHASE 0 retry soon. The nudge is:
1529
+ // - coalesced: one pending timer at a time, so a completion burst schedules a
1530
+ // single early retry pass instead of one per event;
1531
+ // - non-overlapping: skipped while a nudged pass is still in flight (the
1532
+ // periodic tick remains the backstop);
1533
+ // - loss-tolerant: an unregistered/cleared/failed nudge merely means delivery
1534
+ // waits for the next periodic tick (≤ one reconcile interval) — never a loss.
1535
+ const UNRESOLVED_FORWARD_NUDGE_DELAY_MS = 250;
1536
+ let unresolvedForwardNudgeTimer: NodeJS.Timeout | undefined;
1537
+ let unresolvedForwardNudgeRunning = false;
1538
+
1539
+ function scheduleUnresolvedForwardNudge(components: DaemonComponents): void {
1540
+ if (!components.dispatchMeshCommand) return; // no transport → periodic tick handles/no-ops
1541
+ if (unresolvedForwardNudgeTimer) return; // coalesce a burst into one early pass
1542
+ unresolvedForwardNudgeTimer = setTimeout(() => {
1543
+ unresolvedForwardNudgeTimer = undefined;
1544
+ if (unresolvedForwardNudgeRunning) return; // an earlier pass is in flight — tick covers
1545
+ unresolvedForwardNudgeRunning = true;
1546
+ void retryUnresolvedDelegateForwards(components)
1547
+ .catch((e: any) => LOG.warn('MeshReconcile', `Nudged unresolved-forward retry failed: ${e?.message || e}`))
1548
+ .finally(() => { unresolvedForwardNudgeRunning = false; });
1549
+ }, UNRESOLVED_FORWARD_NUDGE_DELAY_MS);
1550
+ // Never keep the process alive solely for a pending nudge.
1551
+ if (typeof unresolvedForwardNudgeTimer.unref === 'function') unresolvedForwardNudgeTimer.unref();
1552
+ }
1553
+
1554
+ function clearUnresolvedForwardNudge(): void {
1555
+ if (unresolvedForwardNudgeTimer) {
1556
+ clearTimeout(unresolvedForwardNudgeTimer);
1557
+ unresolvedForwardNudgeTimer = undefined;
1558
+ }
1559
+ }
1560
+
1561
+ async function retryUnresolvedDelegateForwards(components: DaemonComponents): Promise<void> {
1562
+ const dispatchMeshCommand = components.dispatchMeshCommand;
1563
+ if (!dispatchMeshCommand) return;
1564
+
1565
+ // Drop entries that have exhausted their retry budget (fail-loud inside).
1566
+ expireStaleUnresolvedDelegateForwards();
1567
+
1568
+ const entries = peekUnresolvedDelegateForwards();
1569
+ if (entries.length === 0) {
1570
+ // Nothing queued — clear any stale per-entry rejection counters so the map can't grow.
1571
+ if (unresolvedForwardRejectionCounts.size > 0) unresolvedForwardRejectionCounts.clear();
1572
+ return;
1573
+ }
1574
+
1575
+ // Every id-form THIS daemon answers to. A self-addressed outbox entry (coordinator
1576
+ // == this daemon) must never be cross-dialled — see the self-route branch below.
1577
+ const selfIds = resolveCoordinatorDaemonIds(components);
1578
+ const isSelfCoordinatorId = (id: string): boolean =>
1579
+ selfIds.some(self => daemonIdsEquivalent(self, id));
1580
+
1581
+ for (const entry of entries) {
1582
+ // EVTTRACE correlation context for this outbox entry's retry.
1583
+ const entryTraceCtx = {
1584
+ taskId: (entry.payload as Record<string, unknown>).taskId,
1585
+ sessionId: readNonEmptyString(entry.payload.targetSessionId) || readNonEmptyString(entry.payload.sessionId),
1586
+ nodeId: readNonEmptyString(entry.payload.nodeId),
1587
+ event: readNonEmptyString(entry.payload.event),
1588
+ };
1589
+
1590
+ // Self-addressed forward: the coordinator daemon this entry targets IS this
1591
+ // daemon (a self-coordinating / single-node mesh, or a delegate whose coordinator
1592
+ // anchor resolved to our own id). A cross-daemon mesh_forward_event to our own id
1593
+ // is REFUSED by the dispatch self-dial guard ("Refusing to send ... to this
1594
+ // daemon's own id; route via the local router instead") on every retry, so the
1595
+ // entry can never be acked and loops forever (~every tick), spamming the log and
1596
+ // pinning the outbox row permanently undrained. Honour the guard's own advice:
1597
+ // route the event straight through the local receiver (handleMeshForwardEvent —
1598
+ // the same path the coordinator runs on receiving a remote push), then ack it.
1599
+ // We drain regardless of the local result: a cross-daemon dispatch could not have
1600
+ // resolved it either (the guard rejects before the receiver ever runs), so leaving
1601
+ // it queued only re-spams. handleMeshForwardEvent has the BEST recovery chance —
1602
+ // this daemon hosts the mesh, so its workspace/nodeId → meshId recovery applies.
1603
+ if (isSelfCoordinatorId(entry.coordinatorDaemonId)) {
1604
+ let localResult: any;
1605
+ try {
1606
+ traceMeshEventStage('forward_send', entryTraceCtx, `self → local router (${entry.coordinatorDaemonId})`);
1607
+ localResult = handleMeshForwardEvent(components, entry.payload);
1608
+ } catch (e: any) {
1609
+ LOG.warn('MeshReconcile', `Local route of self-addressed forward to ${entry.coordinatorDaemonId} threw: ${e?.message || e} — draining anyway to break the retry loop`);
1610
+ }
1611
+ ackUnresolvedDelegateForward(entry.id);
1612
+ unresolvedForwardRejectionCounts.delete(entry.id);
1613
+ if (localResult && localResult.success === false) {
1614
+ LOG.warn('MeshReconcile', `Self-addressed unresolved-delegate ${readNonEmptyString(entry.payload.event)} rejected by local router (${readNonEmptyString(localResult.error) || 'no reason'}) — drained to break the self-forward retry loop`);
1615
+ traceMeshEventDrop('self_forward_local_rejected', entryTraceCtx, readNonEmptyString(localResult.error) || 'no reason');
1616
+ } else {
1617
+ LOG.info('MeshReconcile', `Self-addressed unresolved-delegate ${readNonEmptyString(entry.payload.event)} routed via local router (coordinator ${entry.coordinatorDaemonId} is self) — drained`);
1618
+ }
1619
+ continue;
1620
+ }
1621
+
1622
+ // RECONCILE-MESHID-DROP: the stored forward payload was built when the worker
1623
+ // "couldn't resolve" its meshId, so the coordinator rejects it "meshId required"
1624
+ // when its own workspace/nodeId recovery misses. The worker can usually resolve it
1625
+ // now (member node membership / live-session meshNodeFor) — stamp it on so the
1626
+ // coordinator accepts. Covers entries persisted before this fix AND late-bound
1627
+ // sessions. No-op when the payload already carries a meshId or none is resolvable.
1628
+ let pushPayload = entry.payload;
1629
+ if (!readNonEmptyString(pushPayload.meshId)) {
1630
+ const recoveredMeshId = resolveForwardEventMeshId(components, pushPayload);
1631
+ if (recoveredMeshId) {
1632
+ pushPayload = { ...pushPayload, meshId: recoveredMeshId };
1633
+ traceMeshEventStage('forward_meshid_recovered', entryTraceCtx, `meshId=${recoveredMeshId}`);
1634
+ }
1635
+ }
1636
+
1637
+ let result: any;
1638
+ try {
1639
+ traceMeshEventStage('forward_send', entryTraceCtx, `retry → ${entry.coordinatorDaemonId}`);
1640
+ result = await dispatchMeshCommand(entry.coordinatorDaemonId, 'mesh_forward_event', pushPayload);
1641
+ } catch (e: any) {
1642
+ // Coordinator unreachable (transport threw) — keep the entry queued and try again
1643
+ // next tick. This is NOT a hard rejection, so it does not count toward the cap;
1644
+ // the age expiry bounds a permanently-offline coordinator.
1645
+ LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} failed: ${e?.message || e} — left queued`);
1646
+ traceMeshEventDrop('retry_forward_failed', entryTraceCtx, e?.message || String(e));
1647
+ continue;
1648
+ }
1649
+ if (result && result.success === false) {
1650
+ // Hard rejection: the push was delivered and deterministically refused. Retrying
1651
+ // the identical payload can never succeed, so bound it — after MAX_FORWARD_REJECTIONS
1652
+ // drop (drain) the entry with one fail-loud warning instead of re-spamming every tick.
1653
+ const rejections = (unresolvedForwardRejectionCounts.get(entry.id) || 0) + 1;
1654
+ unresolvedForwardRejectionCounts.set(entry.id, rejections);
1655
+ const reason = readNonEmptyString(result.error) || 'no reason';
1656
+ if (rejections >= MAX_FORWARD_REJECTIONS) {
1657
+ ackUnresolvedDelegateForward(entry.id);
1658
+ unresolvedForwardRejectionCounts.delete(entry.id);
1659
+ LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected ${rejections}x (${reason}) — dropping unresolved-delegate ${readNonEmptyString(entry.payload.event)} (sess=${readNonEmptyString(entry.payload.targetSessionId) || readNonEmptyString(entry.payload.sessionId) || '-'}) to stop the retry loop`);
1660
+ traceMeshEventDrop('retry_forward_exhausted', entryTraceCtx, `${reason} (${rejections} rejections)`);
1661
+ } else {
1662
+ LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected (${reason}) — left queued (attempt ${rejections}/${MAX_FORWARD_REJECTIONS})`);
1663
+ traceMeshEventDrop('retry_forward_rejected', entryTraceCtx, reason);
1664
+ }
1665
+ continue;
1666
+ }
1667
+ // Acked — mark the durable copy delivered.
1668
+ ackUnresolvedDelegateForward(entry.id);
1669
+ unresolvedForwardRejectionCounts.delete(entry.id);
1670
+ LOG.info('MeshReconcile', `Retried+delivered unresolved-delegate ${readNonEmptyString(entry.payload.event)} to coordinator ${entry.coordinatorDaemonId}`);
1671
+ }
1672
+ }
1673
+
1674
+ // The remote P2P pull helpers (pullRemoteNodeQueues + payload/envelope utilities)
1675
+ // live in ./mesh-remote-event-pull.ts, and the PHASE-4 completion-synthesis /
1676
+ // PHASE-5 auto-prune (reconcileUnterminatedDirectDispatches,
1677
+ // autoPruneStaleDirectDispatches) live in ./mesh-completion-synthesis.ts
1678
+ // (A-3 extraction). Both are imported at the top of this file.
1679
+ interface ReconcileLoopHandle {
1680
+ stop(): void;
1681
+ }
1682
+
1683
+ // Start the periodic reconcile loop. Returns a handle with stop() for shutdown.
1684
+ export function setupMeshReconcileLoop(components: DaemonComponents): ReconcileLoopHandle {
1685
+ const intervalMs = resolveReconcileIntervalMs();
1686
+ let running = false;
1687
+ const timer = setInterval(() => {
1688
+ if (running) return; // never overlap ticks
1689
+ running = true;
1690
+ void runMeshReconcileTick(components)
1691
+ .catch((e: any) => LOG.warn('MeshReconcile', `Reconcile tick error: ${e?.message || e}`))
1692
+ .finally(() => { running = false; });
1693
+ }, intervalMs);
1694
+ // Don't keep the process alive solely for this timer.
1695
+ if (typeof timer.unref === 'function') timer.unref();
1696
+ // Register the unresolved-forward nudge handler: the enqueue site
1697
+ // (forwardUnresolvedDelegateEvent) fires it after persisting an outbox row so
1698
+ // the PHASE 0 retry runs early instead of waiting for the next periodic tick.
1699
+ registerUnresolvedForwardRetryNudge(() => scheduleUnresolvedForwardNudge(components));
1700
+ LOG.info('MeshReconcile', `Mesh reconcile loop started (interval ${intervalMs}ms)`);
1701
+ return {
1702
+ stop() {
1703
+ clearInterval(timer);
1704
+ registerUnresolvedForwardRetryNudge(undefined);
1705
+ clearUnresolvedForwardNudge();
1706
+ LOG.info('MeshReconcile', 'Mesh reconcile loop stopped');
1707
+ },
1708
+ };
1709
+ }