@adhdev/daemon-core 0.9.82-rc.46 → 0.9.82-rc.460

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (439) hide show
  1. package/dist/boot/daemon-lifecycle.d.ts +9 -0
  2. package/dist/boot/process-hardening.d.ts +50 -0
  3. package/dist/build-info.d.ts +37 -0
  4. package/dist/chat/source-machine.d.ts +166 -0
  5. package/dist/chat/source-resolver.d.ts +104 -0
  6. package/dist/chat/subscription-updates.d.ts +1 -0
  7. package/dist/cli-adapter-types.d.ts +68 -2
  8. package/dist/cli-adapters/cli-script-runner.d.ts +117 -0
  9. package/dist/cli-adapters/cli-state-engine.d.ts +200 -0
  10. package/dist/cli-adapters/provider-cli-adapter.d.ts +126 -64
  11. package/dist/cli-adapters/provider-cli-parse.d.ts +5 -0
  12. package/dist/cli-adapters/provider-cli-runtime.d.ts +34 -0
  13. package/dist/cli-adapters/provider-cli-shared.d.ts +74 -0
  14. package/dist/cli-adapters/pty-write-chunking.d.ts +34 -0
  15. package/dist/cli-adapters/raw-terminal-io.d.ts +37 -0
  16. package/dist/cli-adapters/resolve-executable.d.ts +69 -0
  17. package/dist/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +6 -3
  18. package/dist/cli-adapters/terminal-backends/types.d.ts +10 -2
  19. package/dist/cli-adapters/terminal-screen.d.ts +4 -11
  20. package/dist/commands/chat-commands-debug-bundle.d.ts +14 -0
  21. package/dist/commands/chat-commands-read.d.ts +7 -0
  22. package/dist/commands/chat-commands-scope.d.ts +39 -0
  23. package/dist/commands/chat-commands-shared.d.ts +33 -0
  24. package/dist/commands/chat-commands-write.d.ts +14 -0
  25. package/dist/commands/chat-commands.d.ts +10 -23
  26. package/dist/commands/cli-manager.d.ts +25 -1
  27. package/dist/commands/handler.d.ts +125 -0
  28. package/dist/commands/high-family/index.d.ts +3 -0
  29. package/dist/commands/high-family/mesh-coordinator-launch.d.ts +2 -0
  30. package/dist/commands/high-family/mesh-events.d.ts +2 -0
  31. package/dist/commands/high-family/mesh-status.d.ts +2 -0
  32. package/dist/commands/high-family/types.d.ts +64 -0
  33. package/dist/commands/low-family/coordinator-prompt.d.ts +9 -0
  34. package/dist/commands/low-family/daemon-lifecycle.d.ts +2 -0
  35. package/dist/commands/low-family/diagnostics.d.ts +2 -0
  36. package/dist/commands/low-family/index.d.ts +3 -0
  37. package/dist/commands/low-family/mesh-ledger.d.ts +10 -0
  38. package/dist/commands/low-family/mesh-node-logs.d.ts +2 -0
  39. package/dist/commands/low-family/notification.d.ts +2 -0
  40. package/dist/commands/low-family/refine-config.d.ts +2 -0
  41. package/dist/commands/low-family/session-host.d.ts +2 -0
  42. package/dist/commands/low-family/spec-providerdev.d.ts +11 -0
  43. package/dist/commands/low-family/status-meta.d.ts +2 -0
  44. package/dist/commands/low-family/types.d.ts +33 -0
  45. package/dist/commands/med-family/cli-agent.d.ts +2 -0
  46. package/dist/commands/med-family/fast-forward.d.ts +2 -0
  47. package/dist/commands/med-family/ide.d.ts +10 -0
  48. package/dist/commands/med-family/index.d.ts +3 -0
  49. package/dist/commands/med-family/mesh-crud.d.ts +48 -0
  50. package/dist/commands/med-family/mesh-host-pairing.d.ts +2 -0
  51. package/dist/commands/med-family/mesh-queue.d.ts +2 -0
  52. package/dist/commands/med-family/mesh-restart.d.ts +2 -0
  53. package/dist/commands/med-family/types.d.ts +152 -0
  54. package/dist/commands/mesh-coordinator.d.ts +85 -1
  55. package/dist/commands/router-refine.d.ts +137 -0
  56. package/dist/commands/router-worktree-cleanup.d.ts +133 -0
  57. package/dist/commands/router.d.ts +241 -12
  58. package/dist/commands/upgrade-helper.d.ts +41 -1
  59. package/dist/config/chat-history.d.ts +9 -0
  60. package/dist/config/config.d.ts +19 -0
  61. package/dist/config/mesh-config.d.ts +129 -1
  62. package/dist/config/mesh-json-config.d.ts +199 -0
  63. package/dist/config/registry-resolver.d.ts +54 -0
  64. package/dist/config/repo-settings.d.ts +77 -0
  65. package/dist/daemon/dev-server.d.ts +0 -2
  66. package/dist/detection/ide-detector.d.ts +13 -0
  67. package/dist/detection/win32-ide-version.d.ts +37 -0
  68. package/dist/git/change-impact-config.d.ts +159 -0
  69. package/dist/git/git-commands.d.ts +12 -1
  70. package/dist/git/git-diff.d.ts +6 -0
  71. package/dist/git/git-executor.d.ts +11 -0
  72. package/dist/git/git-status.d.ts +57 -0
  73. package/dist/git/git-types.d.ts +2 -50
  74. package/dist/git/git-worktree.d.ts +71 -1
  75. package/dist/git/index.d.ts +3 -1
  76. package/dist/index.d.ts +67 -15
  77. package/dist/index.js +57943 -25340
  78. package/dist/index.js.map +1 -1
  79. package/dist/index.mjs +57611 -25174
  80. package/dist/index.mjs.map +1 -1
  81. package/dist/installer.d.ts +1 -4
  82. package/dist/ipc/local-ipc-server.d.ts +91 -0
  83. package/dist/launch.d.ts +1 -1
  84. package/dist/logging/async-batch-writer.d.ts +10 -0
  85. package/dist/logging/debug-config.d.ts +16 -0
  86. package/dist/logging/log-redactor.d.ts +24 -0
  87. package/dist/logging/log-tail-reader.d.ts +81 -0
  88. package/dist/logging/logger.d.ts +1 -1
  89. package/dist/mesh/contracts.d.ts +230 -0
  90. package/dist/mesh/coordinator-prompt.d.ts +76 -0
  91. package/dist/mesh/coordinator-registry.d.ts +59 -0
  92. package/dist/mesh/mesh-active-work.d.ts +174 -0
  93. package/dist/mesh/mesh-clone-grace.d.ts +8 -0
  94. package/dist/mesh/mesh-coordinator-config.d.ts +21 -0
  95. package/dist/mesh/mesh-delivery-policy.d.ts +131 -0
  96. package/dist/mesh/mesh-event-classify.d.ts +7 -0
  97. package/dist/mesh/mesh-event-forwarding.d.ts +41 -0
  98. package/dist/mesh/mesh-event-trace.d.ts +21 -0
  99. package/dist/mesh/mesh-events-coordinator.d.ts +4 -0
  100. package/dist/mesh/mesh-events-pending.d.ts +113 -0
  101. package/dist/mesh/mesh-events-stale.d.ts +61 -0
  102. package/dist/mesh/mesh-events-utils.d.ts +75 -0
  103. package/dist/mesh/mesh-events.d.ts +6 -49
  104. package/dist/mesh/mesh-fast-forward.d.ts +81 -0
  105. package/dist/mesh/mesh-host-ownership.d.ts +29 -0
  106. package/dist/mesh/mesh-init.d.ts +128 -0
  107. package/dist/mesh/mesh-ledger-reconciliation.d.ts +23 -4
  108. package/dist/mesh/mesh-ledger.d.ts +133 -1
  109. package/dist/mesh/mesh-magi-status.d.ts +63 -0
  110. package/dist/mesh/mesh-missions.d.ts +189 -0
  111. package/dist/mesh/mesh-node-identity.d.ts +306 -0
  112. package/dist/mesh/mesh-queue-assignment.d.ts +158 -0
  113. package/dist/mesh/mesh-reconcile-loop.d.ts +80 -0
  114. package/dist/mesh/mesh-refine-batch.d.ts +68 -0
  115. package/dist/mesh/mesh-refine-gates.d.ts +428 -0
  116. package/dist/mesh/mesh-refine-status.d.ts +62 -0
  117. package/dist/mesh/mesh-review-inbox.d.ts +90 -0
  118. package/dist/mesh/mesh-routing.d.ts +70 -0
  119. package/dist/mesh/mesh-runtime-store.d.ts +476 -0
  120. package/dist/mesh/mesh-scheduling-runtime.d.ts +78 -0
  121. package/dist/mesh/mesh-task-inflight.d.ts +46 -0
  122. package/dist/mesh/mesh-task-stats.d.ts +49 -0
  123. package/dist/mesh/mesh-unresolved-forward-outbox.d.ts +30 -0
  124. package/dist/mesh/mesh-warmup-deadline.d.ts +68 -0
  125. package/dist/mesh/mesh-work-queue.d.ts +318 -5
  126. package/dist/mesh/preview-freshness.d.ts +28 -0
  127. package/dist/mesh/refine-config.d.ts +216 -0
  128. package/dist/mesh/worktree-bootstrap-config.d.ts +219 -0
  129. package/dist/providers/acp-provider-instance.d.ts +5 -0
  130. package/dist/providers/approval-utils.d.ts +45 -0
  131. package/dist/providers/chat-message-normalization.d.ts +58 -1
  132. package/dist/providers/cli-provider-instance.d.ts +290 -3
  133. package/dist/providers/contracts.d.ts +149 -6
  134. package/dist/providers/external-sources.d.ts +71 -0
  135. package/dist/providers/manual-attendance.d.ts +79 -0
  136. package/dist/providers/native-history/antigravity-claim-registry.d.ts +28 -0
  137. package/dist/providers/native-history/antigravity-cli-transcript.d.ts +137 -0
  138. package/dist/providers/native-history/claude-cli-transcript.d.ts +70 -0
  139. package/dist/providers/native-history/codex-cli-transcript.d.ts +73 -0
  140. package/dist/providers/native-history/constants.d.ts +12 -0
  141. package/dist/providers/native-history/dispatcher.d.ts +31 -0
  142. package/dist/providers/native-history/hermes-cli-transcript.d.ts +30 -0
  143. package/dist/providers/native-history/index.d.ts +13 -0
  144. package/dist/providers/provider-instance-manager.d.ts +29 -0
  145. package/dist/providers/provider-instance.d.ts +30 -1
  146. package/dist/providers/provider-loader.d.ts +42 -5
  147. package/dist/providers/provider-trust.d.ts +31 -0
  148. package/dist/providers/read-chat-contract.d.ts +29 -0
  149. package/dist/providers/sdk/v1/builders/acp/detect-status.d.ts +68 -0
  150. package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +86 -0
  151. package/dist/providers/sdk/v1/builders/cli/parse-approval-squash.d.ts +59 -0
  152. package/dist/providers/sdk/v1/builders/cli/parse-approval.d.ts +65 -0
  153. package/dist/providers/sdk/v1/builders/cli/parse-session.d.ts +91 -0
  154. package/dist/providers/sdk/v1/builders/cli/visible-region.d.ts +42 -0
  155. package/dist/providers/sdk/v1/fixture-tooling/format.d.ts +126 -0
  156. package/dist/providers/sdk/v1/fixture-tooling/index.d.ts +8 -0
  157. package/dist/providers/sdk/v1/fixture-tooling/replay.d.ts +38 -0
  158. package/dist/providers/sdk/v1/index.d.ts +30 -0
  159. package/dist/providers/sdk/v1/sandbox/README-design.d.ts +193 -0
  160. package/dist/providers/sdk/v1/sandbox/require-whitelist.d.ts +74 -0
  161. package/dist/providers/sdk/v1/sandbox/script-runner.d.ts +98 -0
  162. package/dist/providers/sdk/v1/types/cli/index.d.ts +277 -0
  163. package/dist/providers/sdk/v1/types/common/index.d.ts +203 -0
  164. package/dist/providers/sdk/v1/validators/index.d.ts +5 -0
  165. package/dist/providers/sdk/v1/validators/manifest.d.ts +40 -0
  166. package/dist/providers/sdk/v1/validators/taint.d.ts +52 -0
  167. package/dist/providers/spec/adapter.d.ts +98 -0
  168. package/dist/providers/spec/cli-adapter.d.ts +223 -0
  169. package/dist/providers/spec/evaluator.d.ts +45 -0
  170. package/dist/providers/spec/fsm-driver.d.ts +449 -0
  171. package/dist/providers/spec/fsm-evaluator.d.ts +93 -0
  172. package/dist/providers/spec/fsm-loader.d.ts +14 -0
  173. package/dist/providers/spec/fsm-types.d.ts +202 -0
  174. package/dist/providers/spec/native-history-executor.d.ts +41 -0
  175. package/dist/providers/spec/pre-launch-trust.d.ts +16 -0
  176. package/dist/providers/spec/route.d.ts +4 -0
  177. package/dist/providers/spec/types.d.ts +222 -0
  178. package/dist/providers/status-monitor.d.ts +7 -7
  179. package/dist/providers/transcript-v2.d.ts +176 -0
  180. package/dist/providers/types/interactive-prompt.d.ts +84 -0
  181. package/dist/providers/working-dir.d.ts +17 -0
  182. package/dist/repo-mesh-types.d.ts +519 -13
  183. package/dist/runtime-defaults.d.ts +2 -0
  184. package/dist/session-host/managed-host.d.ts +64 -0
  185. package/dist/session-host/runtime-surface.d.ts +10 -16
  186. package/dist/sessions/registry.d.ts +6 -0
  187. package/dist/shared-types-extra.d.ts +2 -4
  188. package/dist/shared-types.d.ts +86 -55
  189. package/dist/status/normalize.d.ts +1 -1
  190. package/dist/status/normalize.js +1 -0
  191. package/dist/status/normalize.js.map +1 -1
  192. package/dist/status/normalize.mjs +1 -0
  193. package/dist/status/normalize.mjs.map +1 -1
  194. package/dist/status/reporter.d.ts +2 -0
  195. package/dist/status/snapshot.d.ts +26 -0
  196. package/dist/system/hash.d.ts +8 -0
  197. package/dist/system/load-better-sqlite3.d.ts +21 -0
  198. package/dist/types.d.ts +5 -0
  199. package/package.json +11 -4
  200. package/src/agent-stream/poller.ts +2 -3
  201. package/src/agent-stream/provider-adapter.ts +1 -1
  202. package/src/boot/daemon-lifecycle.ts +65 -12
  203. package/src/boot/process-hardening.ts +89 -0
  204. package/src/build-info.ts +73 -0
  205. package/src/chat/source-machine.ts +534 -0
  206. package/src/chat/source-resolver.ts +0 -0
  207. package/src/chat/subscription-updates.ts +20 -1
  208. package/src/cli-adapter-types.d.ts +3 -1
  209. package/src/cli-adapter-types.ts +68 -2
  210. package/src/cli-adapters/cli-script-runner.ts +421 -0
  211. package/src/cli-adapters/cli-state-engine.ts +1246 -0
  212. package/src/cli-adapters/provider-cli-adapter.d.ts +1 -1
  213. package/src/cli-adapters/provider-cli-adapter.ts +914 -1144
  214. package/src/cli-adapters/provider-cli-parse.d.ts +2 -0
  215. package/src/cli-adapters/provider-cli-parse.ts +19 -0
  216. package/src/cli-adapters/provider-cli-runtime.ts +56 -7
  217. package/src/cli-adapters/provider-cli-shared.d.ts +4 -0
  218. package/src/cli-adapters/provider-cli-shared.ts +121 -11
  219. package/src/cli-adapters/pty-transport.ts +2 -1
  220. package/src/cli-adapters/pty-write-chunking.ts +106 -0
  221. package/src/cli-adapters/raw-terminal-io.ts +252 -0
  222. package/src/cli-adapters/resolve-executable.ts +204 -0
  223. package/src/cli-adapters/session-host-transport.ts +2 -1
  224. package/src/cli-adapters/terminal-backends/ghostty-vt-backend.ts +63 -29
  225. package/src/cli-adapters/terminal-backends/types.ts +10 -3
  226. package/src/cli-adapters/terminal-screen.ts +16 -81
  227. package/src/commands/chat-commands-debug-bundle.ts +398 -0
  228. package/src/commands/chat-commands-read.ts +2488 -0
  229. package/src/commands/chat-commands-scope.ts +54 -0
  230. package/src/commands/chat-commands-shared.ts +114 -0
  231. package/src/commands/chat-commands-write.ts +891 -0
  232. package/src/commands/chat-commands.ts +19 -1841
  233. package/src/commands/cli-manager.ts +596 -27
  234. package/src/commands/handler.ts +845 -2
  235. package/src/commands/high-family/index.ts +28 -0
  236. package/src/commands/high-family/mesh-coordinator-launch.ts +680 -0
  237. package/src/commands/high-family/mesh-events.ts +89 -0
  238. package/src/commands/high-family/mesh-status.ts +822 -0
  239. package/src/commands/high-family/types.ts +80 -0
  240. package/src/commands/low-family/coordinator-prompt.ts +72 -0
  241. package/src/commands/low-family/daemon-lifecycle.ts +120 -0
  242. package/src/commands/low-family/diagnostics.ts +57 -0
  243. package/src/commands/low-family/index.ts +37 -0
  244. package/src/commands/low-family/mesh-ledger.ts +62 -0
  245. package/src/commands/low-family/mesh-node-logs.ts +87 -0
  246. package/src/commands/low-family/notification.ts +116 -0
  247. package/src/commands/low-family/refine-config.ts +106 -0
  248. package/src/commands/low-family/session-host.ts +274 -0
  249. package/src/commands/low-family/spec-providerdev.ts +217 -0
  250. package/src/commands/low-family/status-meta.ts +112 -0
  251. package/src/commands/low-family/types.ts +39 -0
  252. package/src/commands/med-family/cli-agent.ts +241 -0
  253. package/src/commands/med-family/fast-forward.ts +230 -0
  254. package/src/commands/med-family/ide.ts +163 -0
  255. package/src/commands/med-family/index.ts +37 -0
  256. package/src/commands/med-family/mesh-crud.ts +1287 -0
  257. package/src/commands/med-family/mesh-host-pairing.ts +234 -0
  258. package/src/commands/med-family/mesh-queue.ts +167 -0
  259. package/src/commands/med-family/mesh-restart.ts +92 -0
  260. package/src/commands/med-family/types.ts +153 -0
  261. package/src/commands/mesh-coordinator.ts +334 -124
  262. package/src/commands/router-refine.ts +1711 -0
  263. package/src/commands/router-worktree-cleanup.ts +870 -0
  264. package/src/commands/router.ts +1025 -4070
  265. package/src/commands/stream-commands.ts +8 -0
  266. package/src/commands/upgrade-helper.ts +310 -45
  267. package/src/config/chat-history.ts +493 -24
  268. package/src/config/config.ts +30 -0
  269. package/src/config/mesh-config.ts +619 -23
  270. package/src/config/mesh-json-config.ts +376 -0
  271. package/src/config/recent-activity.ts +8 -2
  272. package/src/config/registry-resolver.ts +100 -0
  273. package/src/config/repo-settings.ts +111 -0
  274. package/src/daemon/dev-auto-implement.ts +3 -2
  275. package/src/daemon/dev-cli-debug.ts +10 -1
  276. package/src/daemon/dev-server.ts +0 -541
  277. package/src/detection/cli-detector.ts +28 -9
  278. package/src/detection/ide-detector.ts +55 -16
  279. package/src/detection/win32-ide-version.ts +106 -0
  280. package/src/git/change-impact-config.ts +354 -0
  281. package/src/git/git-commands.ts +78 -15
  282. package/src/git/git-diff.ts +81 -11
  283. package/src/git/git-executor.ts +12 -0
  284. package/src/git/git-status.ts +767 -48
  285. package/src/git/git-types.ts +14 -62
  286. package/src/git/git-worktree.ts +261 -4
  287. package/src/git/index.ts +17 -0
  288. package/src/index.ts +200 -12
  289. package/src/installer.d.ts +1 -1
  290. package/src/installer.ts +8 -6
  291. package/src/ipc/local-ipc-server.ts +278 -0
  292. package/src/launch.d.ts +1 -1
  293. package/src/launch.ts +37 -28
  294. package/src/logging/async-batch-writer.ts +55 -0
  295. package/src/logging/command-log.ts +7 -5
  296. package/src/logging/debug-config.ts +25 -0
  297. package/src/logging/debug-trace.ts +7 -2
  298. package/src/logging/log-redactor.ts +100 -0
  299. package/src/logging/log-tail-reader.ts +341 -0
  300. package/src/logging/logger.ts +14 -7
  301. package/src/mesh/contracts.ts +469 -0
  302. package/src/mesh/coordinator-prompt.ts +536 -34
  303. package/src/mesh/coordinator-registry.ts +121 -0
  304. package/src/mesh/mesh-active-work.ts +656 -0
  305. package/src/mesh/mesh-clone-grace.ts +68 -0
  306. package/src/mesh/mesh-coordinator-config.ts +97 -0
  307. package/src/mesh/mesh-delivery-policy.ts +315 -0
  308. package/src/mesh/mesh-event-classify.ts +73 -0
  309. package/src/mesh/mesh-event-forwarding.ts +1932 -0
  310. package/src/mesh/mesh-event-trace.ts +67 -0
  311. package/src/mesh/mesh-events-coordinator.ts +32 -0
  312. package/src/mesh/mesh-events-pending.ts +766 -0
  313. package/src/mesh/mesh-events-stale.ts +440 -0
  314. package/src/mesh/mesh-events-utils.ts +443 -0
  315. package/src/mesh/mesh-events.ts +33 -1035
  316. package/src/mesh/mesh-fast-forward.ts +856 -0
  317. package/src/mesh/mesh-host-ownership.ts +111 -0
  318. package/src/mesh/mesh-init.ts +350 -0
  319. package/src/mesh/mesh-ledger-reconciliation.ts +12 -7
  320. package/src/mesh/mesh-ledger.ts +873 -102
  321. package/src/mesh/mesh-magi-status.ts +223 -0
  322. package/src/mesh/mesh-missions.ts +449 -0
  323. package/src/mesh/mesh-node-identity.ts +1994 -0
  324. package/src/mesh/mesh-queue-assignment.ts +2233 -0
  325. package/src/mesh/mesh-reconcile-loop.ts +2390 -0
  326. package/src/mesh/mesh-refine-batch.ts +205 -0
  327. package/src/mesh/mesh-refine-gates.ts +1686 -0
  328. package/src/mesh/mesh-refine-status.ts +231 -0
  329. package/src/mesh/mesh-review-inbox.ts +308 -0
  330. package/src/mesh/mesh-routing.ts +291 -0
  331. package/src/mesh/mesh-runtime-store.ts +2026 -0
  332. package/src/mesh/mesh-scheduling-runtime.ts +198 -0
  333. package/src/mesh/mesh-task-inflight.ts +70 -0
  334. package/src/mesh/mesh-task-stats.ts +161 -0
  335. package/src/mesh/mesh-unresolved-forward-outbox.ts +185 -0
  336. package/src/mesh/mesh-warmup-deadline.ts +152 -0
  337. package/src/mesh/mesh-work-queue.ts +1281 -141
  338. package/src/mesh/preview-freshness.ts +163 -0
  339. package/src/mesh/refine-config.ts +423 -0
  340. package/src/mesh/worktree-bootstrap-config.ts +599 -0
  341. package/src/providers/acp-provider-instance.ts +43 -10
  342. package/src/providers/approval-utils.d.ts +5 -0
  343. package/src/providers/approval-utils.ts +99 -5
  344. package/src/providers/chat-message-normalization.ts +126 -4
  345. package/src/providers/cli-provider-instance.ts +2456 -107
  346. package/src/providers/contracts.d.ts +55 -0
  347. package/src/providers/contracts.ts +160 -6
  348. package/src/providers/extension-provider-instance.ts +12 -7
  349. package/src/providers/external-sources.ts +218 -0
  350. package/src/providers/ide-provider-instance.ts +35 -12
  351. package/src/providers/manual-attendance.ts +105 -0
  352. package/src/providers/native-history/antigravity-claim-registry.ts +131 -0
  353. package/src/providers/native-history/antigravity-cli-transcript.ts +1193 -0
  354. package/src/providers/native-history/claude-cli-transcript.ts +396 -0
  355. package/src/providers/native-history/codex-cli-transcript.ts +479 -0
  356. package/src/providers/native-history/constants.ts +19 -0
  357. package/src/providers/native-history/dispatcher.ts +503 -0
  358. package/src/providers/native-history/hermes-cli-transcript.ts +260 -0
  359. package/src/providers/native-history/index.ts +30 -0
  360. package/src/providers/provider-instance-manager.ts +71 -0
  361. package/src/providers/provider-instance.ts +24 -1
  362. package/src/providers/provider-loader.ts +700 -55
  363. package/src/providers/provider-schema.ts +87 -14
  364. package/src/providers/provider-trust.ts +114 -0
  365. package/src/providers/read-chat-contract.ts +76 -16
  366. package/src/providers/sdk/README.md +49 -0
  367. package/src/providers/sdk/v1/builders/acp/detect-status.ts +144 -0
  368. package/src/providers/sdk/v1/builders/cli/detect-status.ts +313 -0
  369. package/src/providers/sdk/v1/builders/cli/parse-approval-squash.ts +158 -0
  370. package/src/providers/sdk/v1/builders/cli/parse-approval.ts +261 -0
  371. package/src/providers/sdk/v1/builders/cli/parse-session.ts +276 -0
  372. package/src/providers/sdk/v1/builders/cli/visible-region.ts +143 -0
  373. package/src/providers/sdk/v1/fixture-tooling/format.ts +130 -0
  374. package/src/providers/sdk/v1/fixture-tooling/index.ts +22 -0
  375. package/src/providers/sdk/v1/fixture-tooling/replay.ts +352 -0
  376. package/src/providers/sdk/v1/index.ts +152 -0
  377. package/src/providers/sdk/v1/sandbox/README-design.ts +195 -0
  378. package/src/providers/sdk/v1/sandbox/require-whitelist.ts +472 -0
  379. package/src/providers/sdk/v1/sandbox/script-runner.ts +150 -0
  380. package/src/providers/sdk/v1/schemas/cli/provider.schema.json +509 -0
  381. package/src/providers/sdk/v1/schemas/primitives/acp-session-protocol-v1.json +131 -0
  382. package/src/providers/sdk/v1/schemas/primitives/native-history-codex-rollout-v1.json +66 -0
  383. package/src/providers/sdk/v1/schemas/primitives/tui-approval-squash-v1.json +91 -0
  384. package/src/providers/sdk/v1/schemas/primitives/tui-assistant-block-v1.json +91 -0
  385. package/src/providers/sdk/v1/schemas/primitives/tui-cue-ordering-v1.json +47 -0
  386. package/src/providers/sdk/v1/schemas/primitives/tui-dispatch-order-v1.json +32 -0
  387. package/src/providers/sdk/v1/schemas/primitives/tui-footer-chrome-v1.json +42 -0
  388. package/src/providers/sdk/v1/schemas/primitives/tui-index-finder-v1.json +27 -0
  389. package/src/providers/sdk/v1/schemas/primitives/tui-modal-v1.json +125 -0
  390. package/src/providers/sdk/v1/schemas/primitives/tui-prompt-marker-v1.json +45 -0
  391. package/src/providers/sdk/v1/schemas/primitives/tui-session-id-extraction-v1.json +46 -0
  392. package/src/providers/sdk/v1/schemas/primitives/tui-settled-prompt-v1.json +71 -0
  393. package/src/providers/sdk/v1/schemas/primitives/tui-spinner-v1.json +83 -0
  394. package/src/providers/sdk/v1/schemas/primitives/tui-transcript-pty-v1.json +83 -0
  395. package/src/providers/sdk/v1/schemas/primitives/tui-visible-region-v1.json +57 -0
  396. package/src/providers/sdk/v1/schemas/primitives/tui-welcome-screen-v1.json +35 -0
  397. package/src/providers/sdk/v1/types/cli/index.ts +365 -0
  398. package/src/providers/sdk/v1/types/common/index.ts +229 -0
  399. package/src/providers/sdk/v1/validators/index.ts +19 -0
  400. package/src/providers/sdk/v1/validators/manifest.ts +110 -0
  401. package/src/providers/sdk/v1/validators/taint.ts +309 -0
  402. package/src/providers/spec/adapter.ts +246 -0
  403. package/src/providers/spec/cli-adapter.ts +1064 -0
  404. package/src/providers/spec/evaluator.ts +407 -0
  405. package/src/providers/spec/fsm-driver.ts +1528 -0
  406. package/src/providers/spec/fsm-evaluator.ts +290 -0
  407. package/src/providers/spec/fsm-loader.ts +128 -0
  408. package/src/providers/spec/fsm-types.ts +301 -0
  409. package/src/providers/spec/native-history-executor.ts +1174 -0
  410. package/src/providers/spec/pre-launch-trust.ts +104 -0
  411. package/src/providers/spec/route.ts +51 -0
  412. package/src/providers/spec/types.ts +262 -0
  413. package/src/providers/status-monitor.d.ts +7 -7
  414. package/src/providers/status-monitor.ts +37 -22
  415. package/src/providers/transcript-v2.ts +567 -0
  416. package/src/providers/types/interactive-prompt.ts +536 -0
  417. package/src/providers/version-archive.ts +64 -24
  418. package/src/providers/working-dir.ts +23 -0
  419. package/src/repo-mesh-types.ts +758 -14
  420. package/src/runtime-defaults.ts +39 -0
  421. package/src/session-host/managed-host.ts +218 -0
  422. package/src/session-host/runtime-surface.ts +20 -80
  423. package/src/sessions/registry.ts +6 -0
  424. package/src/shared-types-extra.ts +2 -4
  425. package/src/shared-types.d.ts +8 -0
  426. package/src/shared-types.ts +124 -55
  427. package/src/status/builders.ts +26 -6
  428. package/src/status/normalize.ts +2 -0
  429. package/src/status/reporter.ts +19 -1
  430. package/src/status/snapshot.ts +95 -26
  431. package/src/system/hash.ts +23 -0
  432. package/src/system/host-memory.ts +29 -12
  433. package/src/system/load-better-sqlite3.ts +68 -0
  434. package/src/types.ts +5 -0
  435. package/dist/cli-adapters/terminal-backends/xterm-backend.d.ts +0 -17
  436. package/dist/mesh/mesh-sync.d.ts +0 -53
  437. package/src/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +0 -16
  438. package/src/cli-adapters/terminal-backends/xterm-backend.ts +0 -97
  439. package/src/mesh/mesh-sync.ts +0 -111
@@ -0,0 +1,2390 @@
1
+ // ---------------------------------------------------------------------------
2
+ // mesh-reconcile-loop — periodic queue → live coordinator reconciliation
3
+ // ---------------------------------------------------------------------------
4
+ // Single-model replacement for the old event-based "spontaneous forward" paths
5
+ // (remote P2P mesh_forward_event dispatch + live-CLI PTY fire-and-forget inject).
6
+ // Those pushed events at the moment a worker transitioned state, and silently
7
+ // dropped on the network (P2P) or when the coordinator was generating.
8
+ //
9
+ // The reliable backbone has always been the pending-events queue (SQLite +
10
+ // JSONL): every mesh coordinator event is persisted there before anything else
11
+ // (see injectMeshSystemMessage). What was missing was an *active* drainer that
12
+ // runs on a schedule rather than only when the coordinator (an LLM) happens to
13
+ // call a mesh tool.
14
+ //
15
+ // This loop is that drainer. On a fixed interval it:
16
+ // 1. Finds live CLI coordinator sessions on THIS daemon (meshCoordinatorFor
17
+ // stamp). For each mesh, drains the local queue scoped to this daemon and
18
+ // injects pending events into the coordinator. When a coordinator is idle it
19
+ // receives every queued event. When ONLY generating coordinators exist (the
20
+ // common case while the coordinator is blocked awaiting a worker result), the
21
+ // loop force-drains ONLY the force-inject events (completion / approval / stop /
22
+ // refine·bootstrap terminal) and force-writes them into the generating PTY —
23
+ // the same busy-bypass send-guard escape the live-CLI inject used to use.
24
+ // Non-force progress events stay queued for the next idle tick (injecting them
25
+ // mid-generation would be noise). This is what makes a coordinator parked in
26
+ // `generating` while awaiting a worker's completion actually receive it.
27
+ // 2. In cloud mode (dispatchMeshCommand present), pulls each remote worker
28
+ // node daemon's queue over P2P (get_pending_mesh_events) and re-injects via
29
+ // handleMeshForwardEvent — the same pull the MCP drainCoordinatorPendingEvents
30
+ // already does, now driven by the daemon timer instead of an LLM tool call.
31
+ //
32
+ // IMPORTANT — limits of this loop:
33
+ // - It only delivers to *live CLI coordinator instances* on this daemon. A
34
+ // pure stdio MCP coordinator (an LLM with no live CLI session to inject
35
+ // into) has no inject target here; that case stays pull-driven — the LLM
36
+ // drains the queue when it calls mesh_status / mesh_read_chat. We do NOT try
37
+ // to "wake" an LLM from the daemon; that is structurally impossible over a
38
+ // stdio request/response transport. See docs/refactoring/2026-06-15-mesh-event-to-queue-polling.md §4.7.
39
+ // - Queue persistence (queuePendingMeshCoordinatorEvent) and the SQLite
40
+ // drained=1 idempotency are the trust backbone and are untouched by this loop.
41
+ // ---------------------------------------------------------------------------
42
+
43
+ import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
44
+ import type { LocalMeshEntry } from '../repo-mesh-types.js';
45
+ import { loadConfig } from '../config/config.js';
46
+ import { listMeshes } from '../config/mesh-config.js';
47
+ import { LOG, getLogLevel } from '../logging/logger.js';
48
+ import { drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, buildPendingEventFingerprint, queuePendingMeshCoordinatorEvent } from './mesh-events-pending.js';
49
+ import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
50
+ import { appendLedgerEntry } from './mesh-ledger.js';
51
+ import { MeshRuntimeStore } from './mesh-runtime-store.js';
52
+ import { handleMeshForwardEvent, shouldForceInjectMeshEvent, triggerMeshQueue, resolveForwardEventMeshId } from './mesh-events-coordinator.js';
53
+ import { isMeshApprovalEvent, MESH_APPROVAL_EVENTS } from './mesh-event-classify.js';
54
+ import {
55
+ peekUnresolvedDelegateForwards,
56
+ ackUnresolvedDelegateForward,
57
+ expireStaleUnresolvedDelegateForwards,
58
+ } from './mesh-unresolved-forward-outbox.js';
59
+ import { readNonEmptyString, readMeshCompletionSummary, buildMeshSystemMessage } from './mesh-events-utils.js';
60
+ import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
61
+ import { expandDaemonIdForms, daemonIdsEquivalent, sessionIdsEquivalent } from '@adhdev/mesh-shared';
62
+ import { getActiveDirectDispatches, getQueue, reclaimStrandedAssignedTask, updateTaskStatus } from './mesh-work-queue.js';
63
+ import { resolveSessionBusyVerdict } from './mesh-queue-assignment.js';
64
+ import { readLedgerEntries } from './mesh-ledger.js';
65
+ import type { MeshLedgerEntry } from './mesh-ledger.js';
66
+ import { pruneStaleDirectDispatches } from './mesh-active-work.js';
67
+ import { findTerminalLedgerEvidenceForTask, reconcileDirectDispatchCompletionFromTranscript } from './mesh-events-stale.js';
68
+ import { extractFinalAssistantSummaryEvidence } from '../providers/chat-message-normalization.js';
69
+ import type { ChatMessage } from '../types.js';
70
+
71
+ // Default reconcile cadence. approval/completion notifications to a live CLI
72
+ // coordinator land within at most one interval. Overridable via env for tuning.
73
+ const DEFAULT_RECONCILE_INTERVAL_MS = 4_000;
74
+
75
+ // PHASE 5 (auto-prune) conservative age gate. A direct dispatch whose node/session is
76
+ // orphaned (no longer in the live mesh) is only auto-pruned once it is at least this old,
77
+ // measured from its dispatch time. This protects against a node/session that is only
78
+ // *transiently* invisible (a momentary probe failure, a daemon restart) being pruned the
79
+ // instant it disappears. The MANUAL prune (mesh_prune_stale_direct) has no age gate — an
80
+ // operator pruning explicitly wants the orphan gone now. Overridable via env for tuning.
81
+ const DEFAULT_AUTO_PRUNE_MIN_AGE_MS = 24 * 60 * 60_000; // 24h
82
+
83
+ function resolveAutoPruneMinAgeMs(): number {
84
+ const raw = readNonEmptyString(process.env.MESH_AUTO_PRUNE_MIN_AGE_MS);
85
+ if (raw) {
86
+ const parsed = Number.parseInt(raw, 10);
87
+ // Clamp to [1h, 30d] so a mis-set env can't make the gate pathologically aggressive
88
+ // (prune the moment something blinks) or effectively disable it forever.
89
+ if (Number.isFinite(parsed) && parsed >= 60 * 60_000 && parsed <= 30 * 24 * 60 * 60_000) return parsed;
90
+ }
91
+ return DEFAULT_AUTO_PRUNE_MIN_AGE_MS;
92
+ }
93
+
94
+ // PTY-OVERTRUST-DRAIN (Defect B, fix B). Age-based escape for the
95
+ // `generating_no_idle_coordinator` hold. Fix A makes the drain predicate read the RAW
96
+ // adapter (mask-stripped), so the common mask-driven false-busy is gone. But a hold can
97
+ // still arise from a genuine status-source desync that fix A does not reach (e.g. the
98
+ // adapter raw itself momentarily reads generating while the coordinator is actually at a
99
+ // turn end). This is a TIME-BASED BACKSTOP: when a mesh's pending terminal events have
100
+ // been held this long, re-confirm the coordinator's RAW adapter idle on the tick and, if
101
+ // it is genuinely idle, drain ONCE. It NEVER injects into a genuinely-generating PTY —
102
+ // the re-confirmation gates on raw adapter idle, so the intentional removal of
103
+ // force-inject-into-generating (data-loss) is preserved. Default 12s = 3 reconcile ticks
104
+ // at the 4s cadence: long enough that a normal mid-turn settle is not pre-empted, short
105
+ // enough that a desync-stranded completion is not held for minutes. Env-tunable.
106
+ const DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS = 12_000;
107
+
108
+ function resolvePendingHeldDrainEscalateMs(): number {
109
+ // Floor 4s (one tick) so a mis-set env cannot make the escape race a normal settle;
110
+ // ceiling 5min so it cannot be disabled into a permanent strand.
111
+ return resolveTunedReconcileMs('MESH_PENDING_HELD_DRAIN_ESCALATE_MS', DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS, 4_000, 5 * 60_000);
112
+ }
113
+
114
+ function resolveReconcileIntervalMs(): number {
115
+ const raw = readNonEmptyString(process.env.MESH_RECONCILE_INTERVAL_MS);
116
+ if (raw) {
117
+ const parsed = Number.parseInt(raw, 10);
118
+ if (Number.isFinite(parsed) && parsed >= 1_000 && parsed <= 60_000) return parsed;
119
+ }
120
+ return DEFAULT_RECONCILE_INTERVAL_MS;
121
+ }
122
+
123
+ // R4f (GENERATING-BOUNDARY, acked-hold redesign). PHASE 4 only synthesizes a missing completion
124
+ // when the worker session reads `idle`. But a worker that is GENUINELY generating (it emitted
125
+ // agent:generating_started — the dispatch row is 'acked' — and has not yet completed) can
126
+ // momentarily read `idle` mid-turn (a CLI PTY inter-tool-call settle, or the final assistant text
127
+ // already rendered while the turn's generating_completed lifecycle close still lags). A premature
128
+ // synth writes a terminal that then masks the worker's REAL completion when it lands seconds later
129
+ // (drop:duplicate_completion_terminal_ledger; the observed 71s task a250fb44 lost its [System]
130
+ // notification this way; the R4e 53s task synth fired 16s BEFORE the worker's real emit).
131
+ //
132
+ // R4 → R4e used FINITE timers (consecutive ticks / MIN_IDLE_SETTLE / ACKED_TURN_SETTLE) to delay the
133
+ // synth. That class of fix is fundamentally a RACE: the worker's real emit latency is variable and
134
+ // unbounded (win32 idle reads can flip before the emit arrives), so ANY finite timer eventually
135
+ // loses to a slow-enough turn — and the synth pre-empts the real completion. R4e live-FAILED for
136
+ // exactly this reason.
137
+ //
138
+ // R4f redesign (direction B). An `acked` task means the worker ECHOED generating_started (the
139
+ // taskId flip) — it is alive and mid-turn, so it WILL eventually emit a real terminal. We therefore
140
+ // HOLD the synth INDEFINITELY for an acked task. This is safe against the emit actually arriving:
141
+ // when the worker's real generating_completed lands, it writes a terminal ledger, and
142
+ // reconcileDirectDispatchCompletionFromTranscript's hasTerminalLedgerAfterDispatch check makes any
143
+ // later synth an idempotent no-op (alreadyTerminal). So the hold never costs a missed notification —
144
+ // the real emit always wins, no matter how late.
145
+ //
146
+ // The indefinite hold is released ONLY by a genuine-DEATH / emit-loss BACKSTOP — never a finite
147
+ // timer that races normal lag:
148
+ // (a) liveness failure — read_chat reports the session is gone, OR N consecutive read failures
149
+ // accumulate (a transport/session-gone signal, counted as death rather than swallowed via
150
+ // `continue`). A worker that died mid-turn will never emit, so the synth must eventually fire.
151
+ // (b) an absolute LONG death-deadline — time since the generating_started ack exceeds
152
+ // ACKED_DEATH_DEADLINE_MS, a backstop set FAR above any observed emit latency (default 8 min)
153
+ // so it does not race a normal slow turn; it only catches a worker that is genuinely wedged or
154
+ // whose emit was permanently lost. This is a notification-loss net, not a completion timer.
155
+ //
156
+ // A dispatch that was never acked (worker never started) is NOT held here: there is no in-flight
157
+ // generation to protect, so it keeps the existing first-idle-tick synth behavior (its lost-dispatch
158
+ // case is covered by the downstream grace + stale-summary guards). The map is pruned each PHASE-4
159
+ // pass to the set of currently active dispatches, so a completed/pruned task's state is dropped (no
160
+ // unbounded growth). Keyed by `${meshId}::${taskId}`.
161
+
162
+ // R4f backstop (a): how many CONSECUTIVE read_chat failures (transport error / success:false /
163
+ // no payload) for an acked task are treated as a death signal that releases the indefinite hold.
164
+ // A single failed read is a transient probe blip; a session that genuinely died reads-fail every
165
+ // tick, so a small streak distinguishes the two without racing a live-but-slow worker.
166
+ const ACKED_DEATH_CONSECUTIVE_READ_FAILURES = 3;
167
+
168
+ // R4f backstop (b): the absolute death-deadline. An acked task is held indefinitely until this much
169
+ // time has elapsed since its generating_started ack (dispatch.updatedAt); past it, a persistently
170
+ // idle session is synthesized as a notification-loss net. This is set FAR above any observed emit
171
+ // latency (R4e's worst case was ~16s) so it does NOT race a normal slow turn — it only catches a
172
+ // genuinely wedged worker or a permanently-lost emit. Read at call time so tests can tune it.
173
+ function resolveTunedReconcileMs(envName: string, def: number, min: number, max: number): number {
174
+ const raw = readNonEmptyString(process.env[envName]);
175
+ if (raw) {
176
+ const parsed = Number.parseInt(raw, 10);
177
+ if (Number.isFinite(parsed) && parsed >= min && parsed <= max) return parsed;
178
+ }
179
+ return def;
180
+ }
181
+ function resolveAckedDeathDeadlineMs(): number {
182
+ // Default 8 min — FAR above the variable emit latency the finite R4..R4e timers raced (R4e's
183
+ // worst case was ~16s); by the time this fires a live worker would long since have emitted its
184
+ // real terminal. The env-override floor is 0 so tests can force the deadline (production never
185
+ // sets it that low); the ceiling is 60min so a mis-set env cannot disable the loss-net forever.
186
+ return resolveTunedReconcileMs('MESH_INFLIGHT_ACKED_DEATH_DEADLINE_MS', 8 * 60_000, 0, 60 * 60_000);
187
+ }
188
+
189
+ // ACKED-HOLD-IDLE-OVERTRUST (transcript-completion fast-track). The indefinite acked-hold above is
190
+ // safe but SLOW: when the worker's real generating_completed emit is dropped/lost, the only thing
191
+ // that promotes the missing completion is the 8-min death backstop — even though the answer has been
192
+ // FULLY rendered in the transcript for minutes (read_chat reports idle WITH a final visible assistant
193
+ // message every ~4s). Observed live: completions surfaced 144s / 492s late, both incompatible with the
194
+ // provider's own emit ceiling (COMPLETED_FINALIZATION_MAX_WAIT_MS 30s + NATIVE_HISTORY_MESH_IDLE_SETTLE
195
+ // 4s ≈ 34s). That gap = a worker that finished, whose PTY generating→idle edge / real emit was lost,
196
+ // held hostage to the 8-min net.
197
+ //
198
+ // Fast-track: when an acked task reads idle AND a final visible assistant message is present (the same
199
+ // transcript-completion evidence PHASE 4 already requires to synth), and that idle-with-final-assistant
200
+ // state has PERSISTED for a short continuous grace, promote the synth EARLY — ahead of the 8-min
201
+ // backstop. The grace is the correctness gate: a SINGLE idle read could be a mid-turn blip (PTY
202
+ // inter-tool-call settle, or final text rendered while the next tool call is about to start), so we
203
+ // require the idle-with-final-assistant signal to hold continuously for the grace window before
204
+ // trusting it as a genuine turn-end. Any non-idle read (generating / waiting_approval), a read
205
+ // failure, or the disappearance of the final assistant message RESETS the streak — so an actively
206
+ // streaming worker that momentarily reads idle never crosses the grace.
207
+ //
208
+ // Safety: this only changes WHEN an acked synth fires (earlier), never WHETHER it is correct —
209
+ // reconcileDirectDispatchCompletionFromTranscript's hasTerminalLedgerAfterDispatch makes a real
210
+ // emit that lands later an idempotent no-op, exactly as the death-backstop synth relies on. The
211
+ // death backstop (8 min) is PRESERVED unchanged as the final net; the fast-track is a faster path in
212
+ // front of it. The grace is set ABOVE the provider's own emit ceiling (~34s) so a worker still inside
213
+ // its normal finalization window is never pre-empted — we only fast-track once enough continuous idle
214
+ // has elapsed that a live emit would already have arrived.
215
+ function resolveAckedTranscriptFastTrackGraceMs(): number {
216
+ // Default 40s — above the provider emit ceiling (30s COMPLETED_FINALIZATION_MAX_WAIT_MS + 4s
217
+ // NATIVE_HISTORY_MESH_IDLE_SETTLE ≈ 34s): a genuinely-live worker would have emitted its real
218
+ // terminal within that window, so 40s of CONTINUOUS idle-with-final-assistant means the emit was
219
+ // lost, not late. Far below the 8-min death backstop, so the fast-track is the dominant path for a
220
+ // lost emit while the backstop remains the last-resort net. Floor 0 lets tests force an immediate
221
+ // fast-track; ceiling 5min keeps a mis-set env from collapsing it into the death backstop.
222
+ return resolveTunedReconcileMs('MESH_INFLIGHT_ACKED_TRANSCRIPT_FASTTRACK_GRACE_MS', 40_000, 0, 5 * 60_000);
223
+ }
224
+
225
+ // Per-task in-flight hold state for an acked dispatch:
226
+ // - liveConfirmedSinceAck: we have seen at least one conclusive read (idle OR generating) since
227
+ // the ack — proves the session is reachable, so a later read FAILURE is a genuine liveness loss
228
+ // rather than a node that was never reachable.
229
+ // - consecutiveReadFailures: streak of inconclusive read_chat results (death backstop (a)).
230
+ // - transcriptIdleSinceMs: the timestamp of the FIRST tick in the current continuous run of
231
+ // idle-with-final-assistant reads (ACKED-HOLD-IDLE-OVERTRUST fast-track). Cleared to undefined
232
+ // whenever the signal breaks (non-idle read, read failure, or no final assistant message), so a
233
+ // mid-turn idle blip never accumulates grace. When `now - transcriptIdleSinceMs` exceeds the
234
+ // fast-track grace the synth is promoted ahead of the death backstop.
235
+ interface AckedHoldState {
236
+ liveConfirmedSinceAck: boolean;
237
+ consecutiveReadFailures: number;
238
+ transcriptIdleSinceMs?: number;
239
+ }
240
+
241
+ // T2 (B2b): acked-hold state persistence. The Map below is a process-local CACHE;
242
+ // the SSOT is the mesh_inflight_hold table in MeshRuntimeStore. Every read goes
243
+ // read-through (Map miss → load from store, then cache), every mutation goes
244
+ // write-through (Map set → store upsert; Map delete → store delete). On daemon
245
+ // boot the reconcile loop rehydrates the Map from the store per-mesh the first
246
+ // time it touches that mesh (rehydrateAckedHoldsForMesh), so a hold established
247
+ // before a restart survives it — closing the duplicate-emit / drop window the
248
+ // PHASE-4 transcript synth backstop otherwise had to correct after the fact.
249
+ //
250
+ // Store row ↔ AckedHoldState mapping:
251
+ // hold_reason 'live'|'unconfirmed' ↔ liveConfirmedSinceAck (boolean)
252
+ // read_failure_count ↔ consecutiveReadFailures
253
+ // first_idle_since_ack ↔ transcriptIdleSinceMs (undefined ⇒ NULL)
254
+ // mesh_id = the owning mesh (for listByMesh / prune)
255
+ // held_at = ms the hold was first created (store-managed)
256
+ const inFlightAckedHoldState = new Map<string, AckedHoldState>();
257
+ // Meshes whose store rows have already been rehydrated into the Map this process.
258
+ // A restart resets this set, so the first touch of each mesh reloads from disk.
259
+ const rehydratedHoldMeshes = new Set<string>();
260
+
261
+ function inFlightSynthKey(meshId: string, taskId: string): string {
262
+ return `${meshId}::${taskId}`;
263
+ }
264
+
265
+ // Extract the taskId back out of a `${meshId}::${taskId}` synth key. The meshId
266
+ // prefix can itself contain '::' only if the caller passed one (mesh ids are
267
+ // config-derived and never do), so split on the FIRST '::' and treat the remainder
268
+ // as the taskId.
269
+ function taskIdFromSynthKey(meshId: string, synthKey: string): string {
270
+ const prefix = `${meshId}::`;
271
+ return synthKey.startsWith(prefix) ? synthKey.slice(prefix.length) : synthKey;
272
+ }
273
+
274
+ function holdStore(): MeshRuntimeStore | undefined {
275
+ try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
276
+ }
277
+
278
+ // Read-through: Map hit returns the cached state; a miss consults the store and,
279
+ // when a row exists, hydrates the Map from it before returning. A store failure
280
+ // degrades to Map-only (returns undefined on a miss) — identical to the pre-T2
281
+ // in-memory behavior, never worse.
282
+ function getHoldState(synthKey: string, meshId: string): AckedHoldState | undefined {
283
+ const cached = inFlightAckedHoldState.get(synthKey);
284
+ if (cached) return cached;
285
+ const store = holdStore();
286
+ if (!store) return undefined;
287
+ let row;
288
+ try { row = store.getInflightHold(taskIdFromSynthKey(meshId, synthKey)); } catch { return undefined; }
289
+ if (!row) return undefined;
290
+ const state: AckedHoldState = {
291
+ liveConfirmedSinceAck: row.holdReason === 'live',
292
+ consecutiveReadFailures: row.readFailureCount ?? 0,
293
+ ...(row.firstIdleSinceAck !== null && row.firstIdleSinceAck !== undefined
294
+ ? { transcriptIdleSinceMs: row.firstIdleSinceAck }
295
+ : {}),
296
+ };
297
+ inFlightAckedHoldState.set(synthKey, state);
298
+ return state;
299
+ }
300
+
301
+ // Write-through: update the Map cache AND the store row. A store failure leaves the
302
+ // Map authoritative for this process (degrade, never crash the tick).
303
+ function setHoldState(synthKey: string, meshId: string, state: AckedHoldState): void {
304
+ inFlightAckedHoldState.set(synthKey, state);
305
+ const store = holdStore();
306
+ if (!store) return;
307
+ try {
308
+ store.upsertInflightHold({
309
+ taskId: taskIdFromSynthKey(meshId, synthKey),
310
+ meshId,
311
+ holdReason: state.liveConfirmedSinceAck ? 'live' : 'unconfirmed',
312
+ firstIdleSinceAck: state.transcriptIdleSinceMs ?? null,
313
+ readFailureCount: state.consecutiveReadFailures,
314
+ });
315
+ } catch { /* degrade to Map-only */ }
316
+ }
317
+
318
+ // Write-through delete: drop the Map entry AND the store row.
319
+ function deleteHoldState(synthKey: string, meshId: string): void {
320
+ inFlightAckedHoldState.delete(synthKey);
321
+ const store = holdStore();
322
+ if (!store) return;
323
+ try { store.deleteInflightHold(taskIdFromSynthKey(meshId, synthKey)); } catch { /* degrade */ }
324
+ }
325
+
326
+ // Restart rehydration: on the first touch of a mesh this process, pull its persisted
327
+ // acked-hold rows from the store into the Map cache so a hold that outlived a daemon
328
+ // restart is honored again. Idempotent per process via rehydratedHoldMeshes. A store
329
+ // failure just skips rehydration (Map starts empty for the mesh — pre-T2 behavior).
330
+ function rehydrateAckedHoldsForMesh(meshId: string): void {
331
+ if (rehydratedHoldMeshes.has(meshId)) return;
332
+ rehydratedHoldMeshes.add(meshId);
333
+ const store = holdStore();
334
+ if (!store) return;
335
+ let rows;
336
+ try { rows = store.listInflightHoldsByMesh(meshId); } catch { return; }
337
+ for (const row of rows) {
338
+ const synthKey = inFlightSynthKey(meshId, row.taskId);
339
+ if (inFlightAckedHoldState.has(synthKey)) continue; // a live tick already set fresher state
340
+ inFlightAckedHoldState.set(synthKey, {
341
+ liveConfirmedSinceAck: row.holdReason === 'live',
342
+ consecutiveReadFailures: row.readFailureCount ?? 0,
343
+ ...(row.firstIdleSinceAck !== null && row.firstIdleSinceAck !== undefined
344
+ ? { transcriptIdleSinceMs: row.firstIdleSinceAck }
345
+ : {}),
346
+ });
347
+ }
348
+ if (rows.length > 0) {
349
+ LOG.info('MeshReconcile', `Rehydrated ${rows.length} persisted acked-hold row(s) for mesh ${meshId} after (re)start`);
350
+ }
351
+ }
352
+
353
+ // Test hook: clear the in-flight acked-hold state between cases (both the Map cache
354
+ // and the per-mesh rehydrate guard, so each case starts from a clean read-through).
355
+ export function __resetReconcileInFlightSynthDebounceForTests(): void {
356
+ inFlightAckedHoldState.clear();
357
+ rehydratedHoldMeshes.clear();
358
+ }
359
+
360
+ interface LiveCoordinator {
361
+ meshId: string;
362
+ instance: ReturnType<DaemonComponents['instanceManager']['getInstance']>;
363
+ // Runtime session id of this coordinator instance (getState().instanceId). PHASE 2
364
+ // strict-matches an event's targetCoordinatorSessionId against this so a completion
365
+ // routes back to the exact originating coordinator session, not a sibling on the same
366
+ // daemon (the multi-coordinator misroute).
367
+ sessionId: string;
368
+ // PTY-OVERTRUST-DRAIN (Defect B): drain-eligibility, decided on the RAW adapter
369
+ // turn-state (mask-stripped) — NOT on getState().status, which overlays the
370
+ // auto-approve "hold-idle" visual mask that paints a genuinely-idle coordinator
371
+ // `generating` and so used to strand its worker's completion. True only when the
372
+ // raw adapter is at a real turn end AND the session is not modal-parked. When the
373
+ // instance does not expose getDrainStatus() (non-CLI / older), this falls back to
374
+ // the masked `status === 'idle'` (the pre-fix behaviour) so nothing regresses.
375
+ idle: boolean;
376
+ // True when the coordinator session is parked on a harness modal awaiting a
377
+ // human answer — claude-cli AskUserQuestion (waiting_choice) or a tool-consent
378
+ // prompt (waiting_approval). A force-inject into such a session would write raw
379
+ // keystrokes the modal key handler consumes, silently selecting a choice the
380
+ // user never made (data corruption). PHASE 2 excludes these from force-inject
381
+ // and leaves the event queued for a later (modal-resolved) tick.
382
+ modalParked: boolean;
383
+ }
384
+
385
+ // The set of coordinator-daemon ids THIS daemon answers to when draining the
386
+ // pending-events queue. A unicast completion event is stamped with the worker's
387
+ // meshCoordinatorDaemonId, which can be either:
388
+ // - the daemon's canonical status id (`standalone_<machineId>` / `daemon_<machineId>`),
389
+ // stamped by the MCP layer via ctx.localDaemonId (= getStatus().status.instanceId), or
390
+ // - the bare machineId, stamped by the local queue-assignment path (loadConfig().machineId).
391
+ // - the config-form node daemonId (`daemon_<machineId>`), which the MCP layer's
392
+ // resolveCoordinatorDaemonId prefers and stamps onto direct-dispatch workers.
393
+ // Draining with only one of these silently misses events stamped with the other —
394
+ // the exact reason a generating coordinator never self-received local completions,
395
+ // and the base-node completion-surface bug (base completions land full-form
396
+ // `daemon_<machineId>` while a coordinator that only knows itself as bare
397
+ // `<machineId>` never matches them). We expand to EVERY equivalent form so the
398
+ // scope match (host gate, self-node detection, and the drain IN-filter downstream)
399
+ // succeeds regardless of which path stamped the event.
400
+ function resolveCoordinatorDaemonIds(components: DaemonComponents): string[] {
401
+ const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
402
+ const machineId = readNonEmptyString(loadConfig().machineId);
403
+ return expandDaemonIdForms([statusInstanceId, machineId]);
404
+ }
405
+
406
+ // Whether THIS daemon is the coordinator/host for a mesh — i.e. the daemon that
407
+ // owns coordinator ownership and must collect every worker node's completion
408
+ // events into its local queue. This is true regardless of whether a *live CLI*
409
+ // coordinator session currently exists: the coordinator is frequently a pure
410
+ // stdio MCP LLM (no live CLI session to inject into), and that LLM only sees the
411
+ // queue when it next calls a mesh tool. For it to see remote worker completions
412
+ // at all, the daemon must have already pulled them into the local queue on the
413
+ // timer — which is exactly what this predicate gates.
414
+ //
415
+ // Rule: this daemon hosts the mesh when meshHost.role is 'host' (the default for
416
+ // standalone-compat meshes with no host metadata) AND, when a hostDaemonId is
417
+ // pinned, it resolves to one of this daemon's ids. Member-only daemons return
418
+ // false — their own queue is pulled BY the host, not the other way around.
419
+ //
420
+ // `daemonIds` here is the EXPANDED self-identity set (runtime drain ids ∪ this
421
+ // daemon's mesh-config node id forms) — see resolveCoordinatorSelfIds. The
422
+ // pinned hostDaemonId is itself a config-form id and frequently does NOT equal a
423
+ // runtime id (bare machineId / status id), so gating on the runtime ids alone
424
+ // would wrongly classify the real host as a non-host and skip the remote pull
425
+ // entirely.
426
+ function daemonHostsMesh(mesh: LocalMeshEntry, daemonIds: string[]): boolean {
427
+ const host = mesh.meshHost;
428
+ // No metadata → default host (standalone compatibility, see createDefaultMeshHostMetadata).
429
+ if (!host) return true;
430
+ if (host.role && host.role !== 'host') return false;
431
+ const hostDaemonId = readNonEmptyString(host.hostDaemonId);
432
+ // Host role but no pinned hostDaemonId → treat as host (single-daemon / legacy).
433
+ if (!hostDaemonId) return true;
434
+ return daemonIdListIncludes(daemonIds, hostDaemonId);
435
+ }
436
+
437
+ function daemonIdListIncludes(ids: readonly string[], id: string | undefined): boolean {
438
+ if (!id) return false;
439
+ return ids.some(candidate => candidate === id || daemonIdsEquivalent(candidate, id));
440
+ }
441
+
442
+ // Resolve EVERY id-form this daemon answers to FOR A GIVEN MESH: the runtime drain
443
+ // ids (status id + bare machineId) unioned with this daemon's mesh-config identity
444
+ // forms — the self node's daemonId/machineId (the node whose daemonId/machineId
445
+ // matches a runtime id) and the pinned meshHost.hostDaemonId WHEN it is provably
446
+ // ours. This is the single source of truth for "is this id me?" across both the
447
+ // host gate and the remote pull filter; the worker's meshCoordinatorDaemonId stamp
448
+ // is guaranteed to be one of these forms (it comes from resolveCoordinatorDaemonId,
449
+ // which prefers the coordinator node's config-form daemonId over the runtime status id).
450
+ function resolveCoordinatorSelfIds(mesh: LocalMeshEntry, drainDaemonIds: string[]): string[] {
451
+ const ids = new Set<string>(drainDaemonIds);
452
+ // Expand with the config-form id(s) of the self node — the mesh node whose
453
+ // daemonId/machineId matches a runtime id. Its config-form daemonId is exactly
454
+ // what resolveCoordinatorNode()→resolveCoordinatorDaemonId() stamps onto a worker.
455
+ for (const node of mesh.nodes) {
456
+ const nodeDaemonId = readNonEmptyString(node.daemonId);
457
+ const nodeMachineId = readNonEmptyString(node.machineId);
458
+ const isSelf = (nodeDaemonId && daemonIdListIncludes(drainDaemonIds, nodeDaemonId))
459
+ || (nodeMachineId && daemonIdListIncludes(drainDaemonIds, nodeMachineId));
460
+ if (!isSelf) continue;
461
+ if (nodeDaemonId) ids.add(nodeDaemonId);
462
+ if (nodeMachineId) ids.add(nodeMachineId);
463
+ }
464
+ // The pinned host id is included ONLY when it is provably one of THIS daemon's ids
465
+ // (it already matches a runtime id or a resolved self-node id). A hostDaemonId that
466
+ // names a DIFFERENT daemon must NOT be claimed — that would make a member-only
467
+ // daemon believe it is the host and pull queues it does not own. Having a node on
468
+ // this daemon does not make this daemon the host; daemonHostsMesh still honours a
469
+ // foreign hostDaemonId and rejects ownership.
470
+ const hostDaemonId = readNonEmptyString(mesh.meshHost?.hostDaemonId);
471
+ if (hostDaemonId && daemonIdListIncludes([...ids], hostDaemonId)) ids.add(hostDaemonId);
472
+ return [...ids];
473
+ }
474
+
475
+ // Observability: last-seen modal-park state per coordinator session, so we LOG.info
476
+ // only on a TRANSITION (clear → parked, parked → cleared) instead of every 4s tick.
477
+ // Per-process; a restart re-logs the first observation, which is desirable — it
478
+ // re-confirms a coordinator that is still parked after the restart (the exact
479
+ // "restart does not clear it" symptom the operator needs visibility into).
480
+ const coordinatorModalParkState = new Map<string, boolean>();
481
+
482
+ // Find live CLI coordinator instances on THIS daemon, keyed by mesh.
483
+ function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
484
+ const out: LiveCoordinator[] = [];
485
+ for (const inst of components.instanceManager.getByCategory('cli')) {
486
+ const state = inst.getState();
487
+ const settings = state.settings && typeof state.settings === 'object'
488
+ ? state.settings as Record<string, unknown>
489
+ : {};
490
+ const meshId = readNonEmptyString(settings.meshCoordinatorFor);
491
+ if (!meshId) continue;
492
+ const status = readNonEmptyString(state.status).toLowerCase();
493
+ // getState() overlays the modal-park statuses: an active AskUserQuestion
494
+ // prompt surfaces as waiting_choice, a tool-consent prompt as waiting_approval.
495
+ // NOTIF-HELD-DRAIN (Fix 1): consult the instance's own isModalParked() rather than the
496
+ // raw status literal so the corrected classification flows here — a busy mesh
497
+ // coordinator's routine, in-flight tool-consent (auto-approve off) is NOT a human-await
498
+ // modal and must NOT wedge the mesh's pending completion events under `modal_parked`.
499
+ // resolveModalParkStatus() (which isModalParked wraps) already encodes that distinction
500
+ // and the waiting_choice/stalled-auto-approve genuine-modal cases. Fall back to the
501
+ // status literal for any instance that does not expose the method. Lowercase compare —
502
+ // the SessionStatus enum is forked across modules and waiting_choice is absent from some.
503
+ const modalParked = typeof (inst as any).isModalParked === 'function'
504
+ ? (inst as any).isModalParked() === true
505
+ : (status === 'waiting_choice' || status === 'waiting_approval');
506
+ // PTY-OVERTRUST-DRAIN (Defect B, fix A): drain-eligible idle is decided on the
507
+ // RAW adapter turn-state, not getState().status. getState() overlays the
508
+ // auto-approve hold-idle mask that paints a genuinely-idle coordinator
509
+ // `generating` (a UI-flicker suppressant), and the reconcile loop used to trust
510
+ // that mask and HOLD the worker's completion (generating_no_idle_coordinator)
511
+ // even though the PTY was at a real turn end. getDrainStatus() strips the mask
512
+ // (raw adapter idle, modal-park preserved). Fall back to the masked literal for
513
+ // any instance that does not expose it (non-CLI / older) — regression-0.
514
+ const drainStatus: string | null = typeof (inst as any).getDrainStatus === 'function'
515
+ ? (inst as any).getDrainStatus()
516
+ : null;
517
+ const idle = drainStatus !== null ? drainStatus === 'idle' : (status === 'idle');
518
+ const sessionId = readNonEmptyString(state.instanceId);
519
+ // ── NOTIF (B) desync diagnostic (read-only, no behavior change) ───────────
520
+ // The confirmed (B) defect: a coordinator whose FSM is idle (status above ===
521
+ // 'idle' for minutes) is nonetheless classified busy here, so the generating/
522
+ // modal-park hold never drains and a worker completion is stranded until the
523
+ // user's next turn edge. Static analysis found no code path where getState()
524
+ // returns generating while lastStatus and the adapter raw are both idle — so the
525
+ // divergence is a runtime desync between the three status sources. Capture all
526
+ // three (plus the auto-approve mask state that getState() overlays at :803) for
527
+ // EVERY mesh-coordinator candidate on this tick, so the source that diverges from
528
+ // the others can be read directly against the same-tick "skip → generating"/
529
+ // "skip → modal-parked" hold logs below (pair by sessionId + timestamp).
530
+ //
531
+ // CRITICAL: reuse the `state` already fetched above (line ~301) — do NOT call
532
+ // getState() again. getState() runs maybeAutoApproveStatus() as a side effect,
533
+ // which would mutate the very auto-approve mask we are trying to observe. The
534
+ // adapter raw read uses allowParse:false, which only reads engine.activeModal and
535
+ // is side-effect-free.
536
+ if (getLogLevel() === 'debug') {
537
+ let adapterRaw = '?';
538
+ try {
539
+ const a = (inst as any).adapter;
540
+ if (a && typeof a.getStatus === 'function') {
541
+ adapterRaw = readNonEmptyString(a.getStatus({ allowParse: false })?.status) || '?';
542
+ }
543
+ } catch (e: any) {
544
+ adapterRaw = `err:${e?.message || e}`;
545
+ }
546
+ const lastStatus = readNonEmptyString((inst as any).lastStatus) || '?';
547
+ const autoApproveBusy = (inst as any).autoApproveBusy;
548
+ const maskSince = (inst as any).autoApproveMaskSince;
549
+ // PTY-OVERTRUST-DRAIN: include the mask-stripped drainStatus next to the three
550
+ // legacy sources so the divergence (getState=generating while adapterRaw=idle =
551
+ // the mask) is directly readable, and confirm drain now follows adapterRaw.
552
+ LOG.debug('MeshReconcile', `coordDiag sess=${sessionId || '?'} mesh=${meshId} getState=${status || '?'} drainStatus=${drainStatus || 'n/a'} lastStatus=${lastStatus} adapterRaw=${adapterRaw} autoApproveBusy=${autoApproveBusy === true} maskSince=${maskSince || 0}`);
553
+ }
554
+ // Modal-park transition observability: a coordinator entering modal-park is what
555
+ // begins holding completion events under `modal_parked`; one leaving it is what
556
+ // drains them. Both transitions were previously SILENT (the operator had no log
557
+ // to diagnose a stuck/held completion), so emit a single line per edge.
558
+ const stateKey = `${meshId}::${sessionId || '?'}`;
559
+ const prevParked = coordinatorModalParkState.get(stateKey);
560
+ if (prevParked !== modalParked) {
561
+ coordinatorModalParkState.set(stateKey, modalParked);
562
+ if (modalParked) {
563
+ LOG.info('MeshReconcile', `Coordinator ${sessionId || '?'} (mesh ${meshId}) entered modal-park (status=${status}) — terminal events for it will be held until the modal is answered`);
564
+ } else if (prevParked === true) {
565
+ LOG.info('MeshReconcile', `Coordinator ${sessionId || '?'} (mesh ${meshId}) left modal-park (status=${status}) — held events will drain on this/next tick`);
566
+ }
567
+ }
568
+ out.push({ meshId, instance: inst, sessionId, idle, modalParked });
569
+ }
570
+ return out;
571
+ }
572
+
573
+ /**
574
+ * DRAIN-WITHOUT-INJECT guard. Classify, for a mesh on THIS daemon, whether a
575
+ * queue-drain caller (the MCP `get_pending_mesh_events` poll) may safely consume
576
+ * pending coordinator events — i.e. whether there is a surface that will actually
577
+ * deliver them.
578
+ *
579
+ * Root cause being guarded: `get_pending_mesh_events` marks rows drained=1
580
+ * atomically and unconditionally. When the live CLI coordinator for the mesh is
581
+ * GENERATING (or modal-parked), the reconcile loop correctly HOLDS its terminal
582
+ * events (drained=0) for the coordinator's next idle tick — but a concurrent MCP
583
+ * poll draining the SAME queue consumes those held rows into a tool result that
584
+ * the busy coordinator never surfaces as a turn, so the completion is lost
585
+ * forever (drained=1, never re-queued). The reconcile loop is the authoritative
586
+ * delivery path for a live CLI coordinator; the MCP poll must defer to it.
587
+ *
588
+ * Returns:
589
+ * - hasLiveCliCoordinator: a CLI session with meshCoordinatorFor === meshId
590
+ * exists on this daemon (the reconcile loop owns its delivery).
591
+ * - deliverableNow: there is an IDLE live CLI coordinator (reconcile would
592
+ * full-drain into it) — draining now is safe and equivalent.
593
+ * - holdForReconcile: a live CLI coordinator exists but is non-idle
594
+ * (generating / modal-parked). The MCP poll MUST NOT drain; the reconcile
595
+ * loop holds the events undrained and injects them on the next idle tick.
596
+ *
597
+ * A mesh with NO live CLI coordinator on this daemon is a pure stdio MCP / LLM
598
+ * coordinator: the MCP tool result IS the only surface, so the poll legitimately
599
+ * drains (holdForReconcile=false). No regression to that path.
600
+ */
601
+ export function resolveCoordinatorDrainDeliverability(
602
+ components: Pick<DaemonComponents, 'instanceManager'>,
603
+ meshId: string,
604
+ ): { hasLiveCliCoordinator: boolean; deliverableNow: boolean; holdForReconcile: boolean } {
605
+ const coordinators = findLiveCoordinators(components as DaemonComponents).filter(c => c.meshId === meshId);
606
+ if (coordinators.length === 0) {
607
+ return { hasLiveCliCoordinator: false, deliverableNow: false, holdForReconcile: false };
608
+ }
609
+ const hasIdle = coordinators.some(c => c.idle);
610
+ return {
611
+ hasLiveCliCoordinator: true,
612
+ deliverableNow: hasIdle,
613
+ // A live CLI coordinator exists but none is idle → the reconcile loop is
614
+ // holding the events; the poll must not steal them.
615
+ holdForReconcile: !hasIdle,
616
+ };
617
+ }
618
+
619
+ /**
620
+ * DRAIN-WITHOUT-INJECT guard for the `get_pending_mesh_events` daemon handler.
621
+ *
622
+ * Decides whether an incoming pending-events DRAIN must be held (return nothing,
623
+ * leave rows drained=0) because the only surface for those events is a LOCAL live
624
+ * CLI coordinator that is currently busy (generating / modal-parked) — in which
625
+ * case the reconcile loop owns delivery on the coordinator's next idle tick, and
626
+ * the poll draining them now would lose them.
627
+ *
628
+ * The hold applies ONLY when BOTH:
629
+ * 1) a live CLI coordinator for this mesh on THIS daemon is non-idle, AND
630
+ * 2) the drain is targeted at THIS daemon (the requested coordinatorDaemonId is
631
+ * empty/broadcast, or matches one of this daemon's id forms).
632
+ *
633
+ * A REMOTE coordinator pulling our worker's events passes its own (remote)
634
+ * coordinatorDaemonId — condition (2) is false — so the drain proceeds and the
635
+ * remote pull is never blocked by our local coordinator's busy state. A pure
636
+ * stdio MCP coordinator (no live CLI session) never satisfies (1), so its tool
637
+ * result remains the surface and the drain proceeds. No regression to either.
638
+ *
639
+ * SELF-COORDINATOR INBOX LEVEL-DRAIN (Defect 2): the hold above assumes the ONLY
640
+ * surface for a busy local coordinator's events is a future PTY inject on its idle
641
+ * edge, so it defers to the reconcile loop. But when the drain caller IS the local
642
+ * coordinator reading its OWN inbox (the `get_pending_mesh_events` call whose events
643
+ * are returned in the caller's tool RESULT — a data queue the self-coordinating LLM
644
+ * consumes directly), the events ARE surfaced losslessly the moment the tool returns,
645
+ * with NO PTY write. A busy self-coordinating LLM that calls a mesh tool mid-turn would
646
+ * otherwise get an empty inbox (held) and only see the completion on its NEXT busy→idle
647
+ * edge — the measured ~59s strand. `callerIsSelfCoordinatorInboxRead` marks that safe
648
+ * caller: the hold is relaxed for it (return the events), while every OTHER drain (a
649
+ * backfill relay, a broadcast poll, a DIFFERENT coordinator that genuinely needs its PTY)
650
+ * still defers to the reconcile loop. This relaxes delivery INTO the coordinator's own
651
+ * inbox only — it never changes how events are injected into a live PTY prompt.
652
+ */
653
+ export function shouldHoldPendingDrainForBusyLocalCoordinator(
654
+ components: Pick<DaemonComponents, 'instanceManager'> & { statusInstanceId?: string },
655
+ meshId: string,
656
+ requestedCoordinatorDaemonId?: string | null,
657
+ callerIsSelfCoordinatorInboxRead?: boolean,
658
+ ): boolean {
659
+ if (!meshId) return false;
660
+ const deliverability = resolveCoordinatorDrainDeliverability(components, meshId);
661
+ if (!deliverability.holdForReconcile) return false;
662
+ // The local CLI coordinator is busy. Hold only when the drain is for THIS daemon.
663
+ const requested = readNonEmptyString(requestedCoordinatorDaemonId);
664
+ if (!requested) return true; // broadcast drain → would consume the held local events
665
+ const localIds = expandDaemonIdForms([
666
+ readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId),
667
+ readNonEmptyString(loadConfig().machineId),
668
+ ]);
669
+ const targetsLocalCoordinator = localIds.some(id => daemonIdsEquivalent(id, requested));
670
+ if (!targetsLocalCoordinator) return false;
671
+ // SELF-COORDINATOR INBOX LEVEL-DRAIN: the busy local coordinator is itself the caller,
672
+ // reading its own inbox — the drained events return in ITS tool result (lossless data-queue
673
+ // surface, no PTY inject). Do NOT hold; let the self-coordinator see its completions now.
674
+ if (callerIsSelfCoordinatorInboxRead) return false;
675
+ return true;
676
+ }
677
+
678
+ // Inject a drained pending event into a live coordinator session. Force-inject
679
+ // events carry force:true so they bypass the busy send-guard and land in the PTY
680
+ // even while the coordinator is generating (see shouldForceInjectMeshEvent).
681
+ function injectPendingIntoCoordinator(
682
+ coordinator: LiveCoordinator['instance'],
683
+ pending: PendingMeshCoordinatorEvent,
684
+ opts?: { forceOverride?: boolean },
685
+ ): void {
686
+ if (!coordinator) return;
687
+ // NOTIF-DROP-SYNTH-NO-MESSAGE (defence-in-depth): a queued event with no coordinatorMessage
688
+ // used to be dropped here (drain-without-inject) — the row had already been consumed
689
+ // (drained=1) by the caller's drain, so silently returning lost it forever. The primary fix
690
+ // makes the transcript-reconcile synth always carry a coordinatorMessage, but as a backstop,
691
+ // lazily synthesize the [System] text for any force-inject (terminal: completion / approval /
692
+ // stop / refine·bootstrap) event that still arrives message-less, so it surfaces instead of
693
+ // vanishing. A NON-force lifecycle event (agent:ready / generating_started) legitimately
694
+ // carries no message and must NOT be injected (it is queued only to re-drive the claim state
695
+ // machine on pull) — for it we still return without injecting.
696
+ let coordinatorMessage = pending.coordinatorMessage;
697
+ if (!coordinatorMessage) {
698
+ if (!shouldForceInjectMeshEvent(pending.event)) return;
699
+ const metadataEvent = pending.metadataEvent && typeof pending.metadataEvent === 'object'
700
+ ? pending.metadataEvent
701
+ : {};
702
+ coordinatorMessage = buildMeshSystemMessage({
703
+ event: pending.event,
704
+ nodeLabel: pending.nodeLabel,
705
+ metadataEvent,
706
+ });
707
+ if (!coordinatorMessage) return; // builder produced nothing — nothing to surface
708
+ LOG.warn('MeshReconcile', `Lazily synthesized missing coordinatorMessage for ${pending.event} (mesh ${pending.meshId}) at inject time — a queued terminal event arrived message-less`);
709
+ }
710
+ // forceOverride lets the APPROVAL-Q1-REALTIME nudge path deliver into a busy
711
+ // coordinator WITHOUT a raw PTY force-write (force-inject-into-generating stays
712
+ // intentionally removed): a non-force send_message enters the adapter's
713
+ // pendingOutboundQueue and is surfaced at the coordinator's next turn boundary.
714
+ const force = opts?.forceOverride ?? shouldForceInjectMeshEvent(pending.event);
715
+ // EVTTRACE: event surfaced to the coordinator (injected into its live CLI session).
716
+ // This is the terminal happy-path stage. Observation only.
717
+ traceMeshEventStage('surfaced', {
718
+ taskId: pending.metadataEvent?.taskId,
719
+ sessionId: pending.metadataEvent?.targetSessionId ?? pending.targetCoordinatorSessionId,
720
+ nodeId: pending.nodeId,
721
+ meshId: pending.meshId,
722
+ event: pending.event,
723
+ }, force ? 'force-inject' : 'inject');
724
+ coordinator.onEvent('send_message', {
725
+ input: { text: coordinatorMessage, textFallback: coordinatorMessage },
726
+ ...(force ? { force: true } : {}),
727
+ });
728
+ }
729
+
730
+ // Held-event ledger dedup: fingerprints of held terminal events already written as an
731
+ // `event_held` ledger audit record in THIS process. Prevents the 4s reconcile tick from
732
+ // re-logging the same held event every interval while a coordinator stays modal-parked.
733
+ // Per-process only (not persisted) — if the daemon restarts while an event is still held
734
+ // it is re-logged once, which is desirable: it re-confirms the event is still undelivered.
735
+ const heldEventLedgerRecorded = new Set<string>();
736
+
737
+ // C1 (data safety): when a terminal completion/approval/bootstrap event cannot be
738
+ // delivered because the only coordinators are modal-parked, the event is held at
739
+ // drained=0 in the pending queue (SQLite + JSONL) for a later tick. That queue is
740
+ // disk-persisted but carries no operator-visible audit trail and can be silently
741
+ // dropped by the pending-file trim (100 KB / 50-entry cap). To guarantee a held
742
+ // completion's worker summary is never silently lost, mirror each held terminal event
743
+ // into the coordinator's mesh ledger as an `event_held` entry — auditable and
744
+ // recoverable (the finalSummary survives even if the pending copy is later trimmed or
745
+ // the coordinator session is force-resolved before re-drain). Idempotent per process
746
+ // via heldEventLedgerRecorded so a long modal park does not spam the ledger.
747
+ function recordHeldTerminalEventsToLedger(
748
+ meshId: string,
749
+ drainDaemonIds: string[],
750
+ reason: string,
751
+ heldForCoordinatorCount: number,
752
+ ): void {
753
+ let pending: readonly PendingMeshCoordinatorEvent[];
754
+ try {
755
+ pending = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
756
+ } catch {
757
+ return; // best-effort audit — never let a peek failure break the tick
758
+ }
759
+ for (const event of pending) {
760
+ // Only audit terminal/force-inject events (completion / approval / stop / refine·
761
+ // bootstrap). Silent lifecycle events (agent:ready / generating_started) carry no
762
+ // worker output to preserve and re-drain harmlessly, so they need no audit trail.
763
+ if (!shouldForceInjectMeshEvent(event.event)) continue;
764
+ const fingerprint = buildPendingEventFingerprint(event);
765
+ const key = `${meshId}::${fingerprint || `${event.event}::${event.nodeId || ''}::${event.queuedAt}`}`;
766
+ if (heldEventLedgerRecorded.has(key)) continue;
767
+ heldEventLedgerRecorded.add(key);
768
+ const finalSummary = readMeshCompletionSummary(event.metadataEvent);
769
+ try {
770
+ appendLedgerEntry(meshId, {
771
+ kind: 'event_held',
772
+ ...(event.nodeId ? { nodeId: event.nodeId } : {}),
773
+ payload: {
774
+ event: event.event,
775
+ reason,
776
+ recoverable: true,
777
+ heldForCoordinators: heldForCoordinatorCount,
778
+ nodeLabel: event.nodeLabel,
779
+ ...(event.workspace ? { workspace: event.workspace } : {}),
780
+ targetCoordinatorDaemonId: event.targetCoordinatorDaemonId ?? null,
781
+ queuedAt: event.queuedAt,
782
+ ...(fingerprint ? { fingerprint } : {}),
783
+ ...(finalSummary ? { finalSummary } : {}),
784
+ },
785
+ });
786
+ LOG.info('MeshReconcile', `Ledger-recorded held ${event.event} for mesh ${meshId} (reason ${reason}) — recoverable from ledger`);
787
+ } catch (e: any) {
788
+ // Failed to persist — drop the dedup marker so the next tick retries.
789
+ heldEventLedgerRecorded.delete(key);
790
+ LOG.warn('MeshReconcile', `Failed to ledger-record held ${event.event} for mesh ${meshId}: ${e?.message || e}`);
791
+ }
792
+ }
793
+ }
794
+
795
+ // PTY-OVERTRUST-DRAIN (Defect B, fix B). Age of the OLDEST queued terminal/force-inject
796
+ // event for a mesh, in ms — the signal the generating-hold age-escape gates on. Returns 0
797
+ // when there is no held terminal event (no escape needed). Best-effort: a peek failure
798
+ // returns 0 (no escape this tick), never throws into the tick.
799
+ function oldestHeldTerminalEventAgeMs(meshId: string, drainDaemonIds: string[]): number {
800
+ let pending: readonly PendingMeshCoordinatorEvent[];
801
+ try {
802
+ pending = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
803
+ } catch {
804
+ return 0;
805
+ }
806
+ const now = Date.now();
807
+ let maxAge = 0;
808
+ for (const event of pending) {
809
+ if (!shouldForceInjectMeshEvent(event.event)) continue; // only terminal events matter
810
+ const queuedAt = typeof event.queuedAt === 'number' ? event.queuedAt : now;
811
+ const age = now - queuedAt;
812
+ if (age > maxAge) maxAge = age;
813
+ }
814
+ return maxAge;
815
+ }
816
+
817
+ // PTY-OVERTRUST-DRAIN (Defect B, fix B). Re-confirm, on the RAW adapter (mask-stripped),
818
+ // which of the held-as-generating coordinators is GENUINELY idle right now. A coordinator
819
+ // whose getDrainStatus() reads 'idle' is a real drain target the time-based escape may
820
+ // deliver into. One that still reads 'generating'/'modal_parked'/'other' stays held — the
821
+ // escape NEVER injects into a genuinely-busy PTY (that is the data-loss force-inject path
822
+ // intentionally removed; re-confirmation is what keeps this safe). Falls back to the
823
+ // coordinator's already-computed `idle` flag when the instance does not expose
824
+ // getDrainStatus() (non-CLI / older) — that flag is itself raw-adapter-derived post-fix-A.
825
+ function reconfirmGenuinelyIdleCoordinators(generating: LiveCoordinator[]): LiveCoordinator[] {
826
+ const out: LiveCoordinator[] = [];
827
+ for (const c of generating) {
828
+ const inst = c.instance as any;
829
+ const drainStatus: string | null = typeof inst?.getDrainStatus === 'function'
830
+ ? inst.getDrainStatus()
831
+ : null;
832
+ const genuinelyIdle = drainStatus !== null ? drainStatus === 'idle' : c.idle;
833
+ if (genuinelyIdle) out.push({ ...c, idle: true });
834
+ }
835
+ return out;
836
+ }
837
+
838
+ // Full-drain the local pending queue for a mesh and inject every event into the given
839
+ // IDLE target coordinators, honouring strict session routing. Shared by the normal idle
840
+ // delivery path and the Defect-B age-escape so both deliver identically (one drain, one
841
+ // inject-per-event, strict-route hold for an unmatched session). Returns the number of
842
+ // events drained (0 when the queue was empty / drain failed). Callers must have already
843
+ // confirmed the targets are genuinely idle.
844
+ function drainAndInjectIntoTargets(
845
+ meshId: string,
846
+ drainDaemonIds: string[],
847
+ localDaemonId: string | undefined,
848
+ targetCoordinators: LiveCoordinator[],
849
+ logLabel: string,
850
+ ): number {
851
+ let pendingEvents: PendingMeshCoordinatorEvent[] = [];
852
+ try {
853
+ pendingEvents = drainPendingMeshCoordinatorEvents(
854
+ meshId,
855
+ drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
856
+ );
857
+ } catch (e: any) {
858
+ LOG.warn('MeshReconcile', `Drain failed for mesh ${meshId}: ${e?.message || e}`);
859
+ return 0;
860
+ }
861
+ if (pendingEvents.length === 0) return 0;
862
+
863
+ LOG.info('MeshReconcile', `Reconcile inject → ${logLabel}: ${pendingEvents.length} pending event(s) → ${targetCoordinators.length} coordinator(s) for mesh ${meshId}`);
864
+ for (const pending of pendingEvents) {
865
+ // Strict session routing (multi-coordinator): when the event names an
866
+ // originating coordinator session, deliver ONLY to the live coordinator whose
867
+ // session id matches — a sibling coordinator on the same daemon must NOT receive
868
+ // another coordinator's completion. When the event carries no session id (legacy /
869
+ // version-skewed / single-coordinator), fall back to the daemon-level set
870
+ // (unchanged behaviour — regression-0 for the common case).
871
+ const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
872
+ if (wantSession) {
873
+ // Session ids are single-form; sessionIdsEquivalent is the one canonical
874
+ // exact-match predicate — unlike the daemon-level set below it needs no
875
+ // form expansion.
876
+ const matched = targetCoordinators.filter(c => sessionIdsEquivalent(c.sessionId, wantSession));
877
+ if (matched.length === 0) {
878
+ // The originating coordinator session is not deliverable on this daemon
879
+ // right now (gone, or modal-parked and excluded from targets). Strict mode
880
+ // does NOT broadcast to siblings — hold the event for a later tick, and
881
+ // ledger-expire it past a TTL so it can never wedge forever.
882
+ holdOrExpireStrictUnmatchedEvent(pending, wantSession, meshId);
883
+ continue;
884
+ }
885
+ for (const c of matched) injectPendingIntoCoordinator(c.instance, pending);
886
+ continue;
887
+ }
888
+ for (const c of targetCoordinators) {
889
+ injectPendingIntoCoordinator(c.instance, pending);
890
+ }
891
+ }
892
+ return pendingEvents.length;
893
+ }
894
+
895
+ // APPROVAL-Q1-REALTIME stale guard. An approval nudge is RESOLVED once a real terminal
896
+ // ledger entry (task_completed / task_failed) for the same node/session landed at or
897
+ // after the nudge was queued — the worker either finished or died, so it is no longer
898
+ // waiting on that approval. Delivering the nudge then would falsely tell the coordinator
899
+ // the worker is still blocked (the exact UX inversion this fix must avoid), so a resolved
900
+ // nudge is dropped rather than delivered. Ledger-based so the check is daemon-local and
901
+ // deterministic (no dependence on a possibly-remote worker instance's live state).
902
+ function isApprovalNudgeResolved(meshId: string, pending: PendingMeshCoordinatorEvent): boolean {
903
+ const metadataEvent = (pending.metadataEvent && typeof pending.metadataEvent === 'object')
904
+ ? pending.metadataEvent as Record<string, unknown>
905
+ : {};
906
+ const nodeId = readNonEmptyString(pending.nodeId) || readNonEmptyString(metadataEvent.meshNodeId);
907
+ const sessionId = readNonEmptyString(metadataEvent.targetSessionId) || readNonEmptyString(metadataEvent.sessionId);
908
+ if (!nodeId && !sessionId) return false; // nothing to correlate a terminal against
909
+ const queuedAt = typeof pending.queuedAt === 'number' && Number.isFinite(pending.queuedAt) ? pending.queuedAt : 0;
910
+ let entries: MeshLedgerEntry[];
911
+ try {
912
+ entries = readLedgerEntries(meshId);
913
+ } catch {
914
+ return false; // best-effort — a read failure never blocks delivery
915
+ }
916
+ return entries.some(e => {
917
+ if (e.kind !== 'task_completed' && e.kind !== 'task_failed') return false;
918
+ if (queuedAt > 0) {
919
+ const t = new Date(e.timestamp).getTime();
920
+ if (Number.isFinite(t) && t < queuedAt) return false; // terminal predates the nudge
921
+ }
922
+ const nodeMatch = !!nodeId && !!e.nodeId && daemonIdsEquivalent(e.nodeId, nodeId);
923
+ const sessionMatch = !!sessionId && !!e.sessionId && sessionIdsEquivalent(e.sessionId, sessionId);
924
+ return nodeMatch || sessionMatch;
925
+ });
926
+ }
927
+
928
+ // APPROVAL-Q1-REALTIME. Deliver queued approval nudges to a mesh's coordinators every
929
+ // reconcile tick, EVEN when the only coordinators are busy (generating / modal-parked)
930
+ // and there is no idle drain target. This is the crux of the fix: a completion rides the
931
+ // idle-edge hold below (its payload lives only in the pending event), but an approval is
932
+ // LEVEL-backed (task_approval_needed ledger → mesh_status awaiting_approval) so it must
933
+ // NOT wait for an idle edge — during orchestration a coordinator can stay `generating`
934
+ // awaiting the very worker that is blocked on the approval, so the idle edge (the flush
935
+ // point) may never come, and the coordinator's mesh_approve arrives only after a human
936
+ // resolves it ('Not in approval state'). We drain ONLY approval events (leaving every
937
+ // other event for the unchanged hold), drop any already-resolved (stale) nudge, and
938
+ // deliver the rest into each coordinator's inbox WITHOUT a raw PTY force-write (non-force
939
+ // send_message → adapter pendingOutboundQueue → surfaced at the coordinator's next turn
940
+ // boundary). Dropping the pending copy after delivery is safe and prevents re-nudging
941
+ // every 4s — the level ledger state remains the durable, re-derivable source of truth.
942
+ // Returns the number of nudges delivered (0 when none were queued/deliverable).
943
+ function drainAndDeliverApprovalNudges(
944
+ meshId: string,
945
+ drainDaemonIds: string[],
946
+ localDaemonId: string | undefined,
947
+ meshCoordinators: LiveCoordinator[],
948
+ ): number {
949
+ // O(1) guard: only touch the queue when an approval event is actually present.
950
+ let peeked: readonly PendingMeshCoordinatorEvent[];
951
+ try {
952
+ peeked = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
953
+ } catch {
954
+ return 0;
955
+ }
956
+ if (!peeked.some(e => isMeshApprovalEvent(e.event))) return 0;
957
+
958
+ let drained: PendingMeshCoordinatorEvent[];
959
+ try {
960
+ drained = drainPendingMeshCoordinatorEvents(
961
+ meshId,
962
+ drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
963
+ { onlyEvents: MESH_APPROVAL_EVENTS },
964
+ );
965
+ } catch (e: any) {
966
+ LOG.warn('MeshReconcile', `Approval-nudge drain failed for mesh ${meshId}: ${e?.message || e}`);
967
+ return 0;
968
+ }
969
+
970
+ let delivered = 0;
971
+ for (const pending of drained) {
972
+ if (isApprovalNudgeResolved(meshId, pending)) {
973
+ // Stale: already resolved. Drop without delivery — re-surfacing it would
974
+ // mislead the coordinator into believing the worker is still awaiting approval.
975
+ traceMeshEventDrop('approval_nudge_stale_resolved', {
976
+ taskId: readNonEmptyString((pending.metadataEvent as Record<string, unknown>)?.taskId),
977
+ sessionId: readNonEmptyString((pending.metadataEvent as Record<string, unknown>)?.targetSessionId) ?? pending.targetCoordinatorSessionId,
978
+ nodeId: pending.nodeId,
979
+ meshId,
980
+ event: pending.event,
981
+ }, 'approval already resolved (terminal ledger entry present)');
982
+ LOG.info('MeshReconcile', `Dropped stale approval nudge for mesh ${meshId} (${pending.nodeLabel}) — approval already resolved`);
983
+ continue;
984
+ }
985
+ // Strict session routing (multi-coordinator): deliver only to the originating
986
+ // coordinator session when the nudge names one; otherwise broadcast to every
987
+ // coordinator for this mesh. Absent a live matching coordinator we drop the nudge —
988
+ // the level state (awaiting_approval) still surfaces via mesh_status, so nothing is lost.
989
+ const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
990
+ const targets = wantSession
991
+ ? meshCoordinators.filter(c => sessionIdsEquivalent(c.sessionId, wantSession))
992
+ : meshCoordinators;
993
+ if (targets.length === 0) continue;
994
+ for (const c of targets) injectPendingIntoCoordinator(c.instance, pending, { forceOverride: false });
995
+ delivered++;
996
+ LOG.info('MeshReconcile', `Delivered approval nudge (level) for mesh ${meshId} (${pending.nodeLabel}) → ${targets.length} coordinator(s) without waiting for an idle edge`);
997
+ }
998
+ return delivered;
999
+ }
1000
+
1001
+ // One reconcile tick. Two independent phases:
1002
+ //
1003
+ // PHASE 1 — Remote queue pull (the fix for remote worktree completions never
1004
+ // reaching an MCP/LLM coordinator). For EVERY mesh this daemon hosts/
1005
+ // coordinates, pull each remote worker node's pending-events queue over P2P
1006
+ // into THIS daemon's local queue. This runs *regardless of whether a live CLI
1007
+ // coordinator exists* — the coordinator is usually a pure stdio MCP LLM with
1008
+ // no live CLI session, and it can only observe a remote worker's completion
1009
+ // once that event has been pulled into the local queue (which it then drains
1010
+ // on its next mesh tool call). Previously this pull was gated behind a live
1011
+ // CLI coordinator and so never ran for MCP/LLM coordinators — remote
1012
+ // completions sat on the remote node's queue until the LLM happened to call
1013
+ // mesh_read_chat, which triggered the MCP-side pull. The daemon now does it
1014
+ // autonomously on the timer. Standalone (no dispatchMeshCommand) skips this
1015
+ // phase entirely — there are no remote nodes to pull from.
1016
+ //
1017
+ // PHASE 2 — Live CLI inject. For each mesh that has a live CLI coordinator on
1018
+ // THIS daemon, drain the local queue and inject pending events into the PTY.
1019
+ // Unchanged from before.
1020
+ // Bug B: how long a row may sit 'assigned' with an unconfirmed dispatch before the
1021
+ // watchdog reclaims it. Must be comfortably larger than the per-dispatch confirm
1022
+ // timeout (DISPATCH_CONFIRM_TIMEOUT_MS in mesh-events-coordinator) so a slow-but-live
1023
+ // dispatch still inside its normal confirm window is never reclaimed early — this is
1024
+ // the durable backstop for the case the in-process confirm timer can't cover (a timer
1025
+ // lost to a daemon restart between claim and confirm).
1026
+ const ASSIGNED_STRANDED_DEADLINE_MS = 5 * 60_000;
1027
+
1028
+ // COMPLETION-PROPAGATION F3: how long a row may sit 'assigned' with a CONFIRMED delivery
1029
+ // (delivered/acked) but no terminal completion before the watchdog reclaims it as a
1030
+ // delivered-but-lost completion. Distinct from — and deliberately larger than —
1031
+ // ASSIGNED_STRANDED_DEADLINE_MS: a confirmed-delivered dispatch was genuinely handed to a
1032
+ // worker, so the deadline must comfortably exceed any realistic single worker turn (a large
1033
+ // generation) before we treat the missing completion as lost and re-open the task. Paired with
1034
+ // the non-generating + no-terminal-ledger guards below so a worker still mid-turn is never
1035
+ // reclaimed out from under itself.
1036
+ const DELIVERED_NO_TURN_DEADLINE_MS = 15 * 60_000;
1037
+
1038
+ // RECLAIM-FALSEPOS: how many CONSECUTIVE UNKNOWN busy-verdict ticks (past the delivered-no-turn
1039
+ // deadline) must accumulate before a delivered row whose worker session cannot be positively
1040
+ // observed is reclaimed. An UNKNOWN verdict means the assigned session is not present in THIS
1041
+ // daemon's local instance map (remote / gone / id-form skew) — so it may be a REMOTE session that
1042
+ // is genuinely mid-turn. Reclaiming it on a single UNKNOWN tick tears a live remote worker off its
1043
+ // task and re-launches a near-duplicate (observed live 2026-07-04, session 21e34616 / task
1044
+ // a26806c1). We therefore DEFER on UNKNOWN and only reclaim after this bounded grace, so a
1045
+ // transient/remote absence never triggers a false reclaim while a genuinely-lost completion is
1046
+ // still eventually recovered. A GENERATING or IDLE_CONFIRMED verdict (locally-present positive
1047
+ // evidence) resets/bypasses the grace — see recoverStrandedAssignedDispatches.
1048
+ const RECLAIM_UNKNOWN_GRACE_TICKS = 3;
1049
+
1050
+ // Per-row consecutive-UNKNOWN streak for delivered-no-turn reclaim, keyed `${meshId}::${taskId}`.
1051
+ // In-memory (per process); pruned each pass to the set of currently-assigned rows so a
1052
+ // completed/reclaimed/claimed-elsewhere row's counter is dropped (no unbounded growth).
1053
+ const deliveredNoTurnUnknownStreak = new Map<string, number>();
1054
+
1055
+ // Test hook: clear the delivered-no-turn UNKNOWN streak between cases.
1056
+ export function __resetReclaimUnknownStreakForTests(): void {
1057
+ deliveredNoTurnUnknownStreak.clear();
1058
+ }
1059
+
1060
+ // PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
1061
+ // flips a row to 'assigned' BEFORE the fire-and-forget dispatch runs. If that dispatch
1062
+ // neither rejects (→ no .catch requeue) nor is confirmed delivered — a relay that hangs
1063
+ // without acking, or a confirm timer lost across a restart — the row stays 'assigned'
1064
+ // forever: it contributes 0 pending, so PHASE 3 (gated on pendingQueueTaskCount>0) never
1065
+ // re-examines it, and nothing but a manual requeue clears it. This is that missing net.
1066
+ //
1067
+ // Regression guard: a row whose delivery IS confirmed (delivered/acked/completed) is a
1068
+ // genuinely in-flight (or completion-lost) task — left to PHASE 4's completion reconcile,
1069
+ // never reclaimed here. And the deadline is generous so a slow-but-live dispatch still in
1070
+ // its normal confirm window is never reclaimed early. Reclaimed rows return to 'pending'
1071
+ // with ownership cleared, so the PHASE 3 trigger below re-dispatches them this same tick.
1072
+ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId: string, store: MeshRuntimeStore): void {
1073
+ const assigned = getQueue(meshId, { status: ['assigned'] });
1074
+ if (!assigned.length) return;
1075
+ const nowMs = Date.now();
1076
+ // RECLAIM-FALSEPOS: prune UNKNOWN streaks for rows of THIS mesh that are no longer
1077
+ // 'assigned' (completed / reclaimed / claimed elsewhere) so the counter map cannot grow
1078
+ // unbounded and a re-used task id starts its grace fresh.
1079
+ const assignedKeys = new Set(assigned.map(r => `${meshId}::${r.id}`));
1080
+ const meshKeyPrefix = `${meshId}::`;
1081
+ for (const key of [...deliveredNoTurnUnknownStreak.keys()]) {
1082
+ if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) deliveredNoTurnUnknownStreak.delete(key);
1083
+ }
1084
+ for (const row of assigned) {
1085
+ const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
1086
+ if (!Number.isFinite(dispatchedAtMs)) continue; // no dispatch ts → can't age it
1087
+ if (nowMs - dispatchedAtMs < ASSIGNED_STRANDED_DEADLINE_MS) continue; // still in confirm window
1088
+ const terminal = findTerminalLedgerEvidenceForTask({
1089
+ meshId,
1090
+ taskId: row.id,
1091
+ });
1092
+ if (terminal) {
1093
+ const status = terminal.kind === 'task_completed' ? 'completed' : 'failed';
1094
+ updateTaskStatus(meshId, row.id, status);
1095
+ LOG.warn('MeshReconcile', `Skipped stranded reclaim redispatch for terminal task ${row.id} on mesh ${meshId}; ${terminal.kind} ledger evidence already exists`);
1096
+ traceMeshEventDrop('assigned_stranded_terminal_ledger', {
1097
+ taskId: row.id,
1098
+ sessionId: row.assignedSessionId,
1099
+ nodeId: row.assignedNodeId,
1100
+ meshId,
1101
+ event: 'agent:generating_completed',
1102
+ }, terminal.kind);
1103
+ continue;
1104
+ }
1105
+ if (store.taskHasConfirmedDelivery(meshId, row.id)) {
1106
+ // COMPLETION-PROPAGATION F3 (delivered-but-lost completion): the dispatch WAS
1107
+ // confirmed handed to a worker (delivered/acked) but no terminal completion ever
1108
+ // landed and none is in the ledger (checked just above). Normally this is PHASE 4's
1109
+ // job, but PHASE 4 only covers direct-dispatch rows / a live re-read; a claim-path
1110
+ // queue row whose completion event was lost (the manual-launch flip-miss signature)
1111
+ // sits 'assigned' forever.
1112
+ //
1113
+ // RECLAIM-FALSEPOS tri-state verdict: the reclaim used to gate ONLY on
1114
+ // isSessionActivelyGenerating(), whose local instance lookup returns "not generating"
1115
+ // for a REMOTE (or id-form-skewed) session that is genuinely mid-turn — so such a
1116
+ // worker was reclaimed at the deadline and re-launched same tick (near-duplicate
1117
+ // execution; observed live 2026-07-04, session 21e34616 / task a26806c1). Resolve an
1118
+ // explicit GENERATING / IDLE_CONFIRMED / UNKNOWN verdict instead:
1119
+ // - GENERATING → worker demonstrably alive; never reclaim, reset grace.
1120
+ // - IDLE_CONFIRMED → positive LOCAL evidence (present instance, inactive) → reclaim
1121
+ // now (past deadline) with the delivered-no-turn reason.
1122
+ // - UNKNOWN → session not locally observable (remote / gone / id-skew). Do
1123
+ // NOT fold into a definitive idle. DEFER: count consecutive
1124
+ // UNKNOWN ticks and only reclaim after RECLAIM_UNKNOWN_GRACE_TICKS
1125
+ // so a live remote worker is never torn off its task on a single
1126
+ // absent observation; a genuinely-lost completion is still
1127
+ // recovered after the bounded grace.
1128
+ // reclaimStrandedAssignedTask ends the single-flight window (F4), so a subsequent
1129
+ // re-dispatch/requeue is unblocked.
1130
+ if (nowMs - dispatchedAtMs < DELIVERED_NO_TURN_DEADLINE_MS) continue; // still within turn budget
1131
+ const streakKey = `${meshId}::${row.id}`;
1132
+ const verdict = row.assignedSessionId
1133
+ ? resolveSessionBusyVerdict(components, row.assignedSessionId)
1134
+ : 'IDLE_CONFIRMED'; // no session bound → nothing live to protect
1135
+ if (verdict === 'GENERATING') {
1136
+ deliveredNoTurnUnknownStreak.delete(streakKey); // demonstrably alive → reset grace
1137
+ continue; // worker still working
1138
+ }
1139
+ let reclaimReason: 'delivered_no_turn_deadline' | 'reclaim_after_unknown_grace';
1140
+ if (verdict === 'IDLE_CONFIRMED') {
1141
+ deliveredNoTurnUnknownStreak.delete(streakKey);
1142
+ reclaimReason = 'delivered_no_turn_deadline';
1143
+ } else {
1144
+ // UNKNOWN — defer and accumulate the consecutive-UNKNOWN streak.
1145
+ const streak = (deliveredNoTurnUnknownStreak.get(streakKey) ?? 0) + 1;
1146
+ deliveredNoTurnUnknownStreak.set(streakKey, streak);
1147
+ if (streak < RECLAIM_UNKNOWN_GRACE_TICKS) {
1148
+ // Still within grace — hold this tick. Content-free trace (ids + streak only).
1149
+ traceMeshEventDrop('reclaim_deferred_unknown_verdict', {
1150
+ taskId: row.id,
1151
+ sessionId: row.assignedSessionId,
1152
+ nodeId: row.assignedNodeId,
1153
+ meshId,
1154
+ event: 'agent:generating_completed',
1155
+ }, `unknown ${streak}/${RECLAIM_UNKNOWN_GRACE_TICKS}`);
1156
+ continue;
1157
+ }
1158
+ reclaimReason = 'reclaim_after_unknown_grace';
1159
+ }
1160
+ const reclaimedLost = reclaimStrandedAssignedTask(meshId, row.id, {
1161
+ reason: reclaimReason,
1162
+ ageMs: nowMs - dispatchedAtMs,
1163
+ });
1164
+ if (reclaimedLost) {
1165
+ deliveredNoTurnUnknownStreak.delete(streakKey);
1166
+ LOG.warn('MeshReconcile', `Reclaimed delivered-but-lost task ${row.id} on mesh ${meshId} `
1167
+ + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, delivered but no `
1168
+ + `completion in ${Math.round((nowMs - dispatchedAtMs) / 1000)}s, verdict ${verdict} → ${reclaimReason} → ${reclaimedLost.status})`);
1169
+ traceMeshEventDrop('assigned_stranded_delivered_no_turn', {
1170
+ taskId: row.id,
1171
+ sessionId: row.assignedSessionId,
1172
+ nodeId: row.assignedNodeId,
1173
+ meshId,
1174
+ event: 'agent:generating_completed',
1175
+ }, `delivered ${Math.round((nowMs - dispatchedAtMs) / 1000)}s ${reclaimReason} → ${reclaimedLost.status}`);
1176
+ }
1177
+ continue;
1178
+ }
1179
+ const reclaimed = reclaimStrandedAssignedTask(meshId, row.id, {
1180
+ reason: 'assigned_stranded_dispatch_unconfirmed',
1181
+ ageMs: nowMs - dispatchedAtMs,
1182
+ });
1183
+ if (reclaimed) {
1184
+ LOG.warn('MeshReconcile', `Reclaimed stranded assigned task ${row.id} on mesh ${meshId} `
1185
+ + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, dispatched `
1186
+ + `${Math.round((nowMs - dispatchedAtMs) / 1000)}s ago, never confirmed delivered → ${reclaimed.status})`);
1187
+ // EVTTRACE: the dispatch for this task was stranded (assigned, never confirmed
1188
+ // delivered) and reclaimed (CANON-B) — its expected completion event never
1189
+ // arrived. Observation only; the reclaim decision above is unchanged.
1190
+ traceMeshEventDrop('assigned_stranded_reclaim', {
1191
+ taskId: row.id,
1192
+ sessionId: row.assignedSessionId,
1193
+ nodeId: row.assignedNodeId,
1194
+ meshId,
1195
+ event: 'agent:generating_completed',
1196
+ }, `unconfirmed ${Math.round((nowMs - dispatchedAtMs) / 1000)}s → ${reclaimed.status}`);
1197
+ }
1198
+ }
1199
+ }
1200
+
1201
+ export async function runMeshReconcileTick(components: DaemonComponents): Promise<void> {
1202
+ const localDaemonId = readNonEmptyString(loadConfig().machineId) || undefined;
1203
+ // The id-set used to scope the local queue drain (status id + machineId). See
1204
+ // resolveCoordinatorDaemonIds — the status id is what the MCP layer stamps and
1205
+ // is mandatory here for a generating CLI coordinator to self-receive completions.
1206
+ const drainDaemonIds = resolveCoordinatorDaemonIds(components);
1207
+ const dispatchMeshCommand = components.dispatchMeshCommand;
1208
+ const store = (() => {
1209
+ try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
1210
+ })();
1211
+
1212
+ // ── PHASE 0: retry the worker-side unresolved-delegate forward outbox ──────
1213
+ // Cloud-only (needs dispatchMeshCommand). A worker that is NOT a member of the
1214
+ // coordinator's mesh cannot be reached by the coordinator's PHASE 1 pull (it is
1215
+ // in no mesh.node), so its completion must be PUSHED to the coordinator. This
1216
+ // drains the durable outbox enqueued by forwardUnresolvedDelegateEvent and retries
1217
+ // any push that has not yet been acked. See mesh-unresolved-forward-outbox.ts.
1218
+ if (dispatchMeshCommand) {
1219
+ try {
1220
+ await retryUnresolvedDelegateForwards(components);
1221
+ } catch (e: any) {
1222
+ LOG.warn('MeshReconcile', `Unresolved-delegate forward retry failed: ${e?.message || e}`);
1223
+ }
1224
+ }
1225
+
1226
+ // ── PHASE 1: pull remote node queues for every mesh this daemon hosts ──────
1227
+ // Cloud-only (dispatchMeshCommand present). Runs whether or not a live CLI
1228
+ // coordinator exists — this is what lets an MCP/LLM coordinator ever see a
1229
+ // remote worker's completion.
1230
+ if (dispatchMeshCommand) {
1231
+ for (const mesh of listMeshes()) {
1232
+ // Expand to every id-form this daemon answers to for this mesh (runtime
1233
+ // drain ids ∪ config-form node/host ids) and use it for BOTH the host gate
1234
+ // and the remote pull filter, so a worker stamp in any form is recovered.
1235
+ const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1236
+ if (!daemonHostsMesh(mesh, selfIds)) continue;
1237
+ try {
1238
+ await pullRemoteNodeQueues(components, mesh, localDaemonId, selfIds);
1239
+ } catch (e: any) {
1240
+ LOG.warn('MeshReconcile', `Remote node pull failed for mesh ${mesh.id}: ${e?.message || e}`);
1241
+ }
1242
+ }
1243
+ }
1244
+
1245
+ // ── PHASE 2.5: assigned-stranded dispatch watchdog (Bug B) ─────────────────
1246
+ // Runs before PHASE 3 so any row it returns to 'pending' is re-dispatched by the
1247
+ // PHASE 3 trigger in this same tick. See recoverStrandedAssignedDispatches.
1248
+ if (store) {
1249
+ for (const mesh of listMeshes()) {
1250
+ const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1251
+ if (!daemonHostsMesh(mesh, selfIds)) continue;
1252
+ try {
1253
+ recoverStrandedAssignedDispatches(components, mesh.id, store);
1254
+ } catch (e: any) {
1255
+ LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
1256
+ }
1257
+ }
1258
+ }
1259
+
1260
+ // ── PHASE 3: recover pending queue claims for newly-idle sessions ──────────
1261
+ // The event-driven claim paths (agent:ready / agent:generating_completed in
1262
+ // mesh-events-coordinator) re-claim the queue the moment a session goes idle,
1263
+ // but that depends on a single event being emitted AND (for a remote node)
1264
+ // successfully forwarded to this coordinator. If that event is missed/dropped,
1265
+ // a pending task targeting a now-idle session would sit unclaimed forever —
1266
+ // there was no periodic safety net. This phase is that net: for every mesh this
1267
+ // daemon hosts that has at least one pending task, run one triggerMeshQueue so a
1268
+ // session that became idle without a delivered ready-event still gets its work.
1269
+ //
1270
+ // O(1) guard: skip the (relatively expensive) full idle-session + remote-idle
1271
+ // scan entirely when the queue has no pending tasks — a COUNT(*) over the
1272
+ // indexed status column, so an idle mesh costs one cheap query per tick.
1273
+ // claimNextQueueTask is atomic, so racing the event-driven path can only have
1274
+ // one winner; double-claiming is impossible.
1275
+ for (const mesh of listMeshes()) {
1276
+ const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1277
+ if (!daemonHostsMesh(mesh, selfIds)) continue;
1278
+ if (store) {
1279
+ try {
1280
+ if (store.pendingQueueTaskCount(mesh.id) === 0) continue;
1281
+ } catch { /* fall through and let triggerMeshQueue decide */ }
1282
+ }
1283
+ try {
1284
+ await triggerMeshQueue(components, mesh.id);
1285
+ } catch (e: any) {
1286
+ LOG.warn('MeshReconcile', `Pending-claim recovery trigger failed for mesh ${mesh.id}: ${e?.message || e}`);
1287
+ }
1288
+ }
1289
+
1290
+ // ── PHASE 4: synthesize lost completions for unterminated direct dispatches ─
1291
+ // Symmetric to PHASE 3 (which recovers a *lost claim* for a newly-idle session)
1292
+ // but for the opposite gap: a worker that ALREADY completed, went idle, and
1293
+ // whose terminal completion event was never persisted (dropped before reaching
1294
+ // the queue/outbox, or its forward was lost). PHASE 1/2/3 can only deliver an
1295
+ // event that exists in a queue — they cannot recover a completion that was
1296
+ // never recorded, so the coordinator keeps believing the worker is generating.
1297
+ //
1298
+ // reconcileDirectDispatchCompletionFromTranscript already synthesizes the
1299
+ // missing terminal event from the worker's transcript, but until now it ran
1300
+ // ONLY when an LLM coordinator polled mesh_status (mcp_mesh_status_transcript_
1301
+ // reconciliation). This phase pulls that same correction onto the daemon timer
1302
+ // so it no longer depends on the LLM polling. The reconcile is idempotent
1303
+ // (hasTerminalLedgerAfterDispatch guards against re-synthesis), so attempting it
1304
+ // every tick for the same dispatch is safe — once a terminal exists it no-ops.
1305
+ for (const mesh of listMeshes()) {
1306
+ const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1307
+ if (!daemonHostsMesh(mesh, selfIds)) continue;
1308
+ try {
1309
+ await reconcileUnterminatedDirectDispatches(components, mesh, selfIds, localDaemonId);
1310
+ } catch (e: any) {
1311
+ LOG.warn('MeshReconcile', `Completion reconcile failed for mesh ${mesh.id}: ${e?.message || e}`);
1312
+ }
1313
+ }
1314
+
1315
+ // ── PHASE 5: auto-prune orphaned direct dispatch records ───────────────────
1316
+ // staleDirectWork (orphaned direct-dispatch rows whose node/session is no longer in the
1317
+ // live mesh) otherwise accumulates indefinitely: a removed worktree node or a cleanly
1318
+ // terminated session leaves its direct-dispatch row behind, stuck in a non-terminal status
1319
+ // (e.g. generating) for days. This is NOT a false-idle bug — it is the separate problem of
1320
+ // orphaned records that the only existing cleanup path (manual MCP mesh_prune_stale_direct)
1321
+ // never reaches unless an operator runs it by hand.
1322
+ //
1323
+ // This phase runs the SAME prune core the manual tool calls (pruneStaleDirectDispatches),
1324
+ // in execute mode, on the daemon timer. The only difference from the manual path is a
1325
+ // conservative age gate (DEFAULT_AUTO_PRUNE_MIN_AGE_MS): a freshly-orphaned record is held
1326
+ // back until it is provably stale, so a transient probe miss never auto-prunes live work.
1327
+ // Every other safety rule is inherited unchanged from the core — active/pending/generating
1328
+ // work and fresh unacknowledged dispatch failures are never pruned, ledger-only audit entries
1329
+ // are preserved, and the prune itself is recorded with a direct_dispatch_pruned ledger entry.
1330
+ // Idempotent: a pruned row is gone from getActiveDirectDispatches, so the next tick finds
1331
+ // nothing to re-prune. Isolated in its own try/catch per mesh so it can never kill the tick.
1332
+ {
1333
+ const minAgeMs = resolveAutoPruneMinAgeMs();
1334
+ for (const mesh of listMeshes()) {
1335
+ const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1336
+ if (!daemonHostsMesh(mesh, selfIds)) continue;
1337
+ try {
1338
+ await autoPruneStaleDirectDispatches(components, mesh, selfIds, localDaemonId, minAgeMs);
1339
+ } catch (e: any) {
1340
+ LOG.warn('MeshReconcile', `Auto-prune stale direct failed for mesh ${mesh.id}: ${e?.message || e}`);
1341
+ }
1342
+ }
1343
+ }
1344
+
1345
+ // ── PHASE 2: inject into live CLI coordinators on this daemon ──────────────
1346
+ const coordinators = findLiveCoordinators(components);
1347
+ if (coordinators.length === 0) {
1348
+ // No live CLI coordinator on this daemon — nothing to inject into.
1349
+ // (MCP-only LLM coordinators drain the local queue via their own tool
1350
+ // calls; PHASE 1 above has already populated it from remote nodes.)
1351
+ return;
1352
+ }
1353
+
1354
+ // Group coordinators by mesh; multiple coordinator instances for one mesh is
1355
+ // unusual but supported (each gets the same drained events).
1356
+ const byMesh = new Map<string, LiveCoordinator[]>();
1357
+ for (const c of coordinators) {
1358
+ const list = byMesh.get(c.meshId);
1359
+ if (list) list.push(c);
1360
+ else byMesh.set(c.meshId, [c]);
1361
+ }
1362
+
1363
+ for (const [meshId, meshCoordinators] of byMesh) {
1364
+ // Drain the local queue scoped to this coordinator daemon and inject.
1365
+ // - If an idle coordinator exists, FULL-drain and deliver every event to it
1366
+ // (the idle input box accepts the prompt as a real next turn). The drain
1367
+ // marks consumed rows drained=1 atomically, so the pull path can't re-deliver.
1368
+ // - If only GENERATING coordinators exist (no idle target), we HOLD: leave the
1369
+ // events queued (drained=0) for the coordinator's next idle/turn-end tick.
1370
+ //
1371
+ // NOTIF-SURFACE-LOCAL (false-idle hold): we used to force-inject terminal events
1372
+ // (completion/approval/stop/refine·bootstrap) straight into a *generating*
1373
+ // coordinator's PTY (forceSendMessage → atomic content+\r write), on the theory it
1374
+ // bypassed the busy send-guard and broke the await-result deadlock. But a raw PTY
1375
+ // write into a claude-cli that is mid-generation is NOT consumed as a new turn — the
1376
+ // bytes land in the terminal input buffer and the LLM never reads them on its next
1377
+ // turn. The `surfaced/force-inject` trace fired, the row was marked drained=1, and the
1378
+ // genuine completion was lost forever (the exact same-daemon local-worktree miss: the
1379
+ // coordinator's OWN session is generating at the moment its worker completes). The
1380
+ // deadlock the force path guarded against does not actually require force: a
1381
+ // coordinator that dispatched a task via mesh_send_task returns to idle when that
1382
+ // tool call resolves (dispatch is fire-and-forget; the worker runs for minutes while
1383
+ // the coordinator is idle/between turns), so the completion lands on the very next
1384
+ // idle tick (≤ one reconcile interval). Holding the event undrained for that idle
1385
+ // tick is therefore the single, reliable delivery — and it is the SAME skip-and-hold
1386
+ // the modal-park branch below already uses. This also makes double-injection
1387
+ // structurally impossible: there is exactly one delivery path (the idle full-drain),
1388
+ // so we never need a surface-time fingerprint to dedup a force-write against a re-drain.
1389
+ const idleCoordinators = meshCoordinators.filter(c => c.idle);
1390
+ // A coordinator parked on a harness modal (waiting_choice / waiting_approval) is
1391
+ // non-idle; it is held under the modal-park branch (a force-inject into a modal would
1392
+ // write raw keystrokes the modal key handler eats, silently selecting a choice the
1393
+ // user never made). A plainly-generating coordinator (non-idle, non-modal-parked) is
1394
+ // ALSO held now — for the false-idle reason above — but separately, so the C1 ledger
1395
+ // audit and the operator-facing skip log can name the right hold reason.
1396
+ const generatingCoordinators = meshCoordinators.filter(c => !c.idle && !c.modalParked);
1397
+ const modalParkedCoordinators = meshCoordinators.filter(c => !c.idle && c.modalParked);
1398
+ // Only an IDLE coordinator is a deliverable target. A generating coordinator's PTY
1399
+ // does not consume an injected prompt as a turn, so it is held (not a target).
1400
+ const targetCoordinators = idleCoordinators;
1401
+
1402
+ // ── no-idle-target short-circuit (MUST precede the drain) ─────────────────
1403
+ // When there is no IDLE coordinator for this mesh — only generating and/or
1404
+ // modal-parked ones — there is nowhere a queued event can land as a real turn.
1405
+ // We skip-and-hold: by NOT draining we leave the events at drained=0 in the queue,
1406
+ // so a later tick (once a coordinator returns to idle) delivers them. This
1407
+ // short-circuit MUST run BEFORE drainPendingMeshCoordinatorEvents — the drain marks
1408
+ // rows drained=1 atomically, which would lose the events for a coordinator that is
1409
+ // only transiently busy (the false-idle local-worktree miss). Both the generating
1410
+ // hold and the modal-park hold record a C1 ledger audit copy so a held completion's
1411
+ // worker summary is recoverable even if the coordinator never returns or the pending
1412
+ // file is later trimmed.
1413
+ if (targetCoordinators.length === 0) {
1414
+ // ── APPROVAL-Q1-REALTIME: level-deliver approval nudges BEFORE the hold ──
1415
+ // Approval events are LEVEL-backed (task_approval_needed ledger →
1416
+ // mesh_status awaiting_approval), so they must not be edge-held like a
1417
+ // completion (whose payload lives only in the pending event). Drain and
1418
+ // deliver them to the busy coordinator's inbox (non-force, next-turn-boundary)
1419
+ // this tick, dropping any already-resolved (stale) nudge — and leave ONLY the
1420
+ // completion/other events in the queue for the existing hold semantics below
1421
+ // (their behaviour is unchanged: shouldForceInjectMeshEvent no longer sees the
1422
+ // approval rows because this drained them). MUST run first so the modal-park
1423
+ // orphan-escape and the generating-hold audit only ever see non-approval events.
1424
+ drainAndDeliverApprovalNudges(meshId, drainDaemonIds, localDaemonId, meshCoordinators);
1425
+ // If approval nudges were the only queued events, nothing remains to hold — skip
1426
+ // the hold branches (and their "holding pending event(s)" log) entirely.
1427
+ if (store) {
1428
+ try { if (store.pendingEventCount(meshId) === 0) continue; } catch { /* fall through */ }
1429
+ }
1430
+ if (modalParkedCoordinators.length > 0) {
1431
+ // ── orphan escape (MUST precede the blanket modal-park hold) ──────────
1432
+ // A modal-parked coordinator with no idle/generating sibling otherwise
1433
+ // wedges EVERY pending event under `modal_parked` until that modal resolves
1434
+ // — including a STRICT-routed completion whose originating coordinator
1435
+ // session is GONE (an orphan: the worktree/session that produced it was
1436
+ // removed, or that coordinator session died). Such an event will never be
1437
+ // deliverable to its target session no matter what the modal-parked sibling
1438
+ // does, so holding it under modal_parked is a permanent-held leak (the very
1439
+ // "data restart re-reproduces it" symptom — the gate is reconstructed live
1440
+ // from the still-parked modal, so a restart does not clear it). Route those
1441
+ // orphan events through the strict-route hold/expire path so the bounded
1442
+ // STRICT_SESSION_MATCH_TTL eventually expires them (recoverable, ledgered)
1443
+ // instead of leaving them held forever. A strict event whose target session
1444
+ // IS live but merely modal-parked is left to the blanket hold below (it is
1445
+ // genuinely transiently blocked, not orphaned).
1446
+ const liveSessionIds = new Set(
1447
+ meshCoordinators.map(c => readNonEmptyString(c.sessionId)).filter(Boolean),
1448
+ );
1449
+ let orphanEscaped = 0;
1450
+ const hasPendingForOrphanPeek = !store
1451
+ || (() => { try { return store.pendingEventCount(meshId) > 0; } catch { return true; } })();
1452
+ if (hasPendingForOrphanPeek) {
1453
+ // Identify which pending event NAMES correspond to orphan-targeted events
1454
+ // (a strict targetCoordinatorSessionId that matches no live coordinator).
1455
+ let peeked: readonly PendingMeshCoordinatorEvent[] = [];
1456
+ try {
1457
+ peeked = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
1458
+ } catch { peeked = []; }
1459
+ const isOrphan = (e: PendingMeshCoordinatorEvent): boolean => {
1460
+ const want = readNonEmptyString(e.targetCoordinatorSessionId);
1461
+ return !!want && !liveSessionIds.has(want);
1462
+ };
1463
+ const orphanEventNames = new Set(peeked.filter(isOrphan).map(e => e.event));
1464
+ if (orphanEventNames.size > 0) {
1465
+ // The drain filter is event-NAME scoped (not per-row), so draining by the
1466
+ // orphan event names also pulls any non-orphan event sharing that name. Drain
1467
+ // them all, then re-route: orphan-targeted events go through the strict-route
1468
+ // hold/expire path (bounded TTL → eventually ledger-expired, recoverable);
1469
+ // non-orphan events of the same name are re-queued unchanged (queuedAt
1470
+ // preserved) so they remain genuinely held for their still-live, modal-parked
1471
+ // target. This is the same per-event strict routing PHASE 2 does below — just
1472
+ // reached here because the blanket modal-park short-circuit would otherwise
1473
+ // wedge the orphans forever.
1474
+ let drained: PendingMeshCoordinatorEvent[] = [];
1475
+ try {
1476
+ drained = drainPendingMeshCoordinatorEvents(
1477
+ meshId,
1478
+ drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
1479
+ { onlyEvents: orphanEventNames },
1480
+ );
1481
+ } catch (e: any) {
1482
+ LOG.warn('MeshReconcile', `Orphan-escape drain failed for mesh ${meshId}: ${e?.message || e}`);
1483
+ drained = [];
1484
+ }
1485
+ for (const pending of drained) {
1486
+ if (isOrphan(pending)) {
1487
+ holdOrExpireStrictUnmatchedEvent(pending, readNonEmptyString(pending.targetCoordinatorSessionId), meshId);
1488
+ orphanEscaped++;
1489
+ } else {
1490
+ // Still-live (modal-parked) target — re-queue unchanged so it is held
1491
+ // for the next modal-resolved tick, exactly like the blanket hold would.
1492
+ try { queuePendingMeshCoordinatorEvent(pending); } catch { /* best-effort re-queue */ }
1493
+ }
1494
+ }
1495
+ }
1496
+ }
1497
+ LOG.info('MeshReconcile', `Reconcile skip → modal-parked: holding pending event(s) for mesh ${meshId} (${modalParkedCoordinators.length} coordinator(s) awaiting a modal answer; events left queued${orphanEscaped > 0 ? `; ${orphanEscaped} orphan-targeted event(s) routed to strict-route TTL` : ''})`);
1498
+ // NOTIF (B) diagnostic: name the session(s) classified modal-parked so the
1499
+ // same-tick coordDiag line (paired by sessionId) shows whether the modal-park
1500
+ // overlay is a real human-await or an unreleased mask (the getState_overlay origin).
1501
+ if (getLogLevel() === 'debug') {
1502
+ LOG.debug('MeshReconcile', `coordHoldModalParked mesh=${meshId} heldFor=[${modalParkedCoordinators.map(c => c.sessionId || '?').join(',')}] (these were classified modal-parked; cross-ref same-tick coordDiag by sessionId)`);
1503
+ }
1504
+ // C1: mirror held terminal events into the ledger so a held completion's
1505
+ // worker summary is auditable/recoverable even if the modal is never
1506
+ // resolved, the coordinator restarts, or the pending file is later trimmed.
1507
+ // The events stay queued (drained=0) for re-drain on a later tick; this only
1508
+ // adds the durable audit copy. Idempotent per process — only newly-held
1509
+ // events are logged. O(1)-gated: skip the peek when the queue is empty.
1510
+ let hasPending = true;
1511
+ if (store) {
1512
+ try { hasPending = store.pendingEventCount(meshId) > 0; } catch { /* peek below */ }
1513
+ }
1514
+ if (hasPending) {
1515
+ recordHeldTerminalEventsToLedger(
1516
+ meshId,
1517
+ drainDaemonIds.length > 0 ? drainDaemonIds : (localDaemonId ? [localDaemonId] : []),
1518
+ 'modal_parked',
1519
+ modalParkedCoordinators.length,
1520
+ );
1521
+ }
1522
+ } else if (generatingCoordinators.length > 0) {
1523
+ // ── generating hold (NOTIF-SURFACE-LOCAL false-idle fix) ─────────────
1524
+ // The only coordinator(s) for this mesh are plainly generating (no idle, no
1525
+ // modal). A raw force-write into a generating claude-cli PTY is not consumed
1526
+ // as a turn, so we do NOT inject and do NOT drain — the events stay queued
1527
+ // (drained=0) and the next tick that finds the coordinator idle full-drains
1528
+ // them as real turns (the coordinator returns to idle when its current
1529
+ // tool-call/turn resolves; a dispatched worker runs for minutes while the
1530
+ // coordinator is idle, so this lands within one reconcile interval). C1: mirror
1531
+ // any held terminal events into the ledger so a completion's worker summary is
1532
+ // recoverable even before that idle tick. Idempotent per process; O(1)-gated.
1533
+ let hasPending = true;
1534
+ if (store) {
1535
+ try { hasPending = store.pendingEventCount(meshId) > 0; } catch { /* peek below */ }
1536
+ }
1537
+ if (hasPending) {
1538
+ // ── PTY-OVERTRUST-DRAIN (Defect B, fix B): age-based escape ───────────
1539
+ // Fix A already routes the common mask-driven false-busy to the idle path,
1540
+ // so reaching here means the coordinator's RAW adapter reads generating.
1541
+ // That is almost always genuine — but a status-source desync fix A does not
1542
+ // reach can momentarily make the raw adapter read generating while the PTY
1543
+ // is actually at a turn end, stranding the completion across many ticks. As a
1544
+ // TIME-BASED BACKSTOP, once the oldest held terminal event has aged past the
1545
+ // escalate threshold, RE-CONFIRM each held coordinator's raw adapter idle and,
1546
+ // if genuinely idle, drain ONCE into it. The re-confirmation gate is what makes
1547
+ // this safe: it NEVER injects into a genuinely-generating PTY (that is the
1548
+ // data-loss force-inject path intentionally removed). A coordinator still
1549
+ // genuinely generating stays held.
1550
+ const escalateMs = resolvePendingHeldDrainEscalateMs();
1551
+ const heldAgeMs = oldestHeldTerminalEventAgeMs(
1552
+ meshId,
1553
+ drainDaemonIds.length > 0 ? drainDaemonIds : (localDaemonId ? [localDaemonId] : []),
1554
+ );
1555
+ if (heldAgeMs >= escalateMs) {
1556
+ const escapeTargets = reconfirmGenuinelyIdleCoordinators(generatingCoordinators);
1557
+ if (escapeTargets.length > 0) {
1558
+ LOG.info('MeshReconcile', `Reconcile age-escape → generating-hold: held terminal event(s) for mesh ${meshId} aged ${Math.round(heldAgeMs / 1000)}s (≥ ${Math.round(escalateMs / 1000)}s) and ${escapeTargets.length} coordinator(s) re-confirmed genuinely idle on the raw adapter — draining once`);
1559
+ const drained = drainAndInjectIntoTargets(meshId, drainDaemonIds, localDaemonId, escapeTargets, 'age-escape');
1560
+ if (drained > 0) continue; // delivered → no hold this tick
1561
+ }
1562
+ }
1563
+ LOG.info('MeshReconcile', `Reconcile skip → generating: holding pending event(s) for mesh ${meshId} (${generatingCoordinators.length} coordinator(s) busy; events left queued for the next idle tick)`);
1564
+ // NOTIF (B) diagnostic: this is the hold that strands the completion. Name
1565
+ // the sessionId(s) the loop just classified non-idle/non-modal so the
1566
+ // same-tick coordDiag line above (paired by sessionId) reveals which status
1567
+ // source diverged. If a coordDiag for one of these sessions shows getState
1568
+ // (or lastStatus/adapterRaw) === idle, that is the runtime desync origin.
1569
+ if (getLogLevel() === 'debug') {
1570
+ LOG.debug('MeshReconcile', `coordHoldGenerating mesh=${meshId} heldFor=[${generatingCoordinators.map(c => c.sessionId || '?').join(',')}] (these were classified busy; cross-ref same-tick coordDiag by sessionId)`);
1571
+ }
1572
+ recordHeldTerminalEventsToLedger(
1573
+ meshId,
1574
+ drainDaemonIds.length > 0 ? drainDaemonIds : (localDaemonId ? [localDaemonId] : []),
1575
+ 'generating_no_idle_coordinator',
1576
+ generatingCoordinators.length,
1577
+ );
1578
+ }
1579
+ }
1580
+ continue;
1581
+ }
1582
+
1583
+ // O(1) guard: skip the drain entirely when the queue is empty.
1584
+ if (store) {
1585
+ try {
1586
+ if (store.pendingEventCount(meshId) === 0) continue;
1587
+ } catch { /* fall through to drain */ }
1588
+ }
1589
+
1590
+ // An idle coordinator is present (targetCoordinators.length > 0): FULL-drain every
1591
+ // queued event and deliver it to the idle input box as a real turn. The no-idle case
1592
+ // (generating/modal-only) was already held above and never reaches here, so there is
1593
+ // no force-drain-into-generating path left — the single delivery is the idle drain.
1594
+ drainAndInjectIntoTargets(meshId, drainDaemonIds, localDaemonId, targetCoordinators, 'idle');
1595
+ }
1596
+ }
1597
+
1598
+ // Strict-routing TTL: how long a drained completion whose originating coordinator session
1599
+ // is not currently deliverable is held (re-queued for re-drain) before it is ledger-
1600
+ // expired. Bounded so a coordinator session that never returns cannot wedge the event
1601
+ // forever; broad enough to ride out a transient modal-park / brief restart.
1602
+ const STRICT_SESSION_MATCH_TTL_MS = 60_000;
1603
+
1604
+ // Re-queue (hold) a strict-routed event whose coordinator session is not live, or — once it
1605
+ // has aged past STRICT_SESSION_MATCH_TTL_MS — ledger-expire it (recoverable) and drop it.
1606
+ // We deliberately do NOT broadcast an aged-out event to sibling coordinators: that is the
1607
+ // very misroute strict routing exists to prevent. The drain already marked the row drained=1,
1608
+ // so re-queuing re-persists a fresh undrained copy (dedup keys on drained=0 only); queuedAt is
1609
+ // preserved so the TTL measures the event's true age across re-queues.
1610
+ function holdOrExpireStrictUnmatchedEvent(
1611
+ pending: PendingMeshCoordinatorEvent,
1612
+ wantSession: string,
1613
+ meshId: string,
1614
+ ): void {
1615
+ const queuedAt = typeof pending.queuedAt === 'number' ? pending.queuedAt : Date.now();
1616
+ if (Date.now() - queuedAt <= STRICT_SESSION_MATCH_TTL_MS) {
1617
+ try {
1618
+ queuePendingMeshCoordinatorEvent(pending); // preserves queuedAt → true age retained
1619
+ LOG.info('MeshReconcile', `Strict route hold: coordinator session ${wantSession} not live on mesh ${meshId} — re-queued (${pending.event})`);
1620
+ // EVTTRACE: event held (re-queued) — its originating coordinator session is not
1621
+ // currently deliverable. Held, not dropped; surfaces later or expires past TTL.
1622
+ traceMeshEventDrop('strict_route_hold', {
1623
+ taskId: pending.metadataEvent?.taskId,
1624
+ sessionId: pending.metadataEvent?.targetSessionId ?? wantSession,
1625
+ nodeId: pending.nodeId,
1626
+ meshId,
1627
+ event: pending.event,
1628
+ }, `coordinatorSession=${wantSession} not live`);
1629
+ } catch (e: any) {
1630
+ LOG.warn('MeshReconcile', `Strict route re-queue failed for ${pending.event} on mesh ${meshId}: ${e?.message || e}`);
1631
+ }
1632
+ return;
1633
+ }
1634
+ const finalSummary = readMeshCompletionSummary(pending.metadataEvent || {});
1635
+ try {
1636
+ appendLedgerEntry(meshId, {
1637
+ kind: 'event_held',
1638
+ ...(pending.nodeId ? { nodeId: pending.nodeId } : {}),
1639
+ payload: {
1640
+ event: pending.event,
1641
+ reason: 'strict_route_expired',
1642
+ recoverable: true,
1643
+ targetCoordinatorSessionId: wantSession,
1644
+ targetCoordinatorDaemonId: pending.targetCoordinatorDaemonId ?? null,
1645
+ nodeLabel: pending.nodeLabel,
1646
+ ...(pending.workspace ? { workspace: pending.workspace } : {}),
1647
+ queuedAt,
1648
+ ...(finalSummary ? { finalSummary } : {}),
1649
+ },
1650
+ });
1651
+ LOG.warn('MeshReconcile', `Strict route expire: coordinator session ${wantSession} never returned for mesh ${meshId} — recorded to ledger (recoverable), dropped (${pending.event})`);
1652
+ // EVTTRACE: event expired past the strict-route TTL — dropped (recoverable, ledgered).
1653
+ traceMeshEventDrop('strict_route_expired', {
1654
+ taskId: pending.metadataEvent?.taskId,
1655
+ sessionId: pending.metadataEvent?.targetSessionId ?? wantSession,
1656
+ nodeId: pending.nodeId,
1657
+ meshId,
1658
+ event: pending.event,
1659
+ }, `coordinatorSession=${wantSession} never returned`);
1660
+ } catch (e: any) {
1661
+ LOG.warn('MeshReconcile', `Failed to ledger-expire strict-unmatched ${pending.event} for mesh ${meshId}: ${e?.message || e}`);
1662
+ }
1663
+ }
1664
+
1665
+ // Cloud-only: retry the worker-side unresolved-delegate forward outbox. For each
1666
+ // durably-queued entry, push it to its coordinator daemon over P2P (mesh_forward_event)
1667
+ // and ack (mark drained) ONLY on a successful, non-rejected response. A failed or
1668
+ // rejected push leaves the entry queued for the next tick — at-least-once delivery.
1669
+ // Stale entries (coordinator unreachable past the max age) are expired first so the
1670
+ // outbox can't grow without bound. The coordinator dedups duplicate deliveries on its
1671
+ // own fingerprint, so a retry that races the original immediate push is harmless.
1672
+ // RECONCILE-MESHID-DROP: per-entry count of consecutive HARD rejections (the coordinator
1673
+ // returned success:false, e.g. "meshId required"). A rejection means the push was delivered
1674
+ // and deterministically refused — retrying the identical payload every 4s can never succeed,
1675
+ // so it would loop until the 30-minute age expiry, spamming the log the whole time. After
1676
+ // MAX_FORWARD_REJECTIONS such rejections we drop the entry (drain it) with ONE fail-loud
1677
+ // warning. Transient transport failures (the dispatch throws — coordinator momentarily
1678
+ // unreachable) do NOT count here; those legitimately retry until the age expiry. In-memory
1679
+ // (keyed by the durable outbox row id) is sufficient: a daemon restart re-arms the loop, and
1680
+ // the age expiry remains the durable backstop. Cleared whenever an entry is delivered/drained.
1681
+ const unresolvedForwardRejectionCounts = new Map<string, number>();
1682
+ const MAX_FORWARD_REJECTIONS = 5;
1683
+
1684
+ export function __resetUnresolvedForwardRejectionCountsForTests(): void {
1685
+ unresolvedForwardRejectionCounts.clear();
1686
+ }
1687
+
1688
+ async function retryUnresolvedDelegateForwards(components: DaemonComponents): Promise<void> {
1689
+ const dispatchMeshCommand = components.dispatchMeshCommand;
1690
+ if (!dispatchMeshCommand) return;
1691
+
1692
+ // Drop entries that have exhausted their retry budget (fail-loud inside).
1693
+ expireStaleUnresolvedDelegateForwards();
1694
+
1695
+ const entries = peekUnresolvedDelegateForwards();
1696
+ if (entries.length === 0) {
1697
+ // Nothing queued — clear any stale per-entry rejection counters so the map can't grow.
1698
+ if (unresolvedForwardRejectionCounts.size > 0) unresolvedForwardRejectionCounts.clear();
1699
+ return;
1700
+ }
1701
+
1702
+ // Every id-form THIS daemon answers to. A self-addressed outbox entry (coordinator
1703
+ // == this daemon) must never be cross-dialled — see the self-route branch below.
1704
+ const selfIds = resolveCoordinatorDaemonIds(components);
1705
+ const isSelfCoordinatorId = (id: string): boolean =>
1706
+ selfIds.some(self => daemonIdsEquivalent(self, id));
1707
+
1708
+ for (const entry of entries) {
1709
+ // EVTTRACE correlation context for this outbox entry's retry.
1710
+ const entryTraceCtx = {
1711
+ taskId: (entry.payload as Record<string, unknown>).taskId,
1712
+ sessionId: readNonEmptyString(entry.payload.targetSessionId) || readNonEmptyString(entry.payload.sessionId),
1713
+ nodeId: readNonEmptyString(entry.payload.nodeId),
1714
+ event: readNonEmptyString(entry.payload.event),
1715
+ };
1716
+
1717
+ // Self-addressed forward: the coordinator daemon this entry targets IS this
1718
+ // daemon (a self-coordinating / single-node mesh, or a delegate whose coordinator
1719
+ // anchor resolved to our own id). A cross-daemon mesh_forward_event to our own id
1720
+ // is REFUSED by the dispatch self-dial guard ("Refusing to send ... to this
1721
+ // daemon's own id; route via the local router instead") on every retry, so the
1722
+ // entry can never be acked and loops forever (~every tick), spamming the log and
1723
+ // pinning the outbox row permanently undrained. Honour the guard's own advice:
1724
+ // route the event straight through the local receiver (handleMeshForwardEvent —
1725
+ // the same path the coordinator runs on receiving a remote push), then ack it.
1726
+ // We drain regardless of the local result: a cross-daemon dispatch could not have
1727
+ // resolved it either (the guard rejects before the receiver ever runs), so leaving
1728
+ // it queued only re-spams. handleMeshForwardEvent has the BEST recovery chance —
1729
+ // this daemon hosts the mesh, so its workspace/nodeId → meshId recovery applies.
1730
+ if (isSelfCoordinatorId(entry.coordinatorDaemonId)) {
1731
+ let localResult: any;
1732
+ try {
1733
+ traceMeshEventStage('forward_send', entryTraceCtx, `self → local router (${entry.coordinatorDaemonId})`);
1734
+ localResult = handleMeshForwardEvent(components, entry.payload);
1735
+ } catch (e: any) {
1736
+ LOG.warn('MeshReconcile', `Local route of self-addressed forward to ${entry.coordinatorDaemonId} threw: ${e?.message || e} — draining anyway to break the retry loop`);
1737
+ }
1738
+ ackUnresolvedDelegateForward(entry.id);
1739
+ unresolvedForwardRejectionCounts.delete(entry.id);
1740
+ if (localResult && localResult.success === false) {
1741
+ LOG.warn('MeshReconcile', `Self-addressed unresolved-delegate ${readNonEmptyString(entry.payload.event)} rejected by local router (${readNonEmptyString(localResult.error) || 'no reason'}) — drained to break the self-forward retry loop`);
1742
+ traceMeshEventDrop('self_forward_local_rejected', entryTraceCtx, readNonEmptyString(localResult.error) || 'no reason');
1743
+ } else {
1744
+ LOG.info('MeshReconcile', `Self-addressed unresolved-delegate ${readNonEmptyString(entry.payload.event)} routed via local router (coordinator ${entry.coordinatorDaemonId} is self) — drained`);
1745
+ }
1746
+ continue;
1747
+ }
1748
+
1749
+ // RECONCILE-MESHID-DROP: the stored forward payload was built when the worker
1750
+ // "couldn't resolve" its meshId, so the coordinator rejects it "meshId required"
1751
+ // when its own workspace/nodeId recovery misses. The worker can usually resolve it
1752
+ // now (member node membership / live-session meshNodeFor) — stamp it on so the
1753
+ // coordinator accepts. Covers entries persisted before this fix AND late-bound
1754
+ // sessions. No-op when the payload already carries a meshId or none is resolvable.
1755
+ let pushPayload = entry.payload;
1756
+ if (!readNonEmptyString(pushPayload.meshId)) {
1757
+ const recoveredMeshId = resolveForwardEventMeshId(components, pushPayload);
1758
+ if (recoveredMeshId) {
1759
+ pushPayload = { ...pushPayload, meshId: recoveredMeshId };
1760
+ traceMeshEventStage('forward_meshid_recovered', entryTraceCtx, `meshId=${recoveredMeshId}`);
1761
+ }
1762
+ }
1763
+
1764
+ let result: any;
1765
+ try {
1766
+ traceMeshEventStage('forward_send', entryTraceCtx, `retry → ${entry.coordinatorDaemonId}`);
1767
+ result = await dispatchMeshCommand(entry.coordinatorDaemonId, 'mesh_forward_event', pushPayload);
1768
+ } catch (e: any) {
1769
+ // Coordinator unreachable (transport threw) — keep the entry queued and try again
1770
+ // next tick. This is NOT a hard rejection, so it does not count toward the cap;
1771
+ // the age expiry bounds a permanently-offline coordinator.
1772
+ LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} failed: ${e?.message || e} — left queued`);
1773
+ traceMeshEventDrop('retry_forward_failed', entryTraceCtx, e?.message || String(e));
1774
+ continue;
1775
+ }
1776
+ if (result && result.success === false) {
1777
+ // Hard rejection: the push was delivered and deterministically refused. Retrying
1778
+ // the identical payload can never succeed, so bound it — after MAX_FORWARD_REJECTIONS
1779
+ // drop (drain) the entry with one fail-loud warning instead of re-spamming every tick.
1780
+ const rejections = (unresolvedForwardRejectionCounts.get(entry.id) || 0) + 1;
1781
+ unresolvedForwardRejectionCounts.set(entry.id, rejections);
1782
+ const reason = readNonEmptyString(result.error) || 'no reason';
1783
+ if (rejections >= MAX_FORWARD_REJECTIONS) {
1784
+ ackUnresolvedDelegateForward(entry.id);
1785
+ unresolvedForwardRejectionCounts.delete(entry.id);
1786
+ LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected ${rejections}x (${reason}) — dropping unresolved-delegate ${readNonEmptyString(entry.payload.event)} (sess=${readNonEmptyString(entry.payload.targetSessionId) || readNonEmptyString(entry.payload.sessionId) || '-'}) to stop the retry loop`);
1787
+ traceMeshEventDrop('retry_forward_exhausted', entryTraceCtx, `${reason} (${rejections} rejections)`);
1788
+ } else {
1789
+ LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected (${reason}) — left queued (attempt ${rejections}/${MAX_FORWARD_REJECTIONS})`);
1790
+ traceMeshEventDrop('retry_forward_rejected', entryTraceCtx, reason);
1791
+ }
1792
+ continue;
1793
+ }
1794
+ // Acked — mark the durable copy delivered.
1795
+ ackUnresolvedDelegateForward(entry.id);
1796
+ unresolvedForwardRejectionCounts.delete(entry.id);
1797
+ LOG.info('MeshReconcile', `Retried+delivered unresolved-delegate ${readNonEmptyString(entry.payload.event)} to coordinator ${entry.coordinatorDaemonId}`);
1798
+ }
1799
+ }
1800
+
1801
+ // Cloud-only: poll each remote worker node daemon for pending coordinator events
1802
+ // and re-inject them locally via handleMeshForwardEvent (which re-queues +
1803
+ // surfaces to the live coordinator on the next tick / immediately if idle).
1804
+ //
1805
+ // Scoping: the remote handler (get_pending_mesh_events) drains its queue filtered
1806
+ // by coordinatorDaemonId — returning events targeted at that id OR unscoped, and
1807
+ // leaving events targeted at a *different* coordinator. A remote worker stamps the
1808
+ // coordinator id in one of SEVERAL forms (the canonical status id `standalone_`/
1809
+ // `daemon_<machineId>` stamped by the MCP layer, the bare machineId stamped by the
1810
+ // local queue path, OR — most commonly for remote launches — the coordinator mesh
1811
+ // node's config-form `daemonId`, which resolveCoordinatorDaemonId prefers and which
1812
+ // is NOT canonicalised). `candidateDaemonIds` is the already-expanded self-identity
1813
+ // set (resolveCoordinatorSelfIds: runtime drain ids ∪ this daemon's mesh-config node/
1814
+ // host id forms), so we pull ONCE PER candidate id and a completion stamped with any
1815
+ // of them is recovered. The remote drain is atomic (drained=1), so issuing multiple
1816
+ // pulls cannot double-deliver — the first pull that matches consumes the event; the
1817
+ // rest see nothing. When no ids resolve we fall back to a single unscoped pull.
1818
+ async function pullRemoteNodeQueues(
1819
+ components: DaemonComponents,
1820
+ mesh: LocalMeshEntry,
1821
+ localDaemonId: string | undefined,
1822
+ candidateDaemonIds: string[],
1823
+ ): Promise<void> {
1824
+ const dispatchMeshCommand = components.dispatchMeshCommand;
1825
+ if (!dispatchMeshCommand) return;
1826
+ const meshId = mesh.id;
1827
+
1828
+ // One args object per candidate coordinator-id form, or a single unscoped pull
1829
+ // when none resolve.
1830
+ const pulls: Array<Record<string, unknown>> = candidateDaemonIds.length > 0
1831
+ ? candidateDaemonIds.map(id => ({ meshId, coordinatorDaemonId: id }))
1832
+ : [{ meshId }];
1833
+
1834
+ for (const node of mesh.nodes) {
1835
+ const nodeDaemonId = readNonEmptyString(node.daemonId);
1836
+ // Skip nodes without a daemon, and nodes on THIS daemon (their events are
1837
+ // already in the local queue drained in PHASE 2). "This daemon" is matched
1838
+ // against the full self-identity set (candidateDaemonIds), not just the bare
1839
+ // localDaemonId — a self node can be registered under the config-form daemonId
1840
+ // (`daemon_<machineId>`) which would NOT equal bare localDaemonId, and pulling
1841
+ // from ourselves over P2P is both wasteful and a self-dispatch hazard.
1842
+ if (!nodeDaemonId) continue;
1843
+ if (daemonIdsEquivalent(nodeDaemonId, localDaemonId)) continue;
1844
+ if (daemonIdListIncludes(candidateDaemonIds, nodeDaemonId)) continue;
1845
+
1846
+ for (const pendingEventArgs of pulls) {
1847
+ let events: unknown;
1848
+ try {
1849
+ events = await dispatchMeshCommand(nodeDaemonId, 'get_pending_mesh_events', pendingEventArgs);
1850
+ } catch {
1851
+ // Remote pull is best-effort; the node may be offline. Retry next tick.
1852
+ break; // node unreachable — don't bother with the other id form this tick.
1853
+ }
1854
+ const list = extractPendingEvents(events).filter(e => readNonEmptyString(e?.meshId) === meshId);
1855
+ for (const event of list) {
1856
+ const payload = buildForwardPayloadFromPending(event);
1857
+ if (!payload.event || !payload.meshId) continue;
1858
+ try {
1859
+ handleMeshForwardEvent(components, payload);
1860
+ } catch { /* best-effort re-inject */ }
1861
+ }
1862
+ }
1863
+ }
1864
+ }
1865
+
1866
+ // Pull the read_chat payload out of whatever envelope the transport returned.
1867
+ // A local commandHandler.handle() returns the CommandResult directly; a remote
1868
+ // dispatchMeshCommand returns it possibly wrapped in { payload } / { result }.
1869
+ function unwrapReadChatPayload(raw: unknown): Record<string, unknown> | null {
1870
+ let cursor: unknown = raw;
1871
+ for (let depth = 0; depth < 4 && cursor && typeof cursor === 'object'; depth++) {
1872
+ const record = cursor as Record<string, unknown>;
1873
+ if (Array.isArray(record.messages)) return record;
1874
+ if (record.payload && typeof record.payload === 'object') { cursor = record.payload; continue; }
1875
+ if (record.result && typeof record.result === 'object') { cursor = record.result; continue; }
1876
+ if (record.data && typeof record.data === 'object') { cursor = record.data; continue; }
1877
+ break;
1878
+ }
1879
+ return cursor && typeof cursor === 'object' ? cursor as Record<string, unknown> : null;
1880
+ }
1881
+
1882
+ function readChatPayloadStatus(payload: Record<string, unknown> | null): string {
1883
+ return readNonEmptyString(payload?.status).toLowerCase();
1884
+ }
1885
+
1886
+ // R4e fix (3): peek the pending-events queue for a REAL (worker-emitted) terminal completion
1887
+ // already queued for a task — used to yield the in-flight synth to the worker's own emit. Broad
1888
+ // peek (no daemon-id scoping) matched precisely by taskId, so a worker stamp in any daemon-id form
1889
+ // is still recognized. Best-effort: a peek failure returns false (proceed to synth — never block
1890
+ // delivery). A prior SYNTH's still-queued pending event also names this taskId, but a synth always
1891
+ // writes its terminal ledger atomically, so hasTerminalLedgerAfterDispatch downstream already
1892
+ // no-ops that case — this guard is specifically for an as-yet-unledgered worker emit in flight.
1893
+ function realTerminalEmitPendingForTask(meshId: string, taskId: string): boolean {
1894
+ let pending: readonly PendingMeshCoordinatorEvent[];
1895
+ try {
1896
+ pending = getPendingMeshCoordinatorEvents(meshId);
1897
+ } catch {
1898
+ return false;
1899
+ }
1900
+ return pending.some(e =>
1901
+ readNonEmptyString(e.metadataEvent?.taskId) === taskId
1902
+ && (e.event === 'agent:generating_completed' || e.event === 'agent:stopped'));
1903
+ }
1904
+
1905
+ // R4e fix (2): one fresh read_chat status read for the worker session, via the same local/remote
1906
+ // transport PHASE 4 uses. Returns the lowercased status, or null when the read is inconclusive
1907
+ // (transport error, success:false, no payload) — callers treat null as "no new evidence, proceed".
1908
+ async function reprobeWorkerStatus(
1909
+ components: DaemonComponents,
1910
+ args: { isLocalNode: boolean; nodeDaemonId: string; readArgs: Record<string, unknown> },
1911
+ ): Promise<string | null> {
1912
+ try {
1913
+ if (args.isLocalNode) {
1914
+ const r = await components.commandHandler.handle('read_chat', args.readArgs);
1915
+ if (r && (r as { success?: boolean }).success === false) return null;
1916
+ return readChatPayloadStatus(unwrapReadChatPayload(r));
1917
+ }
1918
+ if (components.dispatchMeshCommand) {
1919
+ const r = await components.dispatchMeshCommand(args.nodeDaemonId, 'read_chat', args.readArgs);
1920
+ const p = unwrapReadChatPayload(r);
1921
+ if (p && (p as { success?: boolean }).success === false) return null;
1922
+ return readChatPayloadStatus(p);
1923
+ }
1924
+ } catch {
1925
+ return null;
1926
+ }
1927
+ return null;
1928
+ }
1929
+
1930
+ // PHASE 4 helper. For every active (non-terminal) direct dispatch this daemon
1931
+ // hosts, confirm the worker session is idle via a read_chat and — if a final
1932
+ // assistant summary is present but no terminal ledger exists for that dispatch —
1933
+ // synthesize the missing completion through reconcileDirectDispatchCompletionFromTranscript.
1934
+ //
1935
+ // read_chat is resolved against the target node: a node on THIS daemon is read
1936
+ // through the local commandHandler; a remote node is read over P2P via
1937
+ // dispatchMeshCommand. Both yield the same { messages, status, providerSessionId }
1938
+ // shape. We only synthesize when the session reports idle AND a final assistant
1939
+ // message exists — the same evidence bar the MCP poll path uses — so an actively
1940
+ // generating worker is never falsely completed. The reconcile itself is idempotent.
1941
+ async function reconcileUnterminatedDirectDispatches(
1942
+ components: DaemonComponents,
1943
+ mesh: LocalMeshEntry,
1944
+ selfIds: string[],
1945
+ localDaemonId: string | undefined,
1946
+ ): Promise<void> {
1947
+ const dispatches = getActiveDirectDispatches(mesh.id);
1948
+
1949
+ // T2 (B2b): restart rehydration. Reload this mesh's persisted acked-hold rows into
1950
+ // the Map cache the first time this process touches the mesh — a hold established
1951
+ // before a daemon restart is honored again. Must run BEFORE the prune below so a
1952
+ // rehydrated hold for a still-active task is not seen as absent-from-cache and lost.
1953
+ rehydrateAckedHoldsForMesh(mesh.id);
1954
+
1955
+ // Prune the in-flight acked-hold state to the tasks still active in THIS mesh, so a
1956
+ // completed/pruned task's state is dropped (both the Map cache AND the store row —
1957
+ // the persisted table never grows without bound). Runs even when there are zero
1958
+ // active dispatches so a restart that landed after every task terminated still
1959
+ // reaps orphaned store rows. Iterate the union of Map keys and store rows so a row
1960
+ // that exists ONLY on disk (not yet cached) is pruned too.
1961
+ const activeTaskKeys = new Set(
1962
+ dispatches
1963
+ .map(d => readNonEmptyString(d.taskId))
1964
+ .filter(Boolean)
1965
+ .map(taskId => inFlightSynthKey(mesh.id, taskId)),
1966
+ );
1967
+ const heldKeys = new Set<string>();
1968
+ for (const key of inFlightAckedHoldState.keys()) {
1969
+ if (key.startsWith(`${mesh.id}::`)) heldKeys.add(key);
1970
+ }
1971
+ const store = holdStore();
1972
+ if (store) {
1973
+ try {
1974
+ for (const row of store.listInflightHoldsByMesh(mesh.id)) {
1975
+ heldKeys.add(inFlightSynthKey(mesh.id, row.taskId));
1976
+ }
1977
+ } catch { /* degrade — prune only what's in the Map */ }
1978
+ }
1979
+ for (const key of heldKeys) {
1980
+ if (!activeTaskKeys.has(key)) deleteHoldState(key, mesh.id);
1981
+ }
1982
+
1983
+ if (dispatches.length === 0) return; // nothing left to reconcile after the prune
1984
+
1985
+ const dispatchMeshCommand = components.dispatchMeshCommand;
1986
+ const nodeById = new Map(mesh.nodes.map(n => [n.id, n] as const));
1987
+
1988
+ for (const dispatch of dispatches) {
1989
+ const sessionId = readNonEmptyString(dispatch.sessionId);
1990
+ const nodeId = readNonEmptyString(dispatch.nodeId);
1991
+ const taskId = readNonEmptyString(dispatch.taskId);
1992
+ if (!sessionId || !nodeId || !taskId) continue;
1993
+
1994
+ const node = nodeById.get(nodeId);
1995
+ const nodeDaemonId = readNonEmptyString(node?.daemonId);
1996
+ // A node is local when it has no daemonId, names this daemon, or actually
1997
+ // has a live instance here. Anything else is reached over P2P.
1998
+ const isLocalNode = !nodeDaemonId
1999
+ || daemonIdListIncludes(selfIds, nodeDaemonId)
2000
+ || daemonIdsEquivalent(nodeDaemonId, localDaemonId)
2001
+ || !!components.instanceManager.getInstance(sessionId);
2002
+
2003
+ const providerType = readNonEmptyString(dispatch.providerType);
2004
+ const readArgs: Record<string, unknown> = {
2005
+ sessionId,
2006
+ targetSessionId: sessionId,
2007
+ tailLimit: 10,
2008
+ ...(node?.workspace ? { workspace: node.workspace } : {}),
2009
+ ...(providerType ? { agentType: providerType, providerType } : {}),
2010
+ };
2011
+
2012
+ const synthKey = inFlightSynthKey(mesh.id, taskId);
2013
+ const isAcked = dispatch.status === 'acked';
2014
+
2015
+ // R4f: read the worker session. A FAILED read (transport error / success:false / no payload)
2016
+ // is no longer silently swallowed for an acked task — it is the liveness side of the
2017
+ // death backstop (a). We classify the read result and route an acked failure into the
2018
+ // failure counter; a never-acked (or non-acked) failure keeps the old best-effort `continue`.
2019
+ let payload: Record<string, unknown> | null = null;
2020
+ let readFailed = false;
2021
+ try {
2022
+ if (isLocalNode) {
2023
+ const result = await components.commandHandler.handle('read_chat', readArgs);
2024
+ if (result && (result as { success?: boolean }).success === false) {
2025
+ readFailed = true;
2026
+ } else {
2027
+ payload = unwrapReadChatPayload(result);
2028
+ }
2029
+ } else if (dispatchMeshCommand) {
2030
+ const result = await dispatchMeshCommand(nodeDaemonId, 'read_chat', readArgs);
2031
+ payload = unwrapReadChatPayload(result);
2032
+ if (payload && (payload as { success?: boolean }).success === false) { payload = null; readFailed = true; }
2033
+ } else {
2034
+ continue; // remote node but no P2P transport — can't read; retry next tick (not a death signal)
2035
+ }
2036
+ } catch {
2037
+ readFailed = true; // session may be gone or node offline
2038
+ }
2039
+ if (!payload && !readFailed) continue; // null payload that wasn't a hard failure — retry next tick
2040
+
2041
+ if (readFailed || !payload) {
2042
+ // R4f backstop (a) — liveness failure. For a never-acked dispatch there is no in-flight
2043
+ // turn to protect, so a read failure is a transient probe blip → retry next tick (old
2044
+ // behavior). For an ACKED dispatch that we had previously confirmed live, a streak of
2045
+ // consecutive read failures means the worker session genuinely went away mid-turn and
2046
+ // will never emit its real completion — count it. The actual terminal cleanup of a
2047
+ // gone session is owned by PHASE 2.5 (stranded reclaim) / PHASE 5 (orphan prune); here
2048
+ // we only record the death observation and STOP holding so those nets can take over,
2049
+ // rather than pinning the row on an indefinite hold for a session that is already gone.
2050
+ if (isAcked) {
2051
+ const prior = getHoldState(synthKey, mesh.id);
2052
+ const failures = (prior?.consecutiveReadFailures ?? 0) + 1;
2053
+ const liveConfirmedSinceAck = prior?.liveConfirmedSinceAck ?? false;
2054
+ // A read failure breaks the idle-with-final-assistant run → reset the fast-track streak
2055
+ // (transcriptIdleSinceMs cleared by omission) so it must re-accumulate from scratch.
2056
+ setHoldState(synthKey, mesh.id, { liveConfirmedSinceAck, consecutiveReadFailures: failures });
2057
+ if (liveConfirmedSinceAck && failures >= ACKED_DEATH_CONSECUTIVE_READ_FAILURES) {
2058
+ LOG.warn('MeshReconcile', `Acked-hold death signal: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read_chat failed ${failures}x consecutively after a live-confirmed ack — worker session presumed gone mid-turn; releasing the indefinite synth hold to the stranded-reclaim / orphan-prune nets`);
2059
+ }
2060
+ }
2061
+ continue; // no readable transcript this tick → cannot synth here; retry / let backstops act
2062
+ }
2063
+
2064
+ // Read succeeded (a conclusive idle/generating status) → the session is reachable: reset the
2065
+ // failure streak and mark it live-confirmed-since-ack, so a LATER read failure is recognized
2066
+ // as a genuine liveness loss (backstop a) rather than a node that was never reachable. The
2067
+ // fast-track idle streak (transcriptIdleSinceMs) is PRESERVED across this reset — it is
2068
+ // managed below where the idle + final-assistant signal is actually evaluated.
2069
+ const priorHoldState = getHoldState(synthKey, mesh.id);
2070
+ setHoldState(synthKey, mesh.id, {
2071
+ liveConfirmedSinceAck: true,
2072
+ consecutiveReadFailures: 0,
2073
+ ...(priorHoldState?.transcriptIdleSinceMs !== undefined ? { transcriptIdleSinceMs: priorHoldState.transcriptIdleSinceMs } : {}),
2074
+ });
2075
+
2076
+ // Only act on a session that has actually settled to idle. A generating /
2077
+ // waiting_approval session is mid-turn — synthesizing a completion now would
2078
+ // be wrong. (idle is the only status the MCP poll path reconciles too.)
2079
+ const nowMs = Date.now();
2080
+ if (readChatPayloadStatus(payload) !== 'idle') {
2081
+ // Not idle → the worker is genuinely mid-turn (a clear live signal). Keep the
2082
+ // live-confirmed flag set (above) but RESET the fast-track idle streak: a turn that
2083
+ // resumed generating proves the prior idle was a mid-turn blip, not a settled turn-end.
2084
+ setHoldState(synthKey, mesh.id, { liveConfirmedSinceAck: true, consecutiveReadFailures: 0 });
2085
+ continue;
2086
+ }
2087
+
2088
+ // R4f GENERATING-BOUNDARY (acked-hold): a dispatch whose worker was OBSERVED to start
2089
+ // generating (the agent:generating_started ack flipped the row to 'acked') is ALIVE and
2090
+ // mid-turn — it WILL eventually emit a real terminal. An `idle` read here is therefore
2091
+ // presumed a TRANSIENT mid-turn window (a PTY inter-tool-call settle, or final text already
2092
+ // rendered while the lifecycle close lags), NOT a settled completion. We HOLD the synth
2093
+ // INDEFINITELY rather than racing the worker's (variable, unbounded) emit latency with a
2094
+ // finite timer — the failure mode of R4..R4e. This is safe: when the worker's real emit
2095
+ // lands it writes a terminal ledger, and reconcileDirectDispatchCompletionFromTranscript's
2096
+ // hasTerminalLedgerAfterDispatch makes any later synth an idempotent no-op, so the real emit
2097
+ // always wins no matter how late. The hold is released ONLY by the death backstops:
2098
+ // (a) consecutive read failures after a live-confirmed ack (handled above), or
2099
+ // (b) the absolute ACKED_DEATH_DEADLINE_MS since the ack — a notification-loss net set FAR
2100
+ // above any observed emit latency, so it catches a genuinely-wedged worker / lost emit
2101
+ // without racing a normal slow turn.
2102
+ // A never-acked dispatch (worker never started) is exempt — no in-flight generation to
2103
+ // pre-empt; it keeps the first-idle-tick synth, with the downstream grace + stale-summary
2104
+ // guards as its backstops.
2105
+ //
2106
+ // ACKED-HOLD-IDLE-OVERTRUST: the read is idle. Extract the final-assistant evidence NOW (the
2107
+ // same signal the synth below requires) so the fast-track can gate on idle-WITH-final-assistant
2108
+ // rather than bare idle. Only when a final visible assistant message is present do we treat
2109
+ // this tick as a candidate turn-end and accumulate the fast-track grace streak; a bare idle
2110
+ // with no assistant result is the worker still warming up and resets the streak.
2111
+ const messages = Array.isArray(payload.messages) ? payload.messages as ChatMessage[] : [];
2112
+ const evidence = extractFinalAssistantSummaryEvidence(messages);
2113
+
2114
+ if (isAcked) {
2115
+ const ackedAtMs = Date.parse(readNonEmptyString(dispatch.updatedAt));
2116
+ const sinceAckMs = Number.isFinite(ackedAtMs) ? nowMs - ackedAtMs : Number.POSITIVE_INFINITY;
2117
+ const deathDeadlineMs = resolveAckedDeathDeadlineMs();
2118
+
2119
+ // ACKED-HOLD-IDLE-OVERTRUST fast-track. Maintain the continuous idle-with-final-assistant
2120
+ // streak. The streak starts (or continues) only while a final visible assistant message is
2121
+ // present; a tick with idle-but-no-assistant breaks it (the answer is not yet rendered).
2122
+ const holdState = getHoldState(synthKey, mesh.id);
2123
+ let fastTrackReady = false;
2124
+ if (evidence.finalSummary) {
2125
+ const idleSinceMs = holdState?.transcriptIdleSinceMs ?? nowMs;
2126
+ if (holdState && holdState.transcriptIdleSinceMs === undefined) {
2127
+ setHoldState(synthKey, mesh.id, { ...holdState, transcriptIdleSinceMs: idleSinceMs });
2128
+ }
2129
+ const fastTrackGraceMs = resolveAckedTranscriptFastTrackGraceMs();
2130
+ const idleHeldMs = nowMs - idleSinceMs;
2131
+ if (idleHeldMs >= fastTrackGraceMs) {
2132
+ fastTrackReady = true;
2133
+ LOG.info('MeshReconcile', `Acked-hold transcript fast-track: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle WITH a final assistant message for ${Math.round(idleHeldMs / 1000)}s continuous (grace ${Math.round(fastTrackGraceMs / 1000)}s) — promoting the synth ahead of the ${Math.round(deathDeadlineMs / 1000)}s death backstop; the worker's real emit was lost/late and a later one no-ops idempotently.`);
2134
+ }
2135
+ } else if (holdState?.transcriptIdleSinceMs !== undefined) {
2136
+ // Idle but no final assistant yet → not a turn-end; reset the streak.
2137
+ setHoldState(synthKey, mesh.id, { ...holdState, transcriptIdleSinceMs: undefined });
2138
+ }
2139
+
2140
+ // Hold indefinitely UNLESS the fast-track grace was met OR the absolute death deadline is
2141
+ // reached. The fast-track is the new fast path in front of the (preserved) 8-min backstop.
2142
+ if (!fastTrackReady && sinceAckMs < deathDeadlineMs) {
2143
+ LOG.info('MeshReconcile', `Acked-hold: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read idle ${Number.isFinite(sinceAckMs) ? Math.round(sinceAckMs / 1000) + 's' : '∞'} since the generating_started ack — HOLDING synth (worker presumed alive; a later real emit is idempotent). Transcript fast-track promotes at ${Math.round(resolveAckedTranscriptFastTrackGraceMs() / 1000)}s continuous idle-with-final-assistant; death backstop at ${Math.round(deathDeadlineMs / 1000)}s or on consecutive read failures.`);
2144
+ continue;
2145
+ }
2146
+ if (!fastTrackReady) {
2147
+ LOG.warn('MeshReconcile', `Acked-hold death deadline reached: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) still idle ${Math.round(sinceAckMs / 1000)}s after the ack (deadline ${Math.round(deathDeadlineMs / 1000)}s) — synthesizing the missing completion as a notification-loss net (a real emit, if it ever lands, no-ops idempotently).`);
2148
+ }
2149
+ }
2150
+
2151
+ // R4f (auxiliary, was R4e fix 3) — worker-emit priority. Secondary check: if the worker's
2152
+ // REAL terminal emit for this task has already arrived in the pending-events queue (queued
2153
+ // for delivery to the coordinator) but not yet written a terminal ledger, YIELD — let the
2154
+ // genuine emit surface rather than racing it with a synth that would win the taskId-anchored
2155
+ // fingerprint dedup and mask it. Under the R4f acked-hold this is now an auxiliary belt-and-
2156
+ // suspenders check (the indefinite hold already defers an acked synth); it still guards the
2157
+ // never-acked path and the post-death-deadline acked synth from racing an emit caught in
2158
+ // flight at synth-commit time.
2159
+ if (realTerminalEmitPendingForTask(mesh.id, taskId)) {
2160
+ deleteHoldState(synthKey, mesh.id);
2161
+ LOG.info('MeshReconcile', `Worker-emit priority: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) has a real terminal completion already queued — yielding synth to the worker's own emit`);
2162
+ continue;
2163
+ }
2164
+
2165
+ if (!evidence.finalSummary) continue; // no assistant result yet — nothing to attribute
2166
+
2167
+ // STALE-SUMMARY guard (modal-parked / reused-session misattribution): a direct
2168
+ // dispatch frequently reuses a session that already ran a PRIOR task. read_chat
2169
+ // returns the tail of the WHOLE session, so extractFinalAssistantSummaryEvidence
2170
+ // picks the latest user-facing assistant message — which, for a task that has
2171
+ // barely started (the session momentarily reads idle between turns), is the prior
2172
+ // task's final summary. The downstream reconcile proves the summary is after the
2173
+ // LEDGER task_dispatched entry; here we additionally have the AUTHORITATIVE per-task
2174
+ // dispatchedAt (the dispatch-store row, immune to ledger-ordering quirks), so when
2175
+ // the selected transcript message is provably BEFORE this task's own dispatch we
2176
+ // refuse it outright — it is a prior task's summary, not this task's output (the
2177
+ // 2843ms-duration stale-summary bug where task 2e3f501e copy-pasted 4eca2d9d's
2178
+ // summary). When the message carries no usable timestamp we do NOT block here: the
2179
+ // downstream reconcile already rejects a non-JSON summary it cannot prove is
2180
+ // post-dispatch (transcript_not_proven_after_dispatch), and a structured
2181
+ // final_summary_json is self-attributing — so a timeless provider is not
2182
+ // over-blocked while the provable-stale case is still caught.
2183
+ const dispatchedAtMs = Date.parse(readNonEmptyString(dispatch.dispatchedAt));
2184
+ const transcriptAtMs = Date.parse(evidence.transcriptMessageAt ?? '');
2185
+ if (Number.isFinite(dispatchedAtMs) && Number.isFinite(transcriptAtMs) && transcriptAtMs < dispatchedAtMs) {
2186
+ LOG.info('MeshReconcile', `Stale-summary guard: skipping transcript reconcile for task ${taskId} on node ${nodeId} (mesh ${mesh.id}) — final assistant message (${evidence.transcriptMessageAt}) predates this task's dispatch (${dispatch.dispatchedAt}); it is a prior task's summary`);
2187
+ traceMeshEventDrop('reconcile_stale_summary_before_dispatch', {
2188
+ taskId, sessionId, nodeId, meshId: mesh.id, event: 'agent:generating_completed',
2189
+ }, `transcriptAt=${evidence.transcriptMessageAt} < dispatchedAt=${dispatch.dispatchedAt}`);
2190
+ continue;
2191
+ }
2192
+
2193
+ // R4f (auxiliary, was R4e fix 2) — live re-probe immediately before committing the synth. A
2194
+ // fresh read right now catches a worker that resumed generating since this tick's first read
2195
+ // so it is never falsely completed off a stale snapshot. Best-effort: an inconclusive
2196
+ // re-probe (transport error/null) falls through to the synth — we already hold a valid idle
2197
+ // read from the top of THIS tick, so a re-probe failure must not re-introduce a
2198
+ // notification-miss. Under the R4f acked-hold this matters mainly for the never-acked path
2199
+ // and the post-death-deadline acked synth (the indefinite hold already deferred a live acked
2200
+ // turn); it stays as a final live-state guard at synth-commit time.
2201
+ const reprobeStatus = await reprobeWorkerStatus(components, { isLocalNode, nodeDaemonId, readArgs });
2202
+ if (reprobeStatus && reprobeStatus !== 'idle') {
2203
+ deleteHoldState(synthKey, mesh.id);
2204
+ LOG.info('MeshReconcile', `Live re-probe defer: task ${taskId} on node ${nodeId} (mesh ${mesh.id}) read '${reprobeStatus}' at synth-commit time — worker resumed generating; deferring synth to a later tick`);
2205
+ continue;
2206
+ }
2207
+
2208
+ const providerSessionId = readNonEmptyString(payload.providerSessionId);
2209
+ const coordinatorDaemonId = selfIds.find(id => !!id);
2210
+ try {
2211
+ const result = reconcileDirectDispatchCompletionFromTranscript({
2212
+ meshId: mesh.id,
2213
+ nodeId,
2214
+ sessionId,
2215
+ providerType: providerType || undefined,
2216
+ providerSessionId: providerSessionId || undefined,
2217
+ taskId,
2218
+ finalSummary: evidence.finalSummary,
2219
+ ...(evidence.transcriptMessageAt ? { transcriptMessageAt: evidence.transcriptMessageAt } : {}),
2220
+ ...(coordinatorDaemonId ? { targetCoordinatorDaemonId: coordinatorDaemonId } : {}),
2221
+ source: 'daemon_reconcile_transcript_completion',
2222
+ });
2223
+ if (result.reconciled) {
2224
+ LOG.info('MeshReconcile', `Synthesized missing completion (${result.kind}) for task ${taskId} on node ${nodeId} (mesh ${mesh.id})`);
2225
+ }
2226
+ } catch (e: any) {
2227
+ LOG.warn('MeshReconcile', `Transcript completion reconcile threw for task ${taskId}: ${e?.message || e}`);
2228
+ }
2229
+ }
2230
+ }
2231
+
2232
+ // PHASE 5 helper. Build the live-node view (mesh.nodes decorated with each node's live
2233
+ // session list) and run the shared prune core in execute mode with the conservative age gate.
2234
+ //
2235
+ // Orphan detection needs the SAME live-session evidence the manual MCP prune uses: a node still
2236
+ // in mesh.nodes whose session list no longer contains the dispatched sessionId is "session not
2237
+ // present" (prunable); a node missing from mesh.nodes entirely is "node no longer in live mesh"
2238
+ // (prunable). We obtain live sessions per node via get_status_metadata — local nodes through the
2239
+ // local commandHandler, remote nodes over P2P (dispatchMeshCommand) — exactly the transports
2240
+ // PHASE 4 already uses. A node we cannot probe (offline) keeps an empty session list; combined
2241
+ // with the age gate that only matters once the orphan is genuinely old.
2242
+ //
2243
+ // O(1) fast exit: when there are no active direct dispatches at all there is nothing to prune,
2244
+ // so we skip the (per-node) status probes entirely — an idle mesh costs one indexed query.
2245
+ async function autoPruneStaleDirectDispatches(
2246
+ components: DaemonComponents,
2247
+ mesh: LocalMeshEntry,
2248
+ selfIds: string[],
2249
+ localDaemonId: string | undefined,
2250
+ minAgeMs: number,
2251
+ ): Promise<void> {
2252
+ const directDispatches = getActiveDirectDispatches(mesh.id);
2253
+ if (directDispatches.length === 0) return; // nothing dispatched → nothing to prune
2254
+
2255
+ const liveNodes = await collectLiveNodesWithSessions(components, mesh, selfIds, localDaemonId);
2256
+
2257
+ const result = pruneStaleDirectDispatches({
2258
+ meshId: mesh.id,
2259
+ queue: getQueue(mesh.id),
2260
+ ledgerEntries: readLedgerEntries(mesh.id, { tail: 500 }),
2261
+ directDispatches,
2262
+ nodes: liveNodes,
2263
+ execute: true,
2264
+ minAgeMs,
2265
+ source: 'daemon_reconcile_auto_prune',
2266
+ });
2267
+
2268
+ // Log only when something was actually pruned — silence on the common no-op tick.
2269
+ if (result.prunedCount > 0) {
2270
+ LOG.info('MeshReconcile', `Auto-pruned ${result.prunedCount} orphaned direct dispatch record(s) for mesh ${mesh.id}`);
2271
+ }
2272
+ }
2273
+
2274
+ // Probe each node for its live session list (get_status_metadata) and return mesh.nodes
2275
+ // decorated with a `sessions` array — the shape buildMeshActiveWork / sessionStatusFromNodes
2276
+ // consume to decide whether a dispatched session is still present. Best-effort: an unreachable
2277
+ // node yields an empty session list rather than throwing.
2278
+ async function collectLiveNodesWithSessions(
2279
+ components: DaemonComponents,
2280
+ mesh: LocalMeshEntry,
2281
+ selfIds: string[],
2282
+ localDaemonId: string | undefined,
2283
+ ): Promise<any[]> {
2284
+ const dispatchMeshCommand = components.dispatchMeshCommand;
2285
+ return Promise.all(mesh.nodes.map(async (node) => {
2286
+ const nodeDaemonId = readNonEmptyString(node.daemonId);
2287
+ const isLocalNode = !nodeDaemonId
2288
+ || daemonIdListIncludes(selfIds, nodeDaemonId)
2289
+ || daemonIdsEquivalent(nodeDaemonId, localDaemonId);
2290
+ let statusResult: unknown;
2291
+ try {
2292
+ if (isLocalNode) {
2293
+ statusResult = await components.commandHandler.handle('get_status_metadata', {});
2294
+ } else if (dispatchMeshCommand) {
2295
+ statusResult = await dispatchMeshCommand(nodeDaemonId, 'get_status_metadata', {});
2296
+ } else {
2297
+ return node; // remote node, no P2P transport — leave undecorated
2298
+ }
2299
+ } catch {
2300
+ return node; // unreachable — leave undecorated (empty session list)
2301
+ }
2302
+ const sessions = extractStatusMetadataSessions(statusResult);
2303
+ return sessions.length > 0 ? { ...node, sessions } : node;
2304
+ }));
2305
+ }
2306
+
2307
+ // Pull the live session list out of a get_status_metadata result, tolerating the same
2308
+ // envelope shapes unwrapReadChatPayload handles (direct CommandResult or { payload }/{ result }).
2309
+ function extractStatusMetadataSessions(raw: unknown): any[] {
2310
+ let cursor: unknown = raw;
2311
+ for (let depth = 0; depth < 4 && cursor && typeof cursor === 'object'; depth++) {
2312
+ const record = cursor as Record<string, unknown>;
2313
+ const status = record.status && typeof record.status === 'object' ? record.status as Record<string, unknown> : undefined;
2314
+ if (status && Array.isArray(status.sessions)) return status.sessions;
2315
+ if (Array.isArray(record.sessions)) return record.sessions;
2316
+ if (record.payload && typeof record.payload === 'object') { cursor = record.payload; continue; }
2317
+ if (record.result && typeof record.result === 'object') { cursor = record.result; continue; }
2318
+ if (record.data && typeof record.data === 'object') { cursor = record.data; continue; }
2319
+ break;
2320
+ }
2321
+ return [];
2322
+ }
2323
+
2324
+ function extractPendingEvents(raw: unknown): any[] {
2325
+ if (Array.isArray(raw)) return raw;
2326
+ if (raw && typeof raw === 'object') {
2327
+ const events = (raw as Record<string, unknown>).events;
2328
+ if (Array.isArray(events)) return events;
2329
+ }
2330
+ return [];
2331
+ }
2332
+
2333
+ // Flatten a queued PendingMeshCoordinatorEvent into the flat payload shape
2334
+ // handleMeshForwardEvent expects (mirrors the MCP buildMeshForwardPayloadFromPendingEvent).
2335
+ function buildForwardPayloadFromPending(event: any): Record<string, unknown> {
2336
+ const metadata = event?.metadataEvent && typeof event.metadataEvent === 'object'
2337
+ ? event.metadataEvent as Record<string, unknown>
2338
+ : {};
2339
+ return {
2340
+ event: readNonEmptyString(event?.event),
2341
+ meshId: readNonEmptyString(event?.meshId),
2342
+ nodeId: readNonEmptyString(event?.nodeId) || readNonEmptyString(metadata.meshNodeId),
2343
+ workspace: readNonEmptyString(event?.workspace) || readNonEmptyString(metadata.workspace),
2344
+ // Preserve the originating coordinator session id across the relay. It is normally
2345
+ // carried inside metadataEvent.meshCoordinatorSessionId (spread below), but pass the
2346
+ // top-level field through explicitly too so the handleMeshForwardEvent whitelist
2347
+ // recovers it regardless of which carrier the producing daemon used.
2348
+ ...(readNonEmptyString(event?.targetCoordinatorSessionId)
2349
+ ? { targetCoordinatorSessionId: readNonEmptyString(event.targetCoordinatorSessionId) }
2350
+ : {}),
2351
+ ...metadata,
2352
+ // NOTIF-MISS (FIX 3): surface the dispatch task id at the TOP LEVEL so the relay's
2353
+ // received-stage trace (and buildRelayMetadataEvent) recovers it regardless of which
2354
+ // carrier the producing daemon used. The metadata spread above may carry the id only as
2355
+ // `meshActiveTaskId` (a worker provider event), leaving top-level `taskId` unset and the
2356
+ // received stage rendering `task=-`. Resolve both carriers into an explicit `taskId` so
2357
+ // dedup stays task-scoped end-to-end. Only set when a non-empty id exists (no clobber to
2358
+ // undefined when neither is present).
2359
+ ...((): Record<string, unknown> => {
2360
+ const tid = readNonEmptyString(metadata.taskId) || readNonEmptyString(metadata.meshActiveTaskId);
2361
+ return tid ? { taskId: tid } : {};
2362
+ })(),
2363
+ };
2364
+ }
2365
+
2366
+ interface ReconcileLoopHandle {
2367
+ stop(): void;
2368
+ }
2369
+
2370
+ // Start the periodic reconcile loop. Returns a handle with stop() for shutdown.
2371
+ export function setupMeshReconcileLoop(components: DaemonComponents): ReconcileLoopHandle {
2372
+ const intervalMs = resolveReconcileIntervalMs();
2373
+ let running = false;
2374
+ const timer = setInterval(() => {
2375
+ if (running) return; // never overlap ticks
2376
+ running = true;
2377
+ void runMeshReconcileTick(components)
2378
+ .catch((e: any) => LOG.warn('MeshReconcile', `Reconcile tick error: ${e?.message || e}`))
2379
+ .finally(() => { running = false; });
2380
+ }, intervalMs);
2381
+ // Don't keep the process alive solely for this timer.
2382
+ if (typeof timer.unref === 'function') timer.unref();
2383
+ LOG.info('MeshReconcile', `Mesh reconcile loop started (interval ${intervalMs}ms)`);
2384
+ return {
2385
+ stop() {
2386
+ clearInterval(timer);
2387
+ LOG.info('MeshReconcile', 'Mesh reconcile loop stopped');
2388
+ },
2389
+ };
2390
+ }