@moretea-labs/forge 1.6.0 → 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (469) hide show
  1. package/AGENTS.md +4 -0
  2. package/CHANGELOG.md +39 -0
  3. package/CLAUDE.md +4 -0
  4. package/README.md +2 -2
  5. package/README.zh-CN.md +1 -1
  6. package/SKILL.md +2 -2
  7. package/THIRD_PARTY_NOTICES.md +1 -1
  8. package/agents/openai.yaml +1 -1
  9. package/assets/hooks/hook-input.sh +11 -8
  10. package/assets/hooks/lib/session-state.sh +10 -7
  11. package/assets/hooks/lib/workflow-state.sh +13 -177
  12. package/assets/hooks/post-tool-observer.sh +8 -2
  13. package/assets/hooks/prompt-guard.sh +5 -81
  14. package/assets/hooks/session-start-context.sh +56 -1
  15. package/assets/partials/05-workflow.partial.md +1 -1
  16. package/assets/partials-agents/02-operating-mode.partial.md +2 -0
  17. package/assets/partials-agents/03-orchestration.partial.md +1 -1
  18. package/assets/plugin-registry.v1.json +3 -3
  19. package/assets/reference-configs/document-generation.md +1 -1
  20. package/assets/reference-configs/harness-overview.md +1 -1
  21. package/assets/reference-configs/sprint-contracts.md +3 -3
  22. package/assets/skill-version.json +2 -2
  23. package/assets/templates/contract.template.md +2 -7
  24. package/assets/templates/helpers/capture-plan.sh +2 -2
  25. package/assets/templates/helpers/check-architecture-sync.sh +17 -0
  26. package/assets/templates/helpers/check-task-workflow.sh +4 -4
  27. package/assets/templates/helpers/contract-worktree.sh +1 -21
  28. package/assets/templates/helpers/ensure-task-workflow.sh +11 -36
  29. package/assets/templates/helpers/inspect-project-state.ts +8 -6
  30. package/assets/templates/helpers/migrate-project-template.sh +22 -45
  31. package/assets/templates/helpers/migrate-workflow-docs.ts +9 -7
  32. package/assets/templates/helpers/new-plan.sh +1 -1
  33. package/assets/templates/helpers/plan-to-todo.sh +12 -37
  34. package/assets/templates/helpers/ship-worktrees.sh +2 -29
  35. package/assets/templates/helpers/verify-contract.sh +1 -111
  36. package/assets/templates/helpers/verify-sprint.sh +3 -180
  37. package/assets/templates/helpers/workflow-contract.ts +5 -1
  38. package/assets/templates/plan.template.md +1 -1
  39. package/assets/templates/review.template.md +9 -34
  40. package/assets/workflow-contract.v1.json +10 -19
  41. package/docs/README.md +10 -10
  42. package/docs/architecture/CURRENT.md +178 -0
  43. package/docs/architecture/EVOLUTION.md +42 -0
  44. package/docs/architecture/current/README.md +4 -156
  45. package/docs/architecture/decisions/20260803-release-identity-binding-and-exit-policy.md +2 -2
  46. package/docs/architecture/decisions/20260823-immediate-controller-relay.md +53 -0
  47. package/docs/architecture/decisions/20260826-browser-runtime-v2.md +73 -0
  48. package/docs/architecture/history/README.md +7 -0
  49. package/docs/architecture/{chatgpt-handoff-facade.md → history/chatgpt-handoff-facade.md} +4 -0
  50. package/docs/architecture/{global-hook-runtime.md → history/global-hook-runtime.md} +7 -3
  51. package/docs/architecture/{ios-semantic-automation-provider-v2.md → history/ios-semantic-automation-provider-v2.md} +6 -2
  52. package/docs/architecture/history.md +3 -34
  53. package/docs/architecture/index.md +15 -98
  54. package/docs/architecture/modules/controller-runtime/transport-lifecycle.md +1 -1
  55. package/docs/architecture/reviews/2026-08-22-semantic-harness-convergence-acceptance.md +39 -0
  56. package/docs/architecture/versions/1.6.md +38 -0
  57. package/docs/architecture/versions/1.7.md +41 -0
  58. package/docs/forge-chatgpt-browser-engine.md +2 -2
  59. package/docs/forge-chatgpt-mcp-setup.md +4 -28
  60. package/docs/forge-chatgpt-supervised-automation.md +3 -3
  61. package/docs/forge-github-issue-launcher.md +1 -1
  62. package/docs/forge-plugin-management.md +1 -1
  63. package/docs/forge-runtime-self-healing-loop.md +1 -1
  64. package/docs/operations/controller-browser-plugin.md +26 -6
  65. package/docs/operations/controller-performance-and-502.md +143 -0
  66. package/docs/operations/controller-reliability-runbook.md +1 -1
  67. package/docs/operations/features.md +2 -2
  68. package/docs/operations/github-repository.md +1 -1
  69. package/docs/operations/homebrew.md +2 -2
  70. package/docs/operations/mcp-tool-exposure.md +5 -3
  71. package/docs/wiki/Architecture.md +6 -6
  72. package/docs/wiki/Implementation.md +2 -2
  73. package/docs/wiki/Integrations.md +2 -2
  74. package/docs/wiki/Work-Lifecycle.md +1 -1
  75. package/package.json +33 -11
  76. package/scripts/activate-source-baseline.ts +17 -13
  77. package/scripts/architecture-event.ts +2 -820
  78. package/scripts/architecture-queue.sh +2 -645
  79. package/scripts/archive-architecture-request.sh +2 -242
  80. package/scripts/archive-workflow.sh +1 -291
  81. package/scripts/benchmark-background-check-overlap.ts +228 -0
  82. package/scripts/benchmark-mcp-host.sh +117 -0
  83. package/scripts/benchmark-mcp-transport.ts +170 -0
  84. package/scripts/benchmark-quality-harness.ts +146 -0
  85. package/scripts/benchmark-route-session-concurrency.ts +207 -6
  86. package/scripts/benchmark-thin-harness-mcp-e2e.ts +7 -1
  87. package/scripts/benchmark-typescript-navigation.ts +186 -0
  88. package/scripts/benchmark-work-efficiency.ts +334 -0
  89. package/scripts/capability-config.ts +2 -392
  90. package/scripts/capability-resolver.ts +2 -503
  91. package/scripts/capture-plan.sh +1 -332
  92. package/scripts/check-agent-tooling.sh +2 -1558
  93. package/scripts/check-architecture-sync.sh +2 -371
  94. package/scripts/check-brain-manifest.sh +2 -253
  95. package/scripts/check-context-files.sh +2 -47
  96. package/scripts/check-deploy-sql-order.sh +2 -115
  97. package/scripts/check-mcp-compatibility.ts +14 -0
  98. package/scripts/check-runtime-architecture.mjs +349 -79
  99. package/scripts/check-skill-version.ts +3 -191
  100. package/scripts/check-source-duplication.mjs +61 -0
  101. package/scripts/check-stable-baseline.ts +92 -0
  102. package/scripts/check-task-sync.sh +2 -63
  103. package/scripts/check-task-workflow.sh +2 -1000
  104. package/scripts/codex-handoff-resume.sh +2 -267
  105. package/scripts/context-contract-sync.sh +2 -514
  106. package/scripts/contract-run.ts +2 -463
  107. package/scripts/contract-worktree.sh +1 -722
  108. package/scripts/ensure-task-workflow.sh +1 -1225
  109. package/scripts/host-temporary-cloud-mcp-e2e.sh +265 -0
  110. package/scripts/inspect-project-state.ts +4 -279
  111. package/scripts/install-standalone-recovery.ts +24 -0
  112. package/scripts/lib/project-init-lib.sh +22 -46
  113. package/scripts/lib/run-forge-helper.sh +19 -0
  114. package/scripts/lib/run-forge-helper.ts +26 -0
  115. package/scripts/migrate-project-template.sh +1 -1189
  116. package/scripts/migrate-workflow-docs.ts +4 -412
  117. package/scripts/new-plan.sh +1 -180
  118. package/scripts/new-spec.sh +1 -47
  119. package/scripts/new-sprint.sh +1 -46
  120. package/scripts/plan-to-todo.sh +1 -796
  121. package/scripts/prepare-codex-handoff.sh +2 -201
  122. package/scripts/prepare-handoff.sh +1 -78
  123. package/scripts/refresh-current-status.sh +2 -467
  124. package/scripts/run-governed-gate.ts +5 -1
  125. package/scripts/select-agent-context-blocks.sh +2 -64
  126. package/scripts/ship-worktrees.sh +1 -558
  127. package/scripts/smoke-ios-physical-device.ts +4 -2
  128. package/scripts/smoke-runtime-recovery.ts +1 -35
  129. package/scripts/smoke-schedule-engine.ts +1 -20
  130. package/scripts/sprint-backlog.sh +1 -834
  131. package/scripts/stage-runtime-release.ts +2 -1
  132. package/scripts/summarize-failures.sh +2 -123
  133. package/scripts/switch-plan.sh +1 -150
  134. package/scripts/sync-brain-docs.sh +2 -379
  135. package/scripts/sync-generated-authority.mjs +175 -0
  136. package/scripts/verify-contract.sh +2 -703
  137. package/scripts/verify-forge-runtime.sh +0 -1
  138. package/scripts/verify-sprint.sh +1 -677
  139. package/scripts/workflow-contract.ts +4 -205
  140. package/scripts/workstream-sync.sh +2 -261
  141. package/src/cli/agent-jobs/executor-health.ts +1 -1
  142. package/src/cli/commands/init-hook.ts +3 -3
  143. package/src/cli/commands/mcp.ts +9 -5
  144. package/src/cli/commands/plugin.ts +12 -3
  145. package/src/cli/commands/recovery.ts +25 -3
  146. package/src/cli/commands/runtime.ts +55 -3
  147. package/src/cli/commands/security.ts +12 -1
  148. package/src/cli/commands/setup-profile.ts +31 -21
  149. package/src/cli/controller/check-runner.ts +113 -5
  150. package/src/cli/controller/composite-result.ts +1 -1
  151. package/src/cli/controller/context/focus.ts +51 -0
  152. package/src/cli/controller/context/instruction-context.ts +114 -0
  153. package/src/cli/controller/context/known-paths.ts +76 -0
  154. package/src/cli/controller/context/query-planning.ts +110 -0
  155. package/src/cli/controller/context/rule-resolution.ts +532 -0
  156. package/src/cli/controller/context/source-materializer.ts +278 -0
  157. package/src/cli/controller/context/structural.ts +16 -0
  158. package/src/cli/controller/context/types.ts +286 -0
  159. package/src/cli/controller/context-pack.ts +828 -487
  160. package/src/cli/controller/launch-agents.ts +77 -2
  161. package/src/cli/controller/operational-plan.ts +2 -2
  162. package/src/cli/controller/work-mode.ts +6 -4
  163. package/src/cli/editing/edit-session.ts +60 -7
  164. package/src/cli/github/contracts.ts +16 -0
  165. package/src/cli/github/github.ts +135 -16
  166. package/src/cli/github/session.ts +2 -1
  167. package/src/cli/installer/managed-entries.ts +15 -1
  168. package/src/cli/local-bridge/console-automations.ts +54 -78
  169. package/src/cli/local-bridge/console-view-models.ts +1 -35
  170. package/src/cli/local-bridge/facade-api.ts +37 -264
  171. package/src/cli/local-bridge/frontend/src/App.tsx +3 -3
  172. package/src/cli/local-bridge/frontend/src/features/automations/AutomationsView.tsx +23 -13
  173. package/src/cli/local-bridge/frontend/src/features/capabilities/CapabilitiesView.tsx +37 -4
  174. package/src/cli/local-bridge/frontend/src/features/overview/OverviewView.tsx +6 -4
  175. package/src/cli/local-bridge/frontend/src/lib/api.ts +1 -6
  176. package/src/cli/local-bridge/frontend/src/lib/routes.ts +1 -1
  177. package/src/cli/local-bridge/frontend/src/shell/Sidebar.tsx +2 -2
  178. package/src/cli/local-bridge/frontend/src/styles.css +1 -1
  179. package/src/cli/local-bridge/frontend/src/types.ts +5 -8
  180. package/src/cli/local-bridge/job-store.ts +1 -490
  181. package/src/cli/local-bridge/server.ts +2 -515
  182. package/src/cli/local-bridge/ui-dist/app.css +1 -1
  183. package/src/cli/local-bridge/ui-dist/app.js +3703 -3757
  184. package/src/cli/mcp/auth.ts +5 -5
  185. package/src/cli/mcp/instructions.ts +3 -2
  186. package/src/cli/mcp/legacy-tool-service.ts +10 -222
  187. package/src/cli/mcp/multi-repository.ts +16 -5
  188. package/src/cli/mcp/oauth.ts +175 -26
  189. package/src/cli/mcp/post-finalize-work-attribution.ts +150 -0
  190. package/src/cli/mcp/repository-tools.ts +282 -146
  191. package/src/cli/mcp/server.ts +586 -116
  192. package/src/cli/mcp/tool-capability-cache.ts +1 -1
  193. package/src/cli/mcp/tool-contract.ts +14 -0
  194. package/src/cli/mcp/toolset-selection.ts +1 -1
  195. package/src/cli/mcp/toolset.ts +44 -7
  196. package/src/cli/mcp/transports/http.ts +124 -14
  197. package/src/cli/mcp/transports/session-registry.ts +43 -5
  198. package/src/cli/mcp/types.ts +3 -3
  199. package/src/cli/repositories/command-classifier.ts +103 -21
  200. package/src/cli/repositories/command-executor.ts +78 -526
  201. package/src/cli/repositories/command-process.ts +108 -0
  202. package/src/cli/repositories/command-scope.ts +99 -41
  203. package/src/cli/repositories/controller-home.ts +7 -8
  204. package/src/cli/repositories/git-command-executor.ts +2 -0
  205. package/src/cli/repositories/locks.ts +53 -16
  206. package/src/cli/repositories/repository-snapshot.ts +124 -0
  207. package/src/cli/repositories/selected-path-actions.ts +17 -9
  208. package/src/cli/repositories/structured-git.ts +118 -6
  209. package/src/cli/repository/inspector.ts +563 -65
  210. package/src/cli/repository/session-cache.ts +58 -17
  211. package/src/cli/v81-entry.ts +1 -4
  212. package/src/effects/fs-transaction.ts +85 -17
  213. package/src/runtime/context/codegraph-read-provider.ts +233 -2
  214. package/src/runtime/context/codegraph-sidecar.cjs +74 -40
  215. package/src/runtime/context/generic-lsp-provider.ts +355 -0
  216. package/src/runtime/context/lsp-client.ts +283 -0
  217. package/src/runtime/context/semantic-navigation-contract.ts +61 -0
  218. package/src/runtime/context/semantic-navigation.ts +223 -0
  219. package/src/runtime/context/swift-navigation.ts +565 -0
  220. package/src/runtime/context/typescript-navigation.ts +226 -0
  221. package/src/runtime/control-plane/dispatch-priority.ts +28 -0
  222. package/src/runtime/control-plane/execution/direct-edit-work-completion.ts +48 -16
  223. package/src/runtime/control-plane/execution/edit-validation-coordinator.ts +86 -24
  224. package/src/runtime/control-plane/execution/repository-work-attribution.ts +106 -0
  225. package/src/runtime/control-plane/execution/retained-work-resume.ts +203 -0
  226. package/src/runtime/control-plane/execution/session-store.ts +18 -0
  227. package/src/runtime/control-plane/execution/verification-evidence.ts +58 -3
  228. package/src/runtime/control-plane/execution/work-completion-authority.ts +58 -0
  229. package/src/runtime/control-plane/execution/work-evidence-policy.ts +200 -0
  230. package/src/runtime/control-plane/execution/work-execution-support.ts +386 -0
  231. package/src/runtime/control-plane/execution/work-finalization-service.ts +1913 -0
  232. package/src/runtime/control-plane/execution/work-handle-authority.ts +96 -0
  233. package/src/runtime/control-plane/execution/work-handle-store.ts +27 -2
  234. package/src/runtime/control-plane/execution/work-lifecycle-audit.ts +258 -0
  235. package/src/runtime/control-plane/execution/work-operation-service.ts +468 -0
  236. package/src/runtime/control-plane/execution/work-path-scope.ts +37 -0
  237. package/src/runtime/control-plane/execution/work-preparation-service.ts +479 -0
  238. package/src/runtime/control-plane/execution/work-process-evidence.ts +129 -0
  239. package/src/runtime/control-plane/execution/work-remote-delivery.ts +215 -0
  240. package/src/runtime/control-plane/execution/work-task-receipt.ts +22 -18
  241. package/src/runtime/control-plane/execution/work-terminal-cleanup.ts +156 -11
  242. package/src/runtime/control-plane/execution/work-validation-reconciler.ts +22 -14
  243. package/src/runtime/control-plane/execution/work-verification-context.ts +56 -0
  244. package/src/runtime/control-plane/execution/work-verification-service.ts +388 -0
  245. package/src/runtime/control-plane/execution/work-verification-snapshot.ts +90 -9
  246. package/src/runtime/control-plane/facade/capability-registry.ts +129 -25
  247. package/src/runtime/control-plane/facade/controller-round-relay.ts +1118 -0
  248. package/src/runtime/control-plane/facade/controller-session-store.ts +302 -12
  249. package/src/runtime/control-plane/facade/goal-workloop-access.ts +17 -1
  250. package/src/runtime/control-plane/facade/goal-workloop.ts +928 -190
  251. package/src/runtime/control-plane/facade/index.ts +3 -0
  252. package/src/runtime/control-plane/facade/plan-contract-store.ts +275 -42
  253. package/src/runtime/control-plane/facade/repository-work-admission.ts +147 -0
  254. package/src/runtime/control-plane/facade/requirement-authority.ts +83 -0
  255. package/src/runtime/control-plane/facade/semantic-admission.ts +205 -0
  256. package/src/runtime/control-plane/facade/suggested-actions.ts +5 -1
  257. package/src/runtime/control-plane/facade/types.ts +90 -10
  258. package/src/runtime/control-plane/facade/work-contract-store.ts +157 -261
  259. package/src/runtime/control-plane/facade/work-state-machine.ts +250 -0
  260. package/src/runtime/control-plane/generated-cache-retention.ts +271 -0
  261. package/src/runtime/control-plane/global-scheduler/config.ts +44 -0
  262. package/src/runtime/control-plane/global-scheduler/dispatch-capacity.ts +80 -0
  263. package/src/runtime/control-plane/global-scheduler/durable-admission.ts +46 -0
  264. package/src/runtime/control-plane/global-scheduler/maintenance.ts +152 -0
  265. package/src/runtime/control-plane/global-scheduler/projection-refresh.ts +84 -0
  266. package/src/runtime/control-plane/global-scheduler/reconciliation.ts +3 -3
  267. package/src/runtime/control-plane/global-scheduler/scheduler.ts +278 -543
  268. package/src/runtime/control-plane/global-scheduler/source-scan.ts +59 -0
  269. package/src/runtime/control-plane/global-scheduler/state.ts +88 -0
  270. package/src/runtime/control-plane/global-scheduler/worker-attachment.ts +57 -0
  271. package/src/runtime/control-plane/global-scheduler/worker-exit-decision.ts +38 -0
  272. package/src/runtime/control-plane/global-scheduler/worker-exit-reconciler.ts +175 -0
  273. package/src/runtime/control-plane/global-scheduler/worker-launch.ts +111 -0
  274. package/src/runtime/control-plane/global-scheduler/worker-lifecycle-store.ts +64 -0
  275. package/src/runtime/control-plane/global-scheduler/worker-lifecycle.ts +150 -0
  276. package/src/runtime/control-plane/global-scheduler/worker-process.ts +119 -0
  277. package/src/runtime/control-plane/global-scheduler/worker-stderr.ts +60 -0
  278. package/src/runtime/control-plane/launcher/chatgpt-work-binding-store.ts +63 -0
  279. package/src/runtime/control-plane/launcher/chatgpt-work-continuation.ts +365 -117
  280. package/src/runtime/control-plane/launcher/launch-reservation-store.ts +89 -0
  281. package/src/runtime/control-plane/launcher/thin-launcher.ts +244 -11
  282. package/src/runtime/control-plane/persistence/controller-home-migration.ts +487 -0
  283. package/src/runtime/control-plane/persistence/requirement-store.ts +0 -12
  284. package/src/runtime/control-plane/persistence/sqlite-store.ts +33 -2
  285. package/src/runtime/control-plane/release-retention.ts +62 -2
  286. package/src/runtime/control-plane/repo-actor/registry.ts +4 -2
  287. package/src/runtime/control-plane/routing/route-policy.ts +24 -21
  288. package/src/runtime/control-plane/routing/workspace-admission.ts +70 -0
  289. package/src/runtime/control-plane/runtime-status-client.ts +0 -2
  290. package/src/runtime/diagnostics/mcp-timing.ts +13 -0
  291. package/src/runtime/diagnostics/mcp-transport-benchmark.ts +129 -0
  292. package/src/runtime/diagnostics/performance.ts +24 -1
  293. package/src/runtime/evidence/event-ledger.ts +1 -1
  294. package/src/runtime/evidence/process-check-execution.ts +18 -0
  295. package/src/runtime/execution/jobs/child-reference.ts +6 -2
  296. package/src/runtime/execution/jobs/store.ts +58 -4
  297. package/src/runtime/execution/jobs/types.ts +1 -1
  298. package/src/runtime/execution/managed-workspace.ts +203 -1
  299. package/src/runtime/execution/process-runtime/check-facade.ts +51 -4
  300. package/src/runtime/execution/process-runtime/check-result.ts +88 -0
  301. package/src/runtime/execution/process-runtime/check-runner-sidecar.ts +7 -1
  302. package/src/runtime/execution/process-runtime/check-scheduling.ts +114 -0
  303. package/src/runtime/execution/process-runtime/command-facade.ts +367 -40
  304. package/src/runtime/execution/process-runtime/gc.ts +38 -6
  305. package/src/runtime/execution/process-runtime/lightweight-managed.ts +1201 -0
  306. package/src/runtime/execution/process-runtime/persisted-check.ts +51 -5
  307. package/src/runtime/execution/process-runtime/resource-claims.ts +10 -6
  308. package/src/runtime/execution/process-runtime/runtime.ts +53 -3
  309. package/src/runtime/execution/process-runtime/store.ts +1 -0
  310. package/src/runtime/execution/process-runtime/types.ts +4 -12
  311. package/src/runtime/execution/thin-harness/async-process.ts +12 -2
  312. package/src/runtime/execution/thin-harness/search-worker.ts +54 -5
  313. package/src/runtime/execution/work-activity.ts +21 -0
  314. package/src/runtime/execution/workers/executor.ts +18 -61
  315. package/src/runtime/gateway/mcp/execution-tools.ts +35 -1852
  316. package/src/runtime/gateway/mcp/legacy-ios-tool-adapter.ts +61 -0
  317. package/src/runtime/gateway/mcp/process-tools.ts +83 -37
  318. package/src/runtime/gateway/mcp/router.ts +186 -84
  319. package/src/runtime/gateway/mcp/runtime-tool-definitions.ts +565 -0
  320. package/src/runtime/gateway/mcp/runtime-tools.ts +2378 -1947
  321. package/src/runtime/plugins/app-store-connect-adapter.ts +216 -47
  322. package/src/runtime/plugins/browser-adapter.ts +2662 -829
  323. package/src/runtime/plugins/browser-automation-service.ts +10 -0
  324. package/src/runtime/plugins/browser-handoff-host.ts +2 -25
  325. package/src/runtime/plugins/browser-handoff.ts +110 -5
  326. package/src/runtime/plugins/browser-macos-bridge.ts +701 -63
  327. package/src/runtime/plugins/browser-manifest-surface.ts +603 -0
  328. package/src/runtime/plugins/browser-native-session-service.ts +222 -0
  329. package/src/runtime/plugins/browser-node-bridge.ts +6 -34
  330. package/src/runtime/plugins/browser-provider-registry.ts +109 -0
  331. package/src/runtime/plugins/browser-runtime-contract.ts +154 -0
  332. package/src/runtime/plugins/browser-runtime.ts +193 -0
  333. package/src/runtime/plugins/browser-session-authority.ts +332 -0
  334. package/src/runtime/plugins/browser-session-store.ts +121 -0
  335. package/src/runtime/plugins/browser-session-types.ts +98 -0
  336. package/src/runtime/plugins/capability-authorization-grants.ts +354 -0
  337. package/src/runtime/plugins/desktop-operator-registration.ts +10 -4
  338. package/src/runtime/plugins/external-adapter.ts +358 -3
  339. package/src/runtime/plugins/external-unix-socket.ts +7 -1
  340. package/src/runtime/plugins/first-party-registry.ts +2 -0
  341. package/src/runtime/plugins/github-adapter.ts +125 -0
  342. package/src/runtime/plugins/ios/agent-device-provider.ts +1 -1
  343. package/src/runtime/plugins/ios/agent-device-typed-provider.ts +52 -9
  344. package/src/runtime/plugins/ios/remote-xpc-hid.ts +1171 -0
  345. package/src/runtime/plugins/ios-adapter.ts +106 -16
  346. package/src/runtime/plugins/ios-agent-device.ts +234 -64
  347. package/src/runtime/plugins/ios-physical-device.ts +1143 -392
  348. package/src/runtime/plugins/lightweight-action.ts +153 -0
  349. package/src/runtime/plugins/local-system-adapter.ts +123 -6
  350. package/src/runtime/plugins/macos-capability-broker.ts +47 -9
  351. package/src/runtime/plugins/plugin-action-sidecar.ts +59 -0
  352. package/src/runtime/plugins/plugin-management-adapter.ts +49 -8
  353. package/src/runtime/plugins/resend-adapter.ts +1 -1
  354. package/src/runtime/plugins/store.ts +330 -33
  355. package/src/runtime/plugins/types.ts +38 -3
  356. package/src/runtime/plugins/xiaohongshu-publish.ts +515 -0
  357. package/src/runtime/projections/git-status-sampler.ts +16 -16
  358. package/src/runtime/projections/materialized-view.ts +28 -10
  359. package/src/runtime/recovery/maintenance-executor.ts +426 -25
  360. package/src/runtime/root/mcp-transport.ts +180 -10
  361. package/src/runtime/root/package-connector-service.ts +230 -10
  362. package/src/runtime/root/package-runtime-release.ts +143 -17
  363. package/src/runtime/root/package-runtime-service.ts +461 -22
  364. package/src/runtime/root/release-manifest.ts +144 -18
  365. package/src/runtime/root/release-materialize.ts +125 -32
  366. package/src/runtime/root/release-store.ts +48 -11
  367. package/src/runtime/root/runtime.ts +23 -5
  368. package/src/runtime/root/service.ts +100 -29
  369. package/src/runtime/root/types.ts +12 -0
  370. package/src/runtime/safe-tooling/ios-development.ts +321 -63
  371. package/src/runtime/shared/trusted-executable.ts +62 -0
  372. package/src/runtime/shared/trusted-node-executable.ts +48 -0
  373. package/src/runtime/standalone-recovery/core.ts +1079 -240
  374. package/src/runtime/standalone-recovery/entry.ts +100 -24
  375. package/src/runtime/standalone-recovery/installer.ts +20 -4
  376. package/src/runtime/standalone-recovery/service-labels.ts +2 -0
  377. package/src/runtime/standalone-recovery/watchdog-heartbeat.ts +160 -0
  378. package/src/runtime/workflow/schedules/browser-probe.ts +81 -3
  379. package/src/runtime/workflow/schedules/engine.ts +364 -202
  380. package/src/runtime/workflow/schedules/github-issue-watch.ts +158 -0
  381. package/src/runtime/workflow/schedules/settlement.ts +64 -17
  382. package/src/runtime/workflow/schedules/store.ts +209 -19
  383. package/src/runtime/workflow/schedules/types.ts +2 -4
  384. package/src/runtime/workflow/schedules/work-continuation.ts +26 -13
  385. package/src/testing/test-governance.ts +8 -40
  386. package/assets/reference-configs/heartbeat-triage.md +0 -57
  387. package/assets/templates/helpers/harness-trace-grade.sh +0 -150
  388. package/assets/templates/helpers/heartbeat-triage.sh +0 -361
  389. package/assets/templates/helpers/maintenance-triage.sh +0 -167
  390. package/docs/architecture/current/approved-target-architecture.zh-CN.md +0 -1501
  391. package/docs/architecture/current/architecture-invariants.md +0 -331
  392. package/docs/architecture/current/automation-and-schedule-engine.md +0 -459
  393. package/docs/architecture/current/codegraph-context-provider.md +0 -238
  394. package/docs/architecture/current/control-plane-authority-inventory.md +0 -94
  395. package/docs/architecture/current/controller-repository-decoupling.md +0 -148
  396. package/docs/architecture/current/dispatch-and-agent-strategy.md +0 -373
  397. package/docs/architecture/current/entity-model.md +0 -612
  398. package/docs/architecture/current/external-plugin-governance.md +0 -396
  399. package/docs/architecture/current/failure-recovery.md +0 -447
  400. package/docs/architecture/current/governance.md +0 -213
  401. package/docs/architecture/current/human-interaction-plane.md +0 -87
  402. package/docs/architecture/current/implementation-status.md +0 -128
  403. package/docs/architecture/current/job-and-run-lifecycle.md +0 -413
  404. package/docs/architecture/current/legacy-governance.md +0 -58
  405. package/docs/architecture/current/local-workspace-target-grants.md +0 -70
  406. package/docs/architecture/current/migration-roadmap.md +0 -105
  407. package/docs/architecture/current/multi-repository-execution.md +0 -371
  408. package/docs/architecture/current/operations-runbook.md +0 -45
  409. package/docs/architecture/current/peer-supercontroller-runtime.md +0 -32
  410. package/docs/architecture/current/personal-assistant-plugin-baseline.md +0 -206
  411. package/docs/architecture/current/repository-access-modes.md +0 -102
  412. package/docs/architecture/current/route-policy.md +0 -15
  413. package/docs/architecture/current/route-session-concurrency.md +0 -85
  414. package/docs/architecture/current/runtime-architecture-simplification.md +0 -214
  415. package/docs/architecture/current/runtime-directory-map.md +0 -49
  416. package/docs/architecture/current/runtime-health-and-resource-lifecycle.md +0 -85
  417. package/docs/architecture/current/scheduler-and-resource-claims.md +0 -422
  418. package/docs/architecture/current/session-aware-execution-and-authorization.md +0 -77
  419. package/docs/architecture/current/system-overview.md +0 -169
  420. package/docs/architecture/current/target-requirements-traceability.md +0 -202
  421. package/docs/architecture/current/thin-harness-governance-policy.md +0 -32
  422. package/docs/architecture/current/thin-harness-v1.md +0 -321
  423. package/docs/architecture/current/verification-and-release-gates.md +0 -391
  424. package/docs/forge-autonomous-goal-loop.md +0 -9
  425. package/docs/forge-provider-configuration.md +0 -7
  426. package/docs/operations/chatgpt-personal-assistant-runtime.md +0 -93
  427. package/docs/operations/gmail-assistant-routines.md +0 -186
  428. package/docs/operations/personal-assistant-operator-guide.md +0 -61
  429. package/docs/operations/provider-configuration.md +0 -31
  430. package/scripts/harness-trace-grade.sh +0 -150
  431. package/scripts/heartbeat-triage.sh +0 -361
  432. package/scripts/maintenance-triage.sh +0 -167
  433. package/src/cli/local-bridge/frontend/src/features/settings/SettingsView.tsx +0 -2
  434. package/src/cli/repositories/goal-registry.ts +0 -239
  435. package/src/runtime/assistant/action-proposals.ts +0 -279
  436. package/src/runtime/assistant/intent.ts +0 -395
  437. package/src/runtime/assistant/model-provider.ts +0 -460
  438. package/src/runtime/assistant/openapi.ts +0 -223
  439. package/src/runtime/assistant/policy.ts +0 -89
  440. package/src/runtime/assistant/readiness.ts +0 -258
  441. package/src/runtime/assistant/routine-runtime.ts +0 -477
  442. package/src/runtime/assistant/schedule-binding.ts +0 -171
  443. package/src/runtime/assistant/standing-grants.ts +0 -375
  444. package/src/runtime/assistant/store.ts +0 -243
  445. package/src/runtime/assistant/types.ts +0 -135
  446. package/src/runtime/control-plane/goal-loop/config-facade.ts +0 -700
  447. package/src/runtime/control-plane/goal-loop/config-store.ts +0 -570
  448. package/src/runtime/control-plane/goal-loop/config-types.ts +0 -213
  449. package/src/runtime/control-plane/goal-loop/executor-router.ts +0 -109
  450. package/src/runtime/control-plane/goal-loop/failure-taxonomy.ts +0 -149
  451. package/src/runtime/control-plane/goal-loop/goal-contract-store.ts +0 -304
  452. package/src/runtime/control-plane/goal-loop/goal-loop-engine.ts +0 -1001
  453. package/src/runtime/control-plane/goal-loop/handoff-packet.ts +0 -160
  454. package/src/runtime/control-plane/goal-loop/index.ts +0 -13
  455. package/src/runtime/control-plane/goal-loop/local-tools.ts +0 -290
  456. package/src/runtime/control-plane/goal-loop/policy-gates.ts +0 -203
  457. package/src/runtime/control-plane/goal-loop/provider-dispatch.ts +0 -93
  458. package/src/runtime/control-plane/goal-loop/provider-registry.ts +0 -545
  459. package/src/runtime/control-plane/goal-loop/types.ts +0 -378
  460. package/src/runtime/control-plane/startup-recovery.ts +0 -198
  461. package/src/runtime/gateway/mcp/resource-policy.ts +0 -186
  462. package/src/runtime/personal-assistant/gmail-triage-manager.ts +0 -231
  463. package/src/runtime/personal-assistant/reporting-runtime.ts +0 -344
  464. package/src/runtime/personal-assistant/triage-runtime.ts +0 -748
  465. package/src/runtime/workflow/findings/store.ts +0 -204
  466. package/src/runtime/workflow/findings/types.ts +0 -58
  467. package/src/runtime/workflow/portfolio/engine.ts +0 -20
  468. package/src/runtime/workflow/portfolio/store.ts +0 -185
  469. package/src/runtime/workflow/portfolio/types.ts +0 -29
@@ -4,7 +4,14 @@ import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, re
4
4
  import { homedir } from 'os';
5
5
  import { basename, dirname, isAbsolute, join, resolve } from 'path';
6
6
  import { observeRuntimeStatus } from '../root/status';
7
- import { ensureForgeRuntimeLaunchAgentContract, forgeRuntimeServicePaths, readForgeRuntimeServiceConfig } from '../root/service';
7
+ import {
8
+ activeRuntimeEntrypoint,
9
+ activeRuntimeLaunchSpec,
10
+ ensureForgeRuntimeLaunchAgentContract,
11
+ forgeRuntimeServicePaths,
12
+ inspectForgeRuntimeLaunchAgentContract,
13
+ readForgeRuntimeServiceConfig,
14
+ } from '../root/service';
8
15
  import { loadRuntimeReleaseManifest } from '../root/release-manifest';
9
16
  import { assertRuntimeReleaseFiles, stageRuntimeReleaseFromCandidateSource, type StagedRuntimeRelease } from '../root/release-materialize';
10
17
  import {
@@ -15,7 +22,10 @@ import {
15
22
  type RuntimeReleaseAuthority,
16
23
  } from '../root/release-store';
17
24
  import type { RuntimeReleaseManifest } from '../root/types';
25
+ import { ensurePackageConnectorService, packageConnectorServicePaths, type PackageConnectorReleaseBinding } from '../root/package-connector-service';
18
26
  import { createRecoveryHttpTransport, type RecoveryHttpTransport } from './http-transport';
27
+ import { observeRecoveryWatchdogHealth } from './watchdog-heartbeat';
28
+ import { RECOVERY_GATEWAY_LABEL, RECOVERY_WATCHDOG_LABEL } from './service-labels';
19
29
  import {
20
30
  migrateStoppedRepoLocalControllerHomeStorage,
21
31
  repoLocalControllerHomeStorageNeedsMigration,
@@ -50,6 +60,12 @@ export interface PrimaryConnectorServiceConfig {
50
60
  platform: 'launchd';
51
61
  label: string;
52
62
  plistPath?: string;
63
+ /** Local OAuth MCP endpoint used to distinguish Connector failure from tunnel failure. */
64
+ localMcpUrl?: string;
65
+ minimumFailures?: number;
66
+ minimumFailureDurationMs?: number;
67
+ restartCooldownMs?: number;
68
+ maximumRestartAttempts?: number;
53
69
  postRestartVerifyTimeoutMs?: number;
54
70
  }
55
71
 
@@ -59,6 +75,8 @@ export interface RecoveryConfig {
59
75
  publicMcpUrl?: string;
60
76
  recoveryPublicUrl?: string;
61
77
  recoveryTunnelService?: PublicTunnelServiceConfig;
78
+ /** Optional public tunnel serving publicMcpUrl; independent from the OAuth Connector process. */
79
+ primaryPublicTunnelService?: PublicTunnelServiceConfig;
62
80
  primaryRuntimeService?: PrimaryRuntimeServiceConfig;
63
81
  primaryRuntimeSourceRoot?: string;
64
82
  primaryConnectorService?: PrimaryConnectorServiceConfig;
@@ -95,6 +113,7 @@ export type RecoveryMutationAction =
95
113
  | 'stage_and_activate_runtime_release'
96
114
  | 'restart_primary_connector'
97
115
  | 'restart_recovery_gateway'
116
+ | 'restart_recovery_watchdog'
98
117
  | 'repair_public_tunnel';
99
118
 
100
119
  interface RecoveryLock {
@@ -132,8 +151,8 @@ export interface VerifyStableRuntimeOptions {
132
151
  export interface RecoveryWatchdogDiagnosticEvidence {
133
152
  fingerprint: string;
134
153
  releaseIdentity: string;
135
- components: Array<'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_tunnel'>;
136
- failedProbes: Array<{ name: string; component: 'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_tunnel'; detail: string; status?: number }>;
154
+ components: Array<'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_watchdog' | 'recovery_tunnel'>;
155
+ failedProbes: Array<{ name: string; component: 'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_watchdog' | 'recovery_tunnel'; detail: string; status?: number }>;
137
156
  firstObservedAt: string;
138
157
  lastObservedAt: string;
139
158
  occurrences: number;
@@ -153,7 +172,7 @@ export interface RollbackResult {
153
172
  }
154
173
 
155
174
  export interface WatchdogDecision {
156
- action: 'healthy' | 'degraded' | 'repair_public_tunnel' | 'restart_recovery_gateway' | 'restart_primary_runtime' | 'rollback' | 'recovery_exhausted';
175
+ action: 'healthy' | 'degraded' | 'repair_public_tunnel' | 'restart_recovery_gateway' | 'restart_primary_connector' | 'restart_primary_runtime' | 'rollback' | 'recovery_exhausted';
157
176
  reason: string;
158
177
  }
159
178
 
@@ -175,10 +194,16 @@ export interface WatchdogState {
175
194
  publicTunnelFailures?: number;
176
195
  publicTunnelFirstFailureAt?: number;
177
196
  publicTunnelRepairFailures?: number;
197
+ primaryConnectorFailures?: number;
198
+ primaryConnectorFirstFailureAt?: number;
199
+ primaryConnectorRestartAttempts?: number;
200
+ primaryConnectorRestartFailures?: number;
201
+ primaryConnectorRestartLastAttemptAt?: number;
178
202
  recoveryGatewayRestartUsed?: boolean;
179
203
  recoveryReleaseRevision?: string;
180
204
  lastFullVerifyAt?: number;
181
205
  lastDecision?: WatchdogDecision['action'];
206
+ lastReason?: string;
182
207
  updatedAt?: string;
183
208
  }
184
209
 
@@ -300,6 +325,10 @@ function configuredRecoveryTunnel(config: RecoveryConfig): PublicTunnelServiceCo
300
325
  return config.recoveryTunnelService;
301
326
  }
302
327
 
328
+ function configuredPrimaryPublicTunnel(config: RecoveryConfig): PublicTunnelServiceConfig | undefined {
329
+ return config.primaryPublicTunnelService;
330
+ }
331
+
303
332
  function configuredRecoveryPublicUrl(config: RecoveryConfig): string | undefined {
304
333
  return config.recoveryPublicUrl;
305
334
  }
@@ -324,6 +353,7 @@ export function loadRecoveryConfig(controllerHome: string, explicit?: string): R
324
353
  ...(typeof loaded.publicMcpUrl === 'string' ? { publicMcpUrl: loaded.publicMcpUrl } : {}),
325
354
  ...(typeof loaded.recoveryPublicUrl === 'string' ? { recoveryPublicUrl: loaded.recoveryPublicUrl } : {}),
326
355
  ...(loaded.recoveryTunnelService ? { recoveryTunnelService: loaded.recoveryTunnelService } : {}),
356
+ ...(loaded.primaryPublicTunnelService ? { primaryPublicTunnelService: loaded.primaryPublicTunnelService } : {}),
327
357
  ...(loaded.primaryRuntimeService ? { primaryRuntimeService: loaded.primaryRuntimeService } : {}),
328
358
  ...(typeof loaded.primaryRuntimeSourceRoot === 'string' ? { primaryRuntimeSourceRoot: resolve(loaded.primaryRuntimeSourceRoot) } : {}),
329
359
  ...(loaded.primaryConnectorService ? { primaryConnectorService: loaded.primaryConnectorService } : {}),
@@ -423,6 +453,7 @@ function watchdogProbeComponent(name: string): RecoveryWatchdogDiagnosticEvidenc
423
453
  if (name === 'runtime_status') return 'runtime';
424
454
  if (name === 'active_gateway') return 'gateway';
425
455
  if (name === 'recovery_gateway') return 'recovery_gateway';
456
+ if (name === 'recovery_watchdog') return 'recovery_watchdog';
426
457
  if (name === 'recovery_external_http') return 'recovery_tunnel';
427
458
  return 'public_mcp';
428
459
  }
@@ -482,8 +513,34 @@ function recoveryLockOwnerAlive(lock: RecoveryLock): boolean {
482
513
  return observed === undefined || observed === lock.processStartTime;
483
514
  }
484
515
 
516
+ function recoveryLockOwnerAttributable(owner: RecoveryLock): boolean {
517
+ return Boolean(owner.action && owner.requestId?.trim());
518
+ }
519
+
520
+ function recoveryLockIdentityFailure(owner: RecoveryLock): string {
521
+ return `RECOVERY_OPERATION_LOCK_IDENTITY_UNCERTAIN: live mutation lock pid=${owner.pid} instance=${owner.instanceId} is missing action/request identity`;
522
+ }
523
+
524
+ function assertRecoveryLockOwnerAttributable(config: RecoveryConfig, owner: RecoveryLock): void {
525
+ if (recoveryLockOwnerAttributable(owner)) return;
526
+ audit(config, 'recovery_operation_lock_identity_uncertain', {
527
+ pid: owner.pid,
528
+ instanceId: owner.instanceId,
529
+ action: owner.action ?? null,
530
+ requestId: owner.requestId ?? null,
531
+ acquiredAt: owner.acquiredAt,
532
+ });
533
+ throw new Error(recoveryLockIdentityFailure(owner));
534
+ }
535
+
485
536
  function recoveryBusyDetail(owner: RecoveryLock): string {
486
- return `Recovery mutation already in progress: action=${owner.action ?? 'unknown'} request=${owner.requestId ?? 'unknown'} pid=${owner.pid}`;
537
+ return `Recovery mutation already in progress: action=${owner.action ?? 'invalid'} request=${owner.requestId ?? 'invalid'} pid=${owner.pid} instance=${owner.instanceId}`;
538
+ }
539
+
540
+ function liveRecoveryMutationLock(config: RecoveryConfig): RecoveryLock | undefined {
541
+ const owner = json<RecoveryLock>(lockPath(config));
542
+ if (!owner || !Number.isInteger(owner.pid) || owner.pid <= 0 || typeof owner.instanceId !== 'string' || !owner.instanceId.trim()) return undefined;
543
+ return recoveryLockOwnerAlive(owner) ? owner : undefined;
487
544
  }
488
545
 
489
546
  async function withLock<T>(
@@ -493,14 +550,16 @@ async function withLock<T>(
493
550
  ): Promise<RecoveryLockAttempt<T>> {
494
551
  const path = lockPath(config);
495
552
  mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
553
+ const instanceId = randomUUID();
554
+ const requestId = intent.requestId?.trim() || `internal:${intent.action}:${instanceId}`;
496
555
  const lock: RecoveryLock = {
497
556
  schemaVersion: 1,
498
557
  pid: process.pid,
499
- instanceId: randomUUID(),
558
+ instanceId,
500
559
  processStartTime: processStartTime(process.pid),
501
560
  acquiredAt: new Date().toISOString(),
502
561
  action: intent.action,
503
- ...(intent.requestId ? { requestId: intent.requestId } : {}),
562
+ requestId,
504
563
  };
505
564
 
506
565
  for (let attempt = 0; attempt < 2; attempt += 1) {
@@ -517,13 +576,31 @@ async function withLock<T>(
517
576
  if (!existing || !Number.isInteger(existing.pid) || typeof existing.instanceId !== 'string') {
518
577
  throw new Error('RECOVERY_OPERATION_LOCK_UNCERTAIN');
519
578
  }
520
- if (recoveryLockOwnerAlive(existing)) return { acquired: false, owner: existing };
579
+ if (recoveryLockOwnerAlive(existing)) {
580
+ assertRecoveryLockOwnerAttributable(config, existing);
581
+ return { acquired: false, owner: existing };
582
+ }
521
583
  const latest = json<RecoveryLock>(path);
522
584
  if (!latest || latest.instanceId !== existing.instanceId) {
523
- if (attempt === 1) return { acquired: false, owner: latest ?? existing };
585
+ if (attempt === 1) {
586
+ if (latest && recoveryLockOwnerAlive(latest)) {
587
+ assertRecoveryLockOwnerAttributable(config, latest);
588
+ return { acquired: false, owner: latest };
589
+ }
590
+ audit(config, 'recovery_operation_lock_race', {
591
+ existingInstanceId: existing.instanceId,
592
+ existingPid: existing.pid,
593
+ latestInstanceId: latest?.instanceId ?? null,
594
+ latestPid: latest?.pid ?? null,
595
+ });
596
+ throw new Error('RECOVERY_OPERATION_LOCK_RACE');
597
+ }
524
598
  continue;
525
599
  }
526
- if (recoveryLockOwnerAlive(latest)) return { acquired: false, owner: latest };
600
+ if (recoveryLockOwnerAlive(latest)) {
601
+ assertRecoveryLockOwnerAttributable(config, latest);
602
+ return { acquired: false, owner: latest };
603
+ }
527
604
  try { writeFileSync(`${path}.stale-${Date.now()}-${existing.instanceId}`, readFileSync(path)); } catch { /* evidence best effort */ }
528
605
  rmSync(path, { force: true });
529
606
  continue;
@@ -541,7 +618,23 @@ async function withLock<T>(
541
618
  }
542
619
 
543
620
  export async function runtimeStatus(config: RecoveryConfig) {
544
- return observeRuntimeStatus(config.controllerHome);
621
+ const observation = observeRuntimeStatus(config.controllerHome);
622
+ const watchdog = loadWatchdogState(config);
623
+ return {
624
+ ...observation,
625
+ recoveryWatchdog: {
626
+ lastDecision: watchdog.lastDecision,
627
+ lastReason: watchdog.lastReason,
628
+ updatedAt: watchdog.updatedAt,
629
+ failures: watchdog.failures,
630
+ rollbackUsed: watchdog.rollbackUsed,
631
+ runtimeRestartAttempts: watchdog.runtimeRestartAttempts ?? 0,
632
+ runtimeRestartFailures: watchdog.runtimeRestartFailures ?? 0,
633
+ primaryConnectorFailures: watchdog.primaryConnectorFailures ?? 0,
634
+ primaryConnectorRestartAttempts: watchdog.primaryConnectorRestartAttempts ?? 0,
635
+ primaryConnectorRestartFailures: watchdog.primaryConnectorRestartFailures ?? 0,
636
+ },
637
+ };
545
638
  }
546
639
 
547
640
  async function probe(transport: RecoveryHttpTransport, url: string, timeoutMs = 4_000): Promise<{ ok: boolean; detail: string; status?: number }> {
@@ -583,6 +676,72 @@ async function probeExternalMcp(transport: RecoveryHttpTransport, url: string):
583
676
  } finally { clearTimeout(timer); }
584
677
  }
585
678
 
679
+ function publicGatewayReadinessEndpoint(endpoint: string): string {
680
+ const url = new URL(endpoint);
681
+ url.pathname = '/transport-ready';
682
+ url.search = '';
683
+ url.hash = '';
684
+ return url.toString();
685
+ }
686
+
687
+ async function probePublicGatewayReadiness(
688
+ transport: RecoveryHttpTransport,
689
+ endpoint: string,
690
+ ): Promise<{ ok: boolean; detail: string; status?: number; value?: unknown }> {
691
+ const controller = new AbortController();
692
+ const timer = setTimeout(() => controller.abort('RECOVERY_HTTP_TIMEOUT'), 4_000);
693
+ const surface = 'transport-ready';
694
+ try {
695
+ const response = await transport.request({
696
+ url: publicGatewayReadinessEndpoint(endpoint),
697
+ headers: { accept: 'application/json' },
698
+ timeoutMs: 4_000,
699
+ signal: controller.signal,
700
+ });
701
+ if (response.status === 404 || response.status === 405) {
702
+ // Older Connectors predate the transport-scoped capacity surface. Their
703
+ // whole-control-plane /ready route is intentionally not a transport
704
+ // liveness contract and can depend on the Runtime that Recovery is in the
705
+ // middle of restarting. Falling back to it creates a circular cutover
706
+ // dependency. Verify the actual MCP transport instead: a successful
707
+ // initialize response or the expected OAuth Bearer challenge proves the
708
+ // legacy Connector transport is reachable without weakening 5xx/timeout
709
+ // failure handling.
710
+ const legacy = await probeExternalMcp(transport, endpoint);
711
+ return { ...legacy, detail: `legacy-mcp ${legacy.detail}` };
712
+ }
713
+ let sessionCapacity: unknown;
714
+ try {
715
+ const payload = JSON.parse(response.body) as { sessionCapacity?: unknown };
716
+ sessionCapacity = payload.sessionCapacity;
717
+ } catch { /* malformed readiness payload remains a failed/opaque probe */ }
718
+ const recoveryRecommended = Boolean(
719
+ sessionCapacity
720
+ && typeof sessionCapacity === 'object'
721
+ && (sessionCapacity as { recoveryRecommended?: unknown }).recoveryRecommended === true,
722
+ );
723
+ return {
724
+ ok: response.ok && !recoveryRecommended,
725
+ detail: `${surface} HTTP ${response.status}${recoveryRecommended ? '; session capacity recommends Connector recovery' : ''}`,
726
+ status: response.status,
727
+ ...(sessionCapacity === undefined ? {} : { value: sessionCapacity }),
728
+ };
729
+ } catch (error) {
730
+ return { ok: false, detail: `${surface} ${error instanceof Error ? error.message.slice(0, 180) : 'request failed'}` };
731
+ } finally {
732
+ clearTimeout(timer);
733
+ }
734
+ }
735
+
736
+ function connectorCapacityRecoveryRecommended(verified: VerifyResult): boolean {
737
+ const value = verified.probes.primary_connector_ready?.value;
738
+ return Boolean(
739
+ value
740
+ && typeof value === 'object'
741
+ && (value as { recoveryRecommended?: unknown }).recoveryRecommended === true,
742
+ );
743
+ }
744
+
586
745
  function mainToken(config: RecoveryConfig): string | undefined {
587
746
  const candidate = config.mainMcpTokenFile ?? join(config.controllerHome, 'mcp', 'mcp.tokens.json');
588
747
  const parsed = json<{ bearerToken?: unknown }>(candidate);
@@ -753,8 +912,26 @@ export async function verifyStableRuntime(
753
912
  probes.active_gateway = endpoint
754
913
  ? await probe(transport, runtimeHealthEndpoint(endpoint))
755
914
  : { ok: false, detail: 'canonical Runtime endpoint is unavailable' };
756
- if (config.publicMcpUrl) probes.external_mcp_http = await probeExternalMcp(transport, config.publicMcpUrl);
915
+ if (config.publicMcpUrl) {
916
+ probes.external_mcp_http = await probeExternalMcp(transport, config.publicMcpUrl);
917
+ const connectorReadinessEndpoint = config.primaryConnectorService?.localMcpUrl?.trim() || config.publicMcpUrl;
918
+ probes.primary_connector_ready = await probePublicGatewayReadiness(transport, connectorReadinessEndpoint);
919
+ }
920
+ const primaryConnectorLocal = await probePrimaryConnectorLocal(config, transport);
921
+ if (primaryConnectorLocal) probes.primary_connector_local = primaryConnectorLocal;
757
922
  if (config.gateway) probes.recovery_gateway = await probe(transport, `http://${config.gateway.host}:${config.gateway.port}/health`);
923
+ const watchdogHealth = observeRecoveryWatchdogHealth(config.controllerHome);
924
+ probes.recovery_watchdog = {
925
+ ok: watchdogHealth.ok,
926
+ detail: watchdogHealth.detail,
927
+ value: {
928
+ pulseAgeMs: watchdogHealth.pulseAgeMs,
929
+ tickAgeMs: watchdogHealth.tickAgeMs,
930
+ currentReleaseRevision: watchdogHealth.currentReleaseRevision,
931
+ watchdogReleaseRevision: watchdogHealth.runtimeIdentity?.releaseRevision,
932
+ watchdogPid: watchdogHealth.runtimeIdentity?.pid,
933
+ },
934
+ };
758
935
  const recoveryPublicUrl = configuredRecoveryPublicUrl(config);
759
936
  if (recoveryPublicUrl) probes.recovery_external_http = await probeExternalMcp(transport, recoveryPublicUrl);
760
937
  if (options.probeMcpProtocol !== false) Object.assign(probes, await probeMcp(config, transport));
@@ -801,10 +978,8 @@ export async function verifyStableRuntime(
801
978
  return result;
802
979
  }
803
980
  /** Explicitly records evidence only after the full independent verification passed. */
804
- export async function attestKnownGood(config: RecoveryConfig): Promise<ReleaseEvidence> {
805
- const locked = await withLock(config, { action: 'attest_known_good' }, async () => {
806
- const verified = await verifyStableRuntime(config);
807
- const authority = releaseAuthority(config);
981
+ function persistVerifiedKnownGood(config: RecoveryConfig, verified: VerifyResult): ReleaseEvidence {
982
+ const authority = releaseAuthority(config);
808
983
  const active = verified.releases.active;
809
984
  if (
810
985
  !verified.ok
@@ -825,7 +1000,10 @@ export async function attestKnownGood(config: RecoveryConfig): Promise<ReleaseEv
825
1000
  attestedAt: new Date().toISOString(),
826
1001
  };
827
1002
  const store = knownGood(config);
828
- const releases = [attested, ...store.releases.filter((entry) => entry.path !== active.path)].slice(0, 8);
1003
+ const releases = [
1004
+ attested,
1005
+ ...store.releases.filter((entry) => entry.path !== active.path && existsSync(entry.path)),
1006
+ ].slice(0, 8);
829
1007
  writeJson(statePath(config), { schemaVersion: 1, releases, updatedAt: new Date().toISOString() } satisfies KnownGoodStore);
830
1008
  audit(config, 'known_good_attested', {
831
1009
  revision: active.revision,
@@ -833,8 +1011,13 @@ export async function attestKnownGood(config: RecoveryConfig): Promise<ReleaseEv
833
1011
  manifestSha256: active.manifestSha256,
834
1012
  releaseAuthorityRevision: authority.revision,
835
1013
  });
836
- return attested;
837
- });
1014
+ return attested;
1015
+ }
1016
+
1017
+ export async function attestKnownGood(config: RecoveryConfig): Promise<ReleaseEvidence> {
1018
+ const locked = await withLock(config, { action: 'attest_known_good' }, async () => (
1019
+ persistVerifiedKnownGood(config, await verifyStableRuntime(config))
1020
+ ));
838
1021
  if (!locked.acquired) throw new Error(recoveryBusyDetail(locked.owner));
839
1022
  return locked.value;
840
1023
  }
@@ -849,8 +1032,8 @@ function quarantine(config: RecoveryConfig, release: ReleaseEvidence | undefined
849
1032
  async function rollbackPreviousLocked(config: RecoveryConfig, reason: string): Promise<RollbackResult> {
850
1033
  const before = await verifyStableRuntime(config);
851
1034
  const active = before.releases.active;
852
- if (matchingKnownGood(config, active)) {
853
- return { ok: true, noOp: true, detail: 'active whole-Runtime release is already independently attested known-good', verify: before };
1035
+ if (before.ok && matchingKnownGood(config, active)) {
1036
+ return { ok: true, noOp: true, detail: 'active whole-Runtime release is currently healthy and independently attested known-good', verify: before };
854
1037
  }
855
1038
  if (before.runtime.running) {
856
1039
  return {
@@ -911,7 +1094,9 @@ export async function reconnectMain(config: RecoveryConfig): Promise<{ ok: boole
911
1094
  // intentionally a bounded health/reconnect observation, never a rollout.
912
1095
  const verified = await verifyStableRuntime(config);
913
1096
  const publicProbe = verified.probes.external_mcp_http;
914
- const ok = verified.probes.active_gateway?.ok === true && (publicProbe?.ok === true || publicProbe?.status === 401);
1097
+ const ok = verified.probes.active_gateway?.ok === true
1098
+ && verified.probes.primary_connector_ready?.ok !== false
1099
+ && (publicProbe?.ok === true || publicProbe?.status === 401);
915
1100
  audit(config, 'reconnect_main', { ok, externalStatus: publicProbe?.status });
916
1101
  return { ok, detail: ok ? 'canonical Runtime Gateway and primary endpoint are reachable; client session may refresh' : 'primary endpoint remains unavailable; recovery channel remains independent', verify: verified };
917
1102
  }
@@ -1037,6 +1222,16 @@ export function decideWatchdog(input: {
1037
1222
  rollbackUsed: boolean;
1038
1223
  recoveryGatewayFailed?: boolean;
1039
1224
  recoveryGatewayRestartUsed?: boolean;
1225
+ primaryConnectorConfigured?: boolean;
1226
+ primaryConnectorFailed?: boolean;
1227
+ primaryConnectorFailures?: number;
1228
+ primaryConnectorFirstFailureAt?: number;
1229
+ primaryConnectorRestartAttempts?: number;
1230
+ primaryConnectorMaximumRestartAttempts?: number;
1231
+ primaryConnectorRestartLastAttemptAt?: number;
1232
+ primaryConnectorRestartCooldownMs?: number;
1233
+ primaryConnectorMinimumFailures?: number;
1234
+ primaryConnectorMinimumFailureDurationMs?: number;
1040
1235
  primaryRuntimeFailed?: boolean;
1041
1236
  runtimeRestartAttempts?: number;
1042
1237
  runtimeMaximumRestartAttempts?: number;
@@ -1064,6 +1259,22 @@ export function decideWatchdog(input: {
1064
1259
  if (failures >= minimumFailures && sustained) return { action: 'repair_public_tunnel', reason: 'local runtime and Recovery Gateway are healthy while the dedicated Recovery public endpoint is unavailable' };
1065
1260
  return { action: 'degraded', reason: 'Recovery tunnel failure has not yet met the bounded repair threshold' };
1066
1261
  }
1262
+ if (input.primaryConnectorConfigured && input.primaryConnectorFailed) {
1263
+ const failures = input.primaryConnectorFailures ?? 0;
1264
+ const minimumFailures = Math.max(1, input.primaryConnectorMinimumFailures ?? 2);
1265
+ const minimumDurationMs = Math.max(0, input.primaryConnectorMinimumFailureDurationMs ?? 5_000);
1266
+ const sustained = input.primaryConnectorFirstFailureAt !== undefined && now - input.primaryConnectorFirstFailureAt >= minimumDurationMs;
1267
+ const attempts = input.primaryConnectorRestartAttempts ?? 0;
1268
+ const maximumAttempts = Math.max(1, input.primaryConnectorMaximumRestartAttempts ?? 3);
1269
+ const cooldownMs = Math.max(0, input.primaryConnectorRestartCooldownMs ?? 30_000);
1270
+ const cooldownElapsed = input.primaryConnectorRestartLastAttemptAt === undefined || now - input.primaryConnectorRestartLastAttemptAt >= cooldownMs;
1271
+ if (failures >= minimumFailures && sustained && attempts < maximumAttempts && cooldownElapsed) {
1272
+ return { action: 'restart_primary_connector', reason: `local Runtime is healthy but the primary Connector endpoint is unavailable; restart attempt ${attempts + 1}/${maximumAttempts}` };
1273
+ }
1274
+ return { action: 'degraded', reason: attempts >= maximumAttempts
1275
+ ? `primary Connector restart budget exhausted (${attempts}/${maximumAttempts}); keeping the local Runtime available for independent Recovery`
1276
+ : 'primary Connector failure has not yet met its bounded restart threshold or cooldown' };
1277
+ }
1067
1278
  if (input.failures === 0) return { action: 'healthy', reason: 'all recovery probes healthy' };
1068
1279
  const recoveryRestartSustained = input.firstFailureAt !== undefined && now - input.firstFailureAt >= 5_000;
1069
1280
  if (input.failures >= 2 && recoveryRestartSustained && input.recoveryGatewayFailed && !input.recoveryGatewayRestartUsed) {
@@ -1220,6 +1431,7 @@ export interface PrimaryRuntimeRecoveryDependencies {
1220
1431
  runCommand?: CommandRunner;
1221
1432
  verifyLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
1222
1433
  runtimeRunning?: (config: RecoveryConfig) => boolean;
1434
+ ensureRuntimeLaunchContract?: (controllerHome: string) => void;
1223
1435
  now?: () => number;
1224
1436
  sleep?: (ms: number) => Promise<void>;
1225
1437
  }
@@ -1253,11 +1465,16 @@ function primaryRuntimeLaunchdService(config: RecoveryConfig, uid: number): Laun
1253
1465
  }
1254
1466
 
1255
1467
  export interface PrimaryConnectorRecoveryDependencies {
1468
+ /** Durable caller identity propagated into the Recovery mutation lock and audit trail. */
1469
+ requestId?: string;
1256
1470
  platform?: NodeJS.Platform;
1257
1471
  currentUid?: () => Promise<number | undefined>;
1258
1472
  runCommand?: CommandRunner;
1259
1473
  verifyLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
1260
1474
  reconnect?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string; verify: VerifyResult }>;
1475
+ probeConnectorLocal?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string; status?: number } | undefined>;
1476
+ probeConnectorOwnership?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
1477
+ repairConnectorBinding?: (config: RecoveryConfig) => Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string }>;
1261
1478
  now?: () => number;
1262
1479
  sleep?: (ms: number) => Promise<void>;
1263
1480
  }
@@ -1272,13 +1489,98 @@ function primaryConnectorLaunchdService(config: RecoveryConfig, uid: number): La
1272
1489
  return { uid, domain, target: `${domain}/${label}`, label, plistPath };
1273
1490
  }
1274
1491
 
1492
+ async function probePrimaryConnectorLocal(
1493
+ config: RecoveryConfig,
1494
+ transport = createRecoveryHttpTransport(config.controllerHome),
1495
+ ): Promise<{ ok: boolean; detail: string; status?: number } | undefined> {
1496
+ const localMcpUrl = config.primaryConnectorService?.localMcpUrl?.trim();
1497
+ if (!localMcpUrl) return undefined;
1498
+ return probeExternalMcp(transport, localMcpUrl);
1499
+ }
1500
+
1501
+ function activePackageConnectorReleaseBinding(config: RecoveryConfig): PackageConnectorReleaseBinding | undefined {
1502
+ const configured = config.primaryConnectorService;
1503
+ if (!configured || configured.platform !== 'launchd' || !configured.localMcpUrl?.trim()) return undefined;
1504
+ const authority = readRuntimeReleaseAuthority(config.controllerHome);
1505
+ if (!authority) return undefined;
1506
+ const releaseRoot = dirname(resolve(authority.active.manifestPath));
1507
+ const packageRoot = join(releaseRoot, 'package');
1508
+ const paths = packageConnectorServicePaths(config.controllerHome);
1509
+ const configuredPlist = resolve(configured.plistPath ?? paths.installedPlistPath);
1510
+ if (configured.label !== paths.label || configuredPlist !== resolve(paths.installedPlistPath)) return undefined;
1511
+ if (!existsSync(join(packageRoot, 'src', 'cli', 'index.ts')) || !existsSync(join(packageRoot, 'src', 'runtime', 'shared', 'node-ts-loader.mjs'))) return undefined;
1512
+ return { releaseId: authority.active.releaseId, releaseRoot, packageRoot };
1513
+ }
1514
+
1515
+ async function repairPrimaryConnectorBinding(config: RecoveryConfig): Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string }> {
1516
+ const release = activePackageConnectorReleaseBinding(config);
1517
+ const endpoint = config.primaryConnectorService?.localMcpUrl?.trim();
1518
+ if (!release || !endpoint) return { ok: true, attempted: false, noOp: true, detail: 'primary Connector does not use the canonical package-release binding contract' };
1519
+ try {
1520
+ const result = await ensurePackageConnectorService({ release, controllerHome: config.controllerHome, endpoint, platform: 'darwin' });
1521
+ const attempted = result.reused !== true;
1522
+ return {
1523
+ ok: true,
1524
+ attempted,
1525
+ ...(attempted ? {} : { noOp: true }),
1526
+ detail: attempted
1527
+ ? `primary Connector binding repaired to active immutable Runtime release ${release.releaseId}`
1528
+ : `primary Connector binding already matches active immutable Runtime release ${release.releaseId}`,
1529
+ };
1530
+ } catch (error) {
1531
+ return { ok: false, attempted: true, detail: error instanceof Error ? error.message : 'primary Connector binding repair failed' };
1532
+ }
1533
+ }
1534
+
1535
+ async function probePrimaryConnectorOwnership(
1536
+ config: RecoveryConfig,
1537
+ service: LaunchdService,
1538
+ runCommand: CommandRunner = command,
1539
+ ): Promise<{ ok: boolean; detail: string }> {
1540
+ const localMcpUrl = config.primaryConnectorService?.localMcpUrl?.trim();
1541
+ if (!localMcpUrl) return { ok: false, detail: 'primary Connector local MCP endpoint is not configured' };
1542
+ let port: number;
1543
+ try {
1544
+ const parsed = new URL(localMcpUrl);
1545
+ const rawPort = parsed.port || (parsed.protocol === 'https:' ? '443' : parsed.protocol === 'http:' ? '80' : '');
1546
+ port = Number(rawPort);
1547
+ if (!Number.isInteger(port) || port < 1 || port > 65_535) throw new Error('invalid port');
1548
+ } catch {
1549
+ return { ok: false, detail: `primary Connector local MCP endpoint has no verifiable TCP port: ${localMcpUrl}` };
1550
+ }
1551
+ const printed = await runCommand('launchctl', ['print', service.target], 5_000);
1552
+ const pidMatch = printed.ok ? printed.stdout.match(/\bpid\s*=\s*(\d+)/) : null;
1553
+ const pid = pidMatch ? Number(pidMatch[1]) : undefined;
1554
+ if (!pid || !Number.isInteger(pid)) {
1555
+ return { ok: false, detail: `configured primary Connector launchd service has no live pid: ${service.target}` };
1556
+ }
1557
+ const listening = await runCommand('lsof', ['-nP', '-a', '-p', String(pid), `-iTCP:${port}`, '-sTCP:LISTEN', '-t'], 5_000);
1558
+ const listenerPids = listening.stdout.split(/\s+/).map((value) => Number(value)).filter(Number.isInteger);
1559
+ const ok = listening.ok && listenerPids.includes(pid);
1560
+ return {
1561
+ ok,
1562
+ detail: ok
1563
+ ? `configured primary Connector pid ${pid} owns TCP ${port}`
1564
+ : `configured primary Connector pid ${pid} does not own TCP ${port}`,
1565
+ };
1566
+ }
1567
+
1275
1568
  export async function restartPrimaryConnector(
1276
1569
  config: RecoveryConfig,
1277
1570
  dependencies: PrimaryConnectorRecoveryDependencies = {},
1278
1571
  ): Promise<PrimaryConnectorRestartResult> {
1279
1572
  const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
1280
1573
  const initialLocal = await verifyLocal(config);
1281
- if (!initialLocal.ok) {
1574
+ // `verifyLocalRuntime` also reports the Connector's own loopback probe.
1575
+ // That probe is expected to be false when this recovery action is needed,
1576
+ // so it must not prevent a healthy canonical Runtime from repairing the
1577
+ // Connector service.
1578
+ const canonicalRuntimeHealthy = initialLocal.runtime.ok
1579
+ && initialLocal.runtime.running
1580
+ && initialLocal.runtime.ready
1581
+ && !initialLocal.runtime.stale
1582
+ && initialLocal.probes.active_gateway?.ok === true;
1583
+ if (!canonicalRuntimeHealthy) {
1282
1584
  return {
1283
1585
  ok: false,
1284
1586
  attempted: false,
@@ -1296,21 +1598,107 @@ export async function restartPrimaryConnector(
1296
1598
  return { ok: false, attempted: false, noOp: true, detail: 'primary Connector launchd service is not configured or installed', verify: initialLocal };
1297
1599
  }
1298
1600
  const reconnect = dependencies.reconnect ?? reconnectMain;
1601
+ const probeConnectorLocal = dependencies.probeConnectorLocal ?? probePrimaryConnectorLocal;
1602
+ const runCommand = dependencies.runCommand ?? command;
1603
+ const probeConnectorOwnership = dependencies.probeConnectorOwnership
1604
+ ?? ((candidateConfig: RecoveryConfig) => probePrimaryConnectorOwnership(candidateConfig, service, runCommand));
1299
1605
  const now = dependencies.now ?? Date.now;
1300
1606
  const wait = dependencies.sleep ?? sleep;
1301
1607
  const timeoutMs = config.primaryConnectorService?.postRestartVerifyTimeoutMs ?? 30_000;
1302
- const locked = await withLock(config, { action: 'restart_primary_connector' }, async () => {
1303
- const restarted = await ensureLaunchdServiceStarted(service, dependencies.runCommand ?? command);
1304
- if (!restarted.ok) {
1305
- audit(config, 'primary_connector_restart_failed', { serviceTarget: service.target, detail: restarted.detail });
1306
- return { ok: false, attempted: true, detail: restarted.detail, serviceTarget: service.target, verify: initialLocal } satisfies PrimaryConnectorRestartResult;
1608
+ const locked = await withLock(config, { action: 'restart_primary_connector', requestId: dependencies.requestId }, async () => {
1609
+ const tunnelConfigured = Boolean(configuredPrimaryPublicTunnel(config));
1610
+ const repairConnectorBinding = dependencies.repairConnectorBinding ?? repairPrimaryConnectorBinding;
1611
+ const bindingRepair = await repairConnectorBinding(config);
1612
+ if (bindingRepair.attempted) {
1613
+ audit(config, bindingRepair.ok ? 'primary_connector_binding_repaired' : 'primary_connector_binding_repair_failed', { detail: bindingRepair.detail });
1307
1614
  }
1308
- const deadline = now() + timeoutMs;
1309
- let observed = await reconnect(config);
1310
- while (!observed.ok && now() < deadline) {
1311
- await wait(1_000);
1312
- observed = await reconnect(config);
1615
+ if (!bindingRepair.ok) {
1616
+ return { ok: false, attempted: true, detail: `primary Connector immutable release binding repair failed: ${bindingRepair.detail}`, serviceTarget: service.target, verify: initialLocal } satisfies PrimaryConnectorRestartResult;
1617
+ }
1618
+ let localConnector = await probeConnectorLocal(config);
1619
+ let connectorOwnership = localConnector?.ok ? await probeConnectorOwnership(config) : undefined;
1620
+ let observed = localConnector?.ok && connectorOwnership?.ok ? await reconnect(config) : undefined;
1621
+ if (observed?.ok) {
1622
+ return {
1623
+ ok: true,
1624
+ attempted: bindingRepair.attempted,
1625
+ ...(bindingRepair.attempted ? {} : { noOp: true }),
1626
+ detail: bindingRepair.attempted
1627
+ ? 'primary Connector immutable release binding was repaired and the public MCP endpoint recovered'
1628
+ : 'primary Connector and public MCP endpoint recovered before restart',
1629
+ serviceTarget: service.target,
1630
+ verify: observed.verify,
1631
+ } satisfies PrimaryConnectorRestartResult;
1632
+ }
1633
+
1634
+ // A healthy local OAuth endpoint proves the Connector is not the broken hop.
1635
+ // If a distinct primary tunnel is configured, skip the pointless Gateway
1636
+ // restart and repair the public hop directly.
1637
+ const restartConnector = !localConnector?.ok || !connectorOwnership?.ok || !tunnelConfigured;
1638
+ if (restartConnector) {
1639
+ const restarted = await ensureLaunchdServiceStarted(service, runCommand);
1640
+ if (!restarted.ok) {
1641
+ audit(config, 'primary_connector_restart_failed', { serviceTarget: service.target, detail: restarted.detail });
1642
+ return { ok: false, attempted: true, detail: restarted.detail, serviceTarget: service.target, verify: initialLocal } satisfies PrimaryConnectorRestartResult;
1643
+ }
1644
+
1645
+ const localDeadline = now() + timeoutMs;
1646
+ localConnector = await probeConnectorLocal(config);
1647
+ while (localConnector && !localConnector.ok && now() < localDeadline) {
1648
+ await wait(1_000);
1649
+ localConnector = await probeConnectorLocal(config);
1650
+ }
1651
+ connectorOwnership = localConnector?.ok ? await probeConnectorOwnership(config) : undefined;
1652
+ if (localConnector && (!localConnector.ok || !connectorOwnership?.ok)) {
1653
+ const detail = localConnector.ok
1654
+ ? (connectorOwnership?.detail ?? 'configured primary Connector listener ownership could not be verified')
1655
+ : localConnector.detail;
1656
+ audit(config, 'primary_connector_restart_local_unverified', { serviceTarget: service.target, detail });
1657
+ return {
1658
+ ok: false,
1659
+ attempted: true,
1660
+ detail: localConnector.ok
1661
+ ? 'primary Connector restarted but the configured launchd service does not own its local MCP listener'
1662
+ : 'primary Connector restarted but its local MCP endpoint did not recover before timeout',
1663
+ serviceTarget: service.target,
1664
+ verify: initialLocal,
1665
+ } satisfies PrimaryConnectorRestartResult;
1666
+ }
1667
+ }
1668
+
1669
+ observed ??= await reconnect(config);
1670
+ if (!observed.ok && localConnector?.ok && connectorOwnership?.ok && configuredPrimaryPublicTunnel(config)) {
1671
+ const tunnel = primaryPublicTunnelService(config, service.uid);
1672
+ if (tunnel) {
1673
+ const tunnelRestarted = await ensureLaunchdServiceStarted(tunnel, dependencies.runCommand ?? command);
1674
+ if (!tunnelRestarted.ok) {
1675
+ audit(config, 'primary_public_tunnel_restart_failed', { serviceTarget: tunnel.target, detail: tunnelRestarted.detail });
1676
+ return {
1677
+ ok: false,
1678
+ attempted: true,
1679
+ detail: `primary Connector is locally healthy but public tunnel restart failed: ${tunnelRestarted.detail}`,
1680
+ serviceTarget: service.target,
1681
+ verify: observed.verify,
1682
+ } satisfies PrimaryConnectorRestartResult;
1683
+ }
1684
+ const tunnelDeadline = now() + (configuredPrimaryPublicTunnel(config)?.postRestartVerifyTimeoutMs ?? 20_000);
1685
+ while (!observed.ok && now() < tunnelDeadline) {
1686
+ await wait(1_000);
1687
+ observed = await reconnect(config);
1688
+ }
1689
+ audit(config, observed.ok ? 'primary_public_tunnel_restart_succeeded' : 'primary_public_tunnel_restart_unverified', {
1690
+ serviceTarget: tunnel.target,
1691
+ detail: observed.detail,
1692
+ });
1693
+ }
1694
+ } else {
1695
+ const deadline = now() + timeoutMs;
1696
+ while (!observed.ok && now() < deadline) {
1697
+ await wait(1_000);
1698
+ observed = await reconnect(config);
1699
+ }
1313
1700
  }
1701
+
1314
1702
  audit(config, observed.ok ? 'primary_connector_restart_succeeded' : 'primary_connector_restart_unverified', {
1315
1703
  serviceTarget: service.target,
1316
1704
  detail: observed.detail,
@@ -1318,7 +1706,9 @@ export async function restartPrimaryConnector(
1318
1706
  return {
1319
1707
  ok: observed.ok,
1320
1708
  attempted: true,
1321
- detail: observed.ok ? 'primary Connector restarted and the public MCP endpoint is reachable' : 'primary Connector restarted but the public MCP endpoint did not recover before timeout',
1709
+ detail: observed.ok
1710
+ ? 'primary Connector recovery completed and the public MCP endpoint is reachable'
1711
+ : 'primary Connector recovery completed but the public MCP endpoint did not recover before timeout',
1322
1712
  serviceTarget: service.target,
1323
1713
  verify: observed.verify,
1324
1714
  } satisfies PrimaryConnectorRestartResult;
@@ -1362,6 +1752,44 @@ async function waitForLaunchdServiceUnloaded(input: {
1362
1752
  return !printed.ok;
1363
1753
  }
1364
1754
 
1755
+ interface PrimaryRuntimePortObservation {
1756
+ port: number;
1757
+ pids: number[];
1758
+ uncertain?: string;
1759
+ }
1760
+
1761
+ interface PrimaryRuntimePortCleanupResult {
1762
+ released: boolean;
1763
+ cleaned: boolean;
1764
+ detail: string;
1765
+ pid?: number;
1766
+ signal?: 'SIGTERM' | 'SIGKILL';
1767
+ }
1768
+
1769
+ async function observePrimaryRuntimePort(config: RecoveryConfig, runCommand: CommandRunner): Promise<PrimaryRuntimePortObservation> {
1770
+ const paths = forgeRuntimeServicePaths(config.controllerHome);
1771
+ let port: number;
1772
+ try {
1773
+ port = readForgeRuntimeServiceConfig(paths.configPath).port;
1774
+ } catch {
1775
+ return { port: -1, pids: [], uncertain: 'primary Runtime service port could not be read' };
1776
+ }
1777
+ const result = await runCommand('lsof', ['-nP', '-Fp', `-iTCP:${port}`, '-sTCP:LISTEN'], 3_000);
1778
+ if (result.status === 1 && result.stderr.trim().length === 0) return { port, pids: [] };
1779
+ if (!result.ok) {
1780
+ return { port, pids: [], uncertain: `primary Runtime listener observation failed: ${result.stderr || result.stdout || result.status}` };
1781
+ }
1782
+ const pids = [...new Set(result.stdout.split(/\r?\n/)
1783
+ .map((line) => /^p(\d+)$/.exec(line.trim())?.[1])
1784
+ .filter((value): value is string => Boolean(value))
1785
+ .map(Number)
1786
+ .filter((value) => Number.isInteger(value) && value > 0))];
1787
+ if (result.stdout.trim().length > 0 && pids.length === 0) {
1788
+ return { port, pids: [], uncertain: 'primary Runtime listener identity was not machine-readable' };
1789
+ }
1790
+ return { port, pids };
1791
+ }
1792
+
1365
1793
  async function waitForPrimaryRuntimePortReleased(input: {
1366
1794
  config: RecoveryConfig;
1367
1795
  timeoutMs: number;
@@ -1369,25 +1797,173 @@ async function waitForPrimaryRuntimePortReleased(input: {
1369
1797
  wait: (ms: number) => Promise<void>;
1370
1798
  runCommand: CommandRunner;
1371
1799
  }): Promise<boolean> {
1372
- const paths = forgeRuntimeServicePaths(input.config.controllerHome);
1373
- let port: number;
1374
- try {
1375
- port = readForgeRuntimeServiceConfig(paths.configPath).port;
1376
- } catch {
1377
- return false;
1378
- }
1379
- const isListening = async (): Promise<boolean> => {
1380
- const result = await input.runCommand('lsof', ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN'], 3_000);
1381
- if (result.ok) return result.stdout.trim().length > 0;
1382
- if (result.status === 1 && result.stderr.trim().length === 0) return false;
1383
- return true;
1384
- };
1385
1800
  const deadline = input.now() + input.timeoutMs;
1386
1801
  while (input.now() < deadline) {
1387
- if (!(await isListening())) return true;
1802
+ const observation = await observePrimaryRuntimePort(input.config, input.runCommand);
1803
+ if (!observation.uncertain && observation.pids.length === 0) return true;
1388
1804
  await input.wait(250);
1389
1805
  }
1390
- return !(await isListening());
1806
+ const observation = await observePrimaryRuntimePort(input.config, input.runCommand);
1807
+ return !observation.uncertain && observation.pids.length === 0;
1808
+ }
1809
+
1810
+ function commandLineHasArgument(commandLine: string, flag: string, value: string): boolean {
1811
+ const candidates = [
1812
+ `${flag} ${value}`,
1813
+ `${flag} "${value}"`,
1814
+ `${flag} '${value}'`,
1815
+ ];
1816
+ return candidates.some((candidate) => {
1817
+ const index = commandLine.indexOf(candidate);
1818
+ if (index < 0) return false;
1819
+ const before = index === 0 ? '' : commandLine[index - 1];
1820
+ const afterIndex = index + candidate.length;
1821
+ const after = afterIndex >= commandLine.length ? '' : commandLine[afterIndex];
1822
+ return (!before || /\s/.test(before)) && (!after || /\s/.test(after));
1823
+ });
1824
+ }
1825
+
1826
+ async function verifyPrimaryRuntimeListenerIdentity(input: {
1827
+ config: RecoveryConfig;
1828
+ pid: number;
1829
+ uid: number;
1830
+ port: number;
1831
+ runCommand: CommandRunner;
1832
+ }): Promise<{ ok: boolean; detail: string }> {
1833
+ const paths = forgeRuntimeServicePaths(input.config.controllerHome);
1834
+ const physicalEntrypoint = activeRuntimeEntrypoint(input.config.controllerHome);
1835
+ const launchSpec = activeRuntimeLaunchSpec(input.config.controllerHome);
1836
+ if (!physicalEntrypoint || !launchSpec) return { ok: false, detail: 'active Runtime release identity is unavailable' };
1837
+ const manifestIndex = launchSpec.args.indexOf('--release-manifest');
1838
+ const manifestPath = manifestIndex >= 0 ? launchSpec.args[manifestIndex + 1] : undefined;
1839
+ if (!manifestPath) return { ok: false, detail: 'active Runtime release manifest identity is unavailable' };
1840
+
1841
+ const result = await input.runCommand('ps', ['-ww', '-p', String(input.pid), '-o', 'uid=', '-o', 'command='], 3_000);
1842
+ if (!result.ok) return { ok: false, detail: `listener process identity could not be read: ${result.stderr || result.stdout || result.status}` };
1843
+ const line = result.stdout.trim().split(/\r?\n/).find(Boolean) ?? '';
1844
+ const match = /^\s*(\d+)\s+(.+)$/.exec(line);
1845
+ if (!match) return { ok: false, detail: 'listener process identity was not machine-readable' };
1846
+ const observedUid = Number(match[1]);
1847
+ const commandLine = match[2]!.trim();
1848
+ if (observedUid !== input.uid) return { ok: false, detail: `listener uid ${observedUid} does not match Recovery uid ${input.uid}` };
1849
+ const executableMatches = [resolve(paths.activeEntrypointPath), resolve(physicalEntrypoint)]
1850
+ .some((candidate) => commandLine === candidate || commandLine.startsWith(`${candidate} `));
1851
+ if (!executableMatches) return { ok: false, detail: 'listener executable is not the active Forge Runtime entrypoint' };
1852
+ if (!commandLineHasArgument(commandLine, '--controller-home', resolve(input.config.controllerHome))) {
1853
+ return { ok: false, detail: 'listener controller-home identity does not match' };
1854
+ }
1855
+ if (!commandLineHasArgument(commandLine, '--port', String(input.port))) {
1856
+ return { ok: false, detail: 'listener port identity does not match' };
1857
+ }
1858
+ if (!commandLineHasArgument(commandLine, '--release-manifest', resolve(manifestPath))) {
1859
+ return { ok: false, detail: 'listener release-manifest identity does not match' };
1860
+ }
1861
+ return { ok: true, detail: 'listener matches the current Forge Runtime release and controller identity' };
1862
+ }
1863
+
1864
+ async function cleanupVerifiedStalePrimaryRuntimeListener(input: {
1865
+ config: RecoveryConfig;
1866
+ uid: number;
1867
+ now: () => number;
1868
+ wait: (ms: number) => Promise<void>;
1869
+ runCommand: CommandRunner;
1870
+ }): Promise<PrimaryRuntimePortCleanupResult> {
1871
+ const initial = await observePrimaryRuntimePort(input.config, input.runCommand);
1872
+ if (initial.uncertain) return { released: false, cleaned: false, detail: initial.uncertain };
1873
+ if (initial.pids.length === 0) return { released: true, cleaned: false, detail: 'primary Runtime port is already released' };
1874
+ if (initial.pids.length !== 1) return { released: false, cleaned: false, detail: `primary Runtime port has ${initial.pids.length} listeners; cleanup requires exactly one` };
1875
+ const pid = initial.pids[0]!;
1876
+ const identity = await verifyPrimaryRuntimeListenerIdentity({ config: input.config, pid, uid: input.uid, port: initial.port, runCommand: input.runCommand });
1877
+ if (!identity.ok) return { released: false, cleaned: false, detail: identity.detail, pid };
1878
+
1879
+ const term = await input.runCommand('kill', ['-TERM', String(pid)], 3_000);
1880
+ const releasedAfterTerm = await waitForPrimaryRuntimePortReleased({ ...input, timeoutMs: 5_000 });
1881
+ if (releasedAfterTerm) return { released: true, cleaned: true, detail: 'verified stale Forge Runtime listener released after SIGTERM', pid, signal: 'SIGTERM' };
1882
+ if (!term.ok) return { released: false, cleaned: false, detail: `verified listener SIGTERM failed: ${term.stderr || term.stdout || term.status}`, pid };
1883
+
1884
+ const beforeKill = await observePrimaryRuntimePort(input.config, input.runCommand);
1885
+ if (beforeKill.uncertain || beforeKill.pids.length !== 1 || beforeKill.pids[0] !== pid) {
1886
+ return { released: false, cleaned: false, detail: 'listener identity changed after SIGTERM; refusing SIGKILL', pid };
1887
+ }
1888
+ const identityBeforeKill = await verifyPrimaryRuntimeListenerIdentity({ config: input.config, pid, uid: input.uid, port: beforeKill.port, runCommand: input.runCommand });
1889
+ if (!identityBeforeKill.ok) return { released: false, cleaned: false, detail: `listener identity changed after SIGTERM: ${identityBeforeKill.detail}`, pid };
1890
+ const killed = await input.runCommand('kill', ['-KILL', String(pid)], 3_000);
1891
+ if (!killed.ok) return { released: false, cleaned: false, detail: `verified listener SIGKILL failed: ${killed.stderr || killed.stdout || killed.status}`, pid };
1892
+ const releasedAfterKill = await waitForPrimaryRuntimePortReleased({ ...input, timeoutMs: 5_000 });
1893
+ return releasedAfterKill
1894
+ ? { released: true, cleaned: true, detail: 'verified stale Forge Runtime listener released after SIGKILL', pid, signal: 'SIGKILL' }
1895
+ : { released: false, cleaned: false, detail: 'verified stale Forge Runtime listener remained on the port after SIGKILL', pid };
1896
+ }
1897
+
1898
+ interface PrimaryRuntimeStoppedTransition {
1899
+ ok: boolean;
1900
+ detail: string;
1901
+ staleListenerCleanup?: PrimaryRuntimePortCleanupResult;
1902
+ }
1903
+
1904
+ /**
1905
+ * Stop the one canonical Runtime service and prove that every execution owner
1906
+ * boundary is gone before release authority may change. This is the shared
1907
+ * transition primitive for activation and recovery; callers must not publish
1908
+ * or roll back a release after a partial stop.
1909
+ */
1910
+ async function stopPrimaryRuntimeForReleaseTransition(input: {
1911
+ config: RecoveryConfig;
1912
+ service: LaunchdService;
1913
+ now: () => number;
1914
+ wait: (ms: number) => Promise<void>;
1915
+ runCommand: CommandRunner;
1916
+ runtimeRunning: (config: RecoveryConfig) => boolean;
1917
+ }): Promise<PrimaryRuntimeStoppedTransition> {
1918
+ const stopped = await input.runCommand('launchctl', ['bootout', input.service.target], 15_000);
1919
+ const stoppedCleanly = stopped.ok || /not found|no such process|could not find service|service is not loaded/i.test(`${stopped.stderr}\n${stopped.stdout}`);
1920
+ if (!stoppedCleanly) {
1921
+ return { ok: false, detail: `primary Runtime bootout failed: ${stopped.stderr || stopped.stdout || stopped.status}` };
1922
+ }
1923
+ const runtimeStopped = await waitForPrimaryRuntimeState({
1924
+ config: input.config,
1925
+ expectedRunning: false,
1926
+ timeoutMs: 20_000,
1927
+ now: input.now,
1928
+ wait: input.wait,
1929
+ runtimeRunning: input.runtimeRunning,
1930
+ });
1931
+ if (!runtimeStopped) return { ok: false, detail: 'primary Runtime owner remained live after bounded launchd bootout' };
1932
+ const serviceUnloaded = await waitForLaunchdServiceUnloaded({
1933
+ service: input.service,
1934
+ timeoutMs: 20_000,
1935
+ now: input.now,
1936
+ wait: input.wait,
1937
+ runCommand: input.runCommand,
1938
+ });
1939
+ if (!serviceUnloaded) return { ok: false, detail: 'primary Runtime launchd service remained loaded after bounded bootout' };
1940
+
1941
+ let portReleased = await waitForPrimaryRuntimePortReleased({
1942
+ config: input.config,
1943
+ timeoutMs: 20_000,
1944
+ now: input.now,
1945
+ wait: input.wait,
1946
+ runCommand: input.runCommand,
1947
+ });
1948
+ let staleListenerCleanup: PrimaryRuntimePortCleanupResult | undefined;
1949
+ if (!portReleased) {
1950
+ staleListenerCleanup = await cleanupVerifiedStalePrimaryRuntimeListener({
1951
+ config: input.config,
1952
+ uid: input.service.uid,
1953
+ now: input.now,
1954
+ wait: input.wait,
1955
+ runCommand: input.runCommand,
1956
+ });
1957
+ portReleased = staleListenerCleanup.released;
1958
+ }
1959
+ if (!portReleased) {
1960
+ return {
1961
+ ok: false,
1962
+ detail: `primary Runtime TCP port remained occupied after bounded launchd bootout: ${staleListenerCleanup?.detail ?? 'listener did not release'}`,
1963
+ staleListenerCleanup,
1964
+ };
1965
+ }
1966
+ return { ok: true, detail: 'primary Runtime service, owner, and TCP listener are stopped', staleListenerCleanup };
1391
1967
  }
1392
1968
 
1393
1969
  async function verifyPrimaryRuntimeAfterStart(input: {
@@ -1406,6 +1982,73 @@ async function verifyPrimaryRuntimeAfterStart(input: {
1406
1982
  return observed;
1407
1983
  }
1408
1984
 
1985
+ interface PrimaryRuntimeRebindStartResult {
1986
+ ok: boolean;
1987
+ detail: string;
1988
+ verify: VerifyResult;
1989
+ }
1990
+
1991
+ /**
1992
+ * Complete the only safe restart sequence after release authority changes:
1993
+ * rebind the release-pinned launchd contract, start the sole Runtime service,
1994
+ * and require whole-Runtime verification. A started-but-unverified Runtime is
1995
+ * stopped again so callers never leave a restart loop behind.
1996
+ */
1997
+ async function rebindStartAndVerifyPrimaryRuntime(input: {
1998
+ config: RecoveryConfig;
1999
+ service: LaunchdService;
2000
+ runCommand: CommandRunner;
2001
+ now: () => number;
2002
+ wait: (ms: number) => Promise<void>;
2003
+ verifyLocal: (config: RecoveryConfig) => Promise<VerifyResult>;
2004
+ ensureRuntimeLaunchContract?: (controllerHome: string) => void;
2005
+ beforeStart?: () => void;
2006
+ contractFailureContext?: string;
2007
+ timeoutMs: number;
2008
+ successDetail: string;
2009
+ }): Promise<PrimaryRuntimeRebindStartResult> {
2010
+ try {
2011
+ const ensureRuntimeLaunchContract = input.ensureRuntimeLaunchContract
2012
+ ?? ((controllerHome: string) => { ensureForgeRuntimeLaunchAgentContract({ controllerHome, installUserLaunchAgent: true }); });
2013
+ ensureRuntimeLaunchContract(input.config.controllerHome);
2014
+ } catch (error) {
2015
+ return {
2016
+ ok: false,
2017
+ detail: `primary Runtime launchd contract rebuild failed ${input.contractFailureContext ?? 'after release transition'}: ${error instanceof Error ? error.message : String(error)}`,
2018
+ verify: await input.verifyLocal(input.config),
2019
+ };
2020
+ }
2021
+
2022
+ try {
2023
+ input.beforeStart?.();
2024
+ } catch (error) {
2025
+ return {
2026
+ ok: false,
2027
+ detail: `release transition pre-start preparation failed: ${error instanceof Error ? error.message : String(error)}`,
2028
+ verify: await input.verifyLocal(input.config),
2029
+ };
2030
+ }
2031
+
2032
+ const started = await ensureLaunchdServiceStarted(input.service, input.runCommand);
2033
+ if (!started.ok) {
2034
+ return { ok: false, detail: started.detail, verify: await input.verifyLocal(input.config) };
2035
+ }
2036
+ const verify = await verifyPrimaryRuntimeAfterStart({
2037
+ config: input.config,
2038
+ timeoutMs: input.timeoutMs,
2039
+ now: input.now,
2040
+ wait: input.wait,
2041
+ verifyLocal: input.verifyLocal,
2042
+ });
2043
+ if (verify.ok) return { ok: true, detail: input.successDetail, verify };
2044
+ await input.runCommand('launchctl', ['bootout', input.service.target], 15_000);
2045
+ return {
2046
+ ok: false,
2047
+ detail: 'release transition started a Runtime that failed whole-Runtime verification; service was stopped to prevent a restart loop',
2048
+ verify,
2049
+ };
2050
+ }
2051
+
1409
2052
  export async function restartPrimaryRuntime(
1410
2053
  config: RecoveryConfig,
1411
2054
  dependencies: PrimaryRuntimeRecoveryDependencies = {},
@@ -1469,31 +2112,21 @@ export async function recoverPrimaryRuntime(
1469
2112
  const before = await verifyLocal(config);
1470
2113
  if (before.ok) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime recovered before rollback', serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRecoveryResult;
1471
2114
 
1472
- const stopped = await runCommand('launchctl', ['bootout', service.target], 15_000);
1473
- const stoppedCleanly = stopped.ok || /not found|no such process|could not find service|service is not loaded/i.test(`${stopped.stderr}\n${stopped.stdout}`);
1474
- if (!stoppedCleanly) {
1475
- const detail = `primary Runtime bootout failed: ${stopped.stderr || stopped.stdout || stopped.status}`;
1476
- audit(config, 'primary_runtime_recovery_stop_failed', { serviceTarget: service.target, detail });
1477
- return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRecoveryResult;
2115
+ const stopped = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
2116
+ if (!stopped.ok) {
2117
+ const action = stopped.detail.startsWith('primary Runtime bootout failed')
2118
+ ? 'primary_runtime_recovery_stop_failed'
2119
+ : 'primary_runtime_recovery_stop_unverified';
2120
+ audit(config, action, { serviceTarget: service.target, detail: stopped.detail, pid: stopped.staleListenerCleanup?.pid });
2121
+ return { ok: false, attempted: true, detail: stopped.detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
1478
2122
  }
1479
- const runtimeStopped = await waitForPrimaryRuntimeState({ config, expectedRunning: false, timeoutMs: 20_000, now, wait, runtimeRunning });
1480
- if (!runtimeStopped) {
1481
- const detail = 'primary Runtime owner remained live after bounded launchd bootout';
1482
- audit(config, 'primary_runtime_recovery_stop_unverified', { serviceTarget: service.target });
1483
- return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
1484
- }
1485
- const serviceUnloaded = await waitForLaunchdServiceUnloaded({ service, timeoutMs: 20_000, now, wait, runCommand });
1486
- if (!serviceUnloaded) {
1487
- const detail = 'primary Runtime launchd service remained loaded after bounded bootout';
1488
- audit(config, 'primary_runtime_recovery_stop_unverified', { serviceTarget: service.target, detail });
1489
- return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
1490
- }
1491
-
1492
- const portReleased = await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand });
1493
- if (!portReleased) {
1494
- const detail = 'primary Runtime TCP port remained occupied after bounded launchd bootout';
1495
- audit(config, 'primary_runtime_recovery_stop_unverified', { serviceTarget: service.target, detail });
1496
- return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
2123
+ if (stopped.staleListenerCleanup?.cleaned) {
2124
+ audit(config, 'primary_runtime_recovery_stale_listener_cleaned', {
2125
+ serviceTarget: service.target,
2126
+ pid: stopped.staleListenerCleanup.pid,
2127
+ signal: stopped.staleListenerCleanup.signal,
2128
+ detail: stopped.staleListenerCleanup.detail,
2129
+ });
1497
2130
  }
1498
2131
 
1499
2132
  const rollback = await rollbackPreviousLocked(config, reason);
@@ -1502,26 +2135,30 @@ export async function recoverPrimaryRuntime(
1502
2135
  return { ok: false, attempted: true, detail: rollback.detail, serviceTarget: service.target, rollback, verify: rollback.verify ?? await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
1503
2136
  }
1504
2137
 
1505
- const started = await ensureLaunchdServiceStarted(service, runCommand);
1506
- if (!started.ok) {
1507
- audit(config, 'primary_runtime_recovery_start_failed', { serviceTarget: service.target, detail: started.detail, rollbackOperationId: rollback.operationId });
1508
- return { ok: false, attempted: true, detail: started.detail, serviceTarget: service.target, rollback, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
1509
- }
1510
- const after = await verifyPrimaryRuntimeAfterStart({
2138
+ const restarted = await rebindStartAndVerifyPrimaryRuntime({
1511
2139
  config,
1512
- timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 45_000,
2140
+ service,
2141
+ runCommand,
1513
2142
  now,
1514
2143
  wait,
1515
2144
  verifyLocal,
2145
+ ensureRuntimeLaunchContract: dependencies.ensureRuntimeLaunchContract,
2146
+ contractFailureContext: 'after rollback',
2147
+ timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 45_000,
2148
+ successDetail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified',
1516
2149
  });
1517
- if (after.ok) {
1518
- audit(config, 'primary_runtime_recovery_succeeded', { serviceTarget: service.target, rollbackOperationId: rollback.operationId, restoredRelease: after.releases.active?.revision });
1519
- return { ok: true, attempted: true, detail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified', serviceTarget: service.target, rollback, verify: after } satisfies PrimaryRuntimeRecoveryResult;
2150
+ if (restarted.ok) {
2151
+ audit(config, 'primary_runtime_recovery_succeeded', { serviceTarget: service.target, rollbackOperationId: rollback.operationId, restoredRelease: restarted.verify.releases.active?.revision });
2152
+ return { ok: true, attempted: true, detail: restarted.detail, serviceTarget: service.target, rollback, verify: restarted.verify } satisfies PrimaryRuntimeRecoveryResult;
1520
2153
  }
1521
-
1522
- await runCommand('launchctl', ['bootout', service.target], 15_000);
1523
- audit(config, 'primary_runtime_recovery_unverified', { serviceTarget: service.target, rollbackOperationId: rollback.operationId, reasonCodes: after.runtime.reasonCodes });
1524
- return { ok: false, attempted: true, detail: 'previous whole-Runtime release started but failed verification; service was stopped to prevent a restart loop', serviceTarget: service.target, rollback, verify: after } satisfies PrimaryRuntimeRecoveryResult;
2154
+ const contractFailure = restarted.detail.startsWith('primary Runtime launchd contract rebuild failed');
2155
+ audit(config, contractFailure ? 'primary_runtime_recovery_launchd_contract_failed' : 'primary_runtime_recovery_unverified', {
2156
+ serviceTarget: service.target,
2157
+ rollbackOperationId: rollback.operationId,
2158
+ detail: restarted.detail,
2159
+ reasonCodes: restarted.verify.runtime.reasonCodes,
2160
+ });
2161
+ return { ok: false, attempted: true, detail: restarted.detail, serviceTarget: service.target, rollback, verify: restarted.verify } satisfies PrimaryRuntimeRecoveryResult;
1525
2162
  });
1526
2163
  if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), serviceTarget: service.target, verify: initial };
1527
2164
  return locked.value;
@@ -1668,7 +2305,25 @@ export async function activateRuntimeRelease(
1668
2305
  } satisfies RuntimeReleaseActivationResult;
1669
2306
  }
1670
2307
  if (current) {
1671
- if (runtimeBehaviorEquivalent(current.active.manifestPath, candidate.manifestPath) && !storageMigrationRequired) {
2308
+ const behaviorEquivalent = runtimeBehaviorEquivalent(current.active.manifestPath, candidate.manifestPath);
2309
+ let serviceContractMatches = false;
2310
+ if (behaviorEquivalent && !storageMigrationRequired) {
2311
+ try {
2312
+ serviceContractMatches = inspectForgeRuntimeLaunchAgentContract({
2313
+ controllerHome: config.controllerHome,
2314
+ inspectUserLaunchAgent: true,
2315
+ }).matches;
2316
+ } catch (error) {
2317
+ audit(config, 'runtime_release_activation_service_contract_inspection_failed', {
2318
+ operationId,
2319
+ requestId: lockRequestId,
2320
+ activeRevision: current.active.releaseId,
2321
+ candidateRevision: candidate.manifest.releaseId,
2322
+ detail: error instanceof Error ? error.message : String(error),
2323
+ });
2324
+ }
2325
+ }
2326
+ if (behaviorEquivalent && !storageMigrationRequired && serviceContractMatches) {
1672
2327
  const verify = await verifyLocal(config);
1673
2328
  audit(config, 'runtime_release_activation_behavior_identical', {
1674
2329
  operationId,
@@ -1682,8 +2337,8 @@ export async function activateRuntimeRelease(
1682
2337
  attempted: false,
1683
2338
  noOp: true,
1684
2339
  detail: verify.ok
1685
- ? 'candidate Runtime behavior is identical to the active release; restart skipped'
1686
- : 'candidate Runtime behavior is identical to the active release, but the active Runtime is unhealthy',
2340
+ ? 'candidate Runtime behavior and launchd service contract are identical to the active release; restart skipped'
2341
+ : 'candidate Runtime behavior and launchd service contract are identical to the active release, but the active Runtime is unhealthy',
1687
2342
  operationId,
1688
2343
  verify,
1689
2344
  } satisfies RuntimeReleaseActivationResult;
@@ -1701,30 +2356,22 @@ export async function activateRuntimeRelease(
1701
2356
  return { ok: false, attempted: false, noOp: true, detail, operationId } satisfies RuntimeReleaseActivationResult;
1702
2357
  }
1703
2358
  const before = await verifyLocal(config);
1704
- const stopped = await runCommand('launchctl', ['bootout', service.target], 15_000);
1705
- const stoppedCleanly = stopped.ok || /not found|no such process|could not find service|service is not loaded/i.test(`${stopped.stderr}\n${stopped.stdout}`);
1706
- if (!stoppedCleanly) {
1707
- const detail = `primary Runtime bootout failed: ${stopped.stderr || stopped.stdout || stopped.status}`;
1708
- audit(config, 'runtime_release_activation_stop_failed', { serviceTarget: service.target, operationId, detail });
1709
- return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
1710
- }
1711
- const runtimeStopped = await waitForPrimaryRuntimeState({ config, expectedRunning: false, timeoutMs: 20_000, now, wait, runtimeRunning });
1712
- if (!runtimeStopped) {
1713
- const detail = 'primary Runtime owner remained live after bounded launchd bootout';
1714
- audit(config, 'runtime_release_activation_stop_unverified', { serviceTarget: service.target, operationId });
1715
- return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
2359
+ const stopped = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
2360
+ if (!stopped.ok) {
2361
+ const action = stopped.detail.startsWith('primary Runtime bootout failed')
2362
+ ? 'runtime_release_activation_stop_failed'
2363
+ : 'runtime_release_activation_stop_unverified';
2364
+ audit(config, action, { serviceTarget: service.target, operationId, detail: stopped.detail, pid: stopped.staleListenerCleanup?.pid });
2365
+ return { ok: false, attempted: true, detail: stopped.detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
1716
2366
  }
1717
- const serviceUnloaded = await waitForLaunchdServiceUnloaded({ service, timeoutMs: 20_000, now, wait, runCommand });
1718
- if (!serviceUnloaded) {
1719
- const detail = 'primary Runtime launchd service remained loaded after bounded bootout';
1720
- audit(config, 'runtime_release_activation_stop_unverified', { serviceTarget: service.target, operationId, detail });
1721
- return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
1722
- }
1723
- const portReleased = await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand });
1724
- if (!portReleased) {
1725
- const detail = 'primary Runtime TCP port remained occupied after bounded launchd bootout';
1726
- audit(config, 'runtime_release_activation_stop_unverified', { serviceTarget: service.target, operationId, detail });
1727
- return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
2367
+ if (stopped.staleListenerCleanup?.cleaned) {
2368
+ audit(config, 'runtime_release_activation_stale_listener_cleaned', {
2369
+ serviceTarget: service.target,
2370
+ operationId,
2371
+ pid: stopped.staleListenerCleanup.pid,
2372
+ signal: stopped.staleListenerCleanup.signal,
2373
+ detail: stopped.staleListenerCleanup.detail,
2374
+ });
1728
2375
  }
1729
2376
  let committed: RuntimeReleaseAuthority;
1730
2377
  try {
@@ -1739,73 +2386,59 @@ export async function activateRuntimeRelease(
1739
2386
  audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId });
1740
2387
  return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
1741
2388
  }
1742
- try {
1743
- ensureForgeRuntimeLaunchAgentContract({ controllerHome: config.controllerHome, installUserLaunchAgent: true });
1744
- } catch (error) {
1745
- const detail = `runtime launchd stable-release contract update failed: ${error instanceof Error ? error.message : String(error)}`;
1746
- audit(config, 'runtime_release_activation_launchd_contract_failed', { serviceTarget: service.target, operationId, detail });
1747
- return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
1748
- }
1749
2389
  let storageMigration: ControllerHomeStorageMigration | undefined;
1750
- let after: VerifyResult;
2390
+ const activated = await rebindStartAndVerifyPrimaryRuntime({
2391
+ config,
2392
+ service,
2393
+ runCommand,
2394
+ now,
2395
+ wait,
2396
+ verifyLocal,
2397
+ timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
2398
+ successDetail: 'requested Runtime release started and passed whole-Runtime verification',
2399
+ beforeStart: () => {
2400
+ storageMigration = migrateStoppedRepoLocalControllerHomeStorage(config.controllerHome, platform);
2401
+ if (storageMigration.migrated) {
2402
+ audit(config, 'runtime_controller_home_noindex_migrated', {
2403
+ serviceTarget: service.target,
2404
+ operationId,
2405
+ logicalHome: storageMigration.logicalHome,
2406
+ physicalHome: storageMigration.physicalHome,
2407
+ });
2408
+ }
2409
+ },
2410
+ });
2411
+ let after = activated.verify;
1751
2412
  let activationFailureDetail: string | undefined;
1752
- try {
1753
- storageMigration = migrateStoppedRepoLocalControllerHomeStorage(config.controllerHome, platform);
1754
- if (storageMigration.migrated) {
1755
- audit(config, 'runtime_controller_home_noindex_migrated', {
1756
- serviceTarget: service.target,
1757
- operationId,
1758
- logicalHome: storageMigration.logicalHome,
1759
- physicalHome: storageMigration.physicalHome,
1760
- });
1761
- }
1762
- } catch (error) {
1763
- activationFailureDetail = `Controller Home .noindex migration failed: ${error instanceof Error ? error.message : String(error)}`;
1764
- audit(config, 'runtime_controller_home_noindex_migration_failed', { serviceTarget: service.target, operationId, detail: activationFailureDetail });
2413
+ if (activated.ok && after.releases.active?.revision === candidate.manifest.releaseId) {
2414
+ audit(config, 'runtime_release_activation_succeeded', { serviceTarget: service.target, operationId, requestId: lockRequestId, activeRevision: after.releases.active?.revision, expectedAuthorityRevision: guard.expectedAuthorityRevision, expectedActiveReleaseId: guard.expectedActiveReleaseId, controllerHomeStorageMigrated: storageMigration?.migrated === true });
2415
+ return { ok: true, attempted: true, detail: storageMigration?.migrated ? 'requested Runtime release activated, Controller Home migrated to .noindex storage, and whole-Runtime verification passed' : 'requested Runtime release activated and passed whole-Runtime verification', serviceTarget: service.target, operationId, verify: after } satisfies RuntimeReleaseActivationResult;
1765
2416
  }
1766
-
1767
- if (activationFailureDetail) {
1768
- after = await verifyLocal(config);
2417
+ if (!activated.ok) {
2418
+ activationFailureDetail = activated.detail;
2419
+ const auditAction = activated.detail.startsWith('primary Runtime launchd contract rebuild failed')
2420
+ ? 'runtime_release_activation_launchd_contract_failed'
2421
+ : activated.detail.startsWith('release transition pre-start preparation failed')
2422
+ ? 'runtime_controller_home_noindex_migration_failed'
2423
+ : activated.detail.includes('start')
2424
+ ? 'runtime_release_activation_start_failed'
2425
+ : 'runtime_release_activation_unverified';
2426
+ audit(config, auditAction, { serviceTarget: service.target, operationId, detail: activated.detail });
1769
2427
  } else {
1770
- const started = await ensureLaunchdServiceStarted(service, runCommand);
1771
- if (!started.ok) {
1772
- activationFailureDetail = started.detail;
1773
- audit(config, 'runtime_release_activation_start_failed', { serviceTarget: service.target, operationId, detail: started.detail });
1774
- after = await verifyLocal(config);
1775
- } else {
1776
- after = await verifyPrimaryRuntimeAfterStart({
1777
- config,
1778
- timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
1779
- now,
1780
- wait,
1781
- verifyLocal,
1782
- });
1783
- if (after.ok && after.releases.active?.revision === candidate.manifest.releaseId) {
1784
- audit(config, 'runtime_release_activation_succeeded', { serviceTarget: service.target, operationId, requestId: lockRequestId, activeRevision: after.releases.active?.revision, expectedAuthorityRevision: guard.expectedAuthorityRevision, expectedActiveReleaseId: guard.expectedActiveReleaseId, controllerHomeStorageMigrated: storageMigration?.migrated === true });
1785
- return { ok: true, attempted: true, detail: storageMigration?.migrated ? 'requested Runtime release activated, Controller Home migrated to .noindex storage, and whole-Runtime verification passed' : 'requested Runtime release activated and passed whole-Runtime verification', serviceTarget: service.target, operationId, verify: after } satisfies RuntimeReleaseActivationResult;
1786
- }
1787
- }
2428
+ activationFailureDetail = `activated Runtime release identity mismatch: expected ${candidate.manifest.releaseId}, observed ${after.releases.active?.revision ?? 'none'}`;
2429
+ audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId, detail: activationFailureDetail });
1788
2430
  }
1789
2431
 
1790
2432
  // Activation failed: stop, restore the previous whole release and its SQLite
1791
2433
  // backup, restart the one service, and require verification again.
1792
- await runCommand('launchctl', ['bootout', service.target], 15_000);
1793
2434
  let rollback: RollbackResult;
1794
- const rollbackRuntimeStopped = await waitForPrimaryRuntimeState({ config, expectedRunning: false, timeoutMs: 20_000, now, wait, runtimeRunning });
1795
- const rollbackServiceUnloaded = rollbackRuntimeStopped
1796
- ? await waitForLaunchdServiceUnloaded({ service, timeoutMs: 20_000, now, wait, runCommand })
1797
- : false;
1798
- const rollbackPortReleased = rollbackRuntimeStopped && rollbackServiceUnloaded
1799
- ? await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand })
1800
- : false;
1801
- if (!rollbackRuntimeStopped || !rollbackServiceUnloaded || !rollbackPortReleased) {
2435
+ const rollbackStop = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
2436
+ if (!rollbackStop.ok) {
1802
2437
  rollback = {
1803
2438
  ok: false,
1804
- detail: !rollbackRuntimeStopped
1805
- ? 'candidate Runtime owner remained live after bounded rollback bootout'
1806
- : !rollbackServiceUnloaded
1807
- ? 'candidate Runtime launchd service remained loaded after bounded rollback bootout'
1808
- : 'candidate Runtime TCP port remained occupied after bounded rollback bootout',
2439
+ detail: rollbackStop.detail
2440
+ .replace('primary Runtime', 'candidate Runtime')
2441
+ .replace('bounded launchd bootout', 'bounded rollback bootout'),
1809
2442
  };
1810
2443
  } else {
1811
2444
  try {
@@ -1821,7 +2454,6 @@ export async function activateRuntimeRelease(
1821
2454
  physicalHome: storageMigration.physicalHome,
1822
2455
  });
1823
2456
  }
1824
- ensureForgeRuntimeLaunchAgentContract({ controllerHome: config.controllerHome, installUserLaunchAgent: true });
1825
2457
  const previousRevision = previousActive?.releaseId ?? '';
1826
2458
  const previousIdentity = previousActive?.artifactIdentity;
1827
2459
  if (
@@ -1830,24 +2462,18 @@ export async function activateRuntimeRelease(
1830
2462
  ) {
1831
2463
  throw new Error('RECOVERY_RUNTIME_RELEASE_ROLLBACK_AUTHORITY_MISMATCH');
1832
2464
  }
1833
- const restarted = await ensureLaunchdServiceStarted(service, runCommand);
1834
- if (!restarted.ok) {
1835
- rollback = { ok: false, operationId: rollbackOperationId, detail: `previous release authority restored but service start failed: ${restarted.detail}` };
1836
- } else {
1837
- const rolledBack = await verifyPrimaryRuntimeAfterStart({
1838
- config,
1839
- timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
1840
- now,
1841
- wait,
1842
- verifyLocal,
1843
- });
1844
- if (rolledBack.ok) {
1845
- rollback = { ok: true, operationId: rollbackOperationId, detail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified', verify: rolledBack };
1846
- } else {
1847
- await runCommand('launchctl', ['bootout', service.target], 15_000);
1848
- rollback = { ok: false, operationId: rollbackOperationId, detail: 'previous whole-Runtime release started but failed verification; service was stopped to prevent a restart loop', verify: rolledBack };
1849
- }
1850
- }
2465
+ const restarted = await rebindStartAndVerifyPrimaryRuntime({
2466
+ config,
2467
+ service,
2468
+ runCommand,
2469
+ now,
2470
+ wait,
2471
+ verifyLocal,
2472
+ contractFailureContext: 'after rollback',
2473
+ timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
2474
+ successDetail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified',
2475
+ });
2476
+ rollback = { ok: restarted.ok, operationId: rollbackOperationId, detail: restarted.detail, verify: restarted.verify };
1851
2477
  } catch (error) {
1852
2478
  rollback = { ok: false, detail: error instanceof Error ? error.message : 'previous whole-Runtime release rollback failed' };
1853
2479
  }
@@ -1896,15 +2522,26 @@ export async function stageAndActivateConfiguredRuntimeRelease(
1896
2522
  // Capture the base before staging. Staging can take long enough for another
1897
2523
  // activation to win; the later publish must prove this base is still current.
1898
2524
  const expectedAuthority = releaseAuthority(config);
1899
- let staged: StagedRuntimeRelease;
1900
- try {
1901
- staged = (dependencies.stage ?? stageRuntimeReleaseFromCandidateSource)({ controllerHome: config.controllerHome, sourceRoot });
1902
- assertRuntimeReleaseFiles(staged);
1903
- } catch (error) {
1904
- const detail = error instanceof Error ? error.message : 'Runtime release staging failed';
1905
- audit(config, 'runtime_release_stage_failed', { sourceRoot, detail });
1906
- return { ok: false, attempted: false, noOp: true, detail };
2525
+ const stagedAttempt = await withLock(config, {
2526
+ action: 'stage_and_activate_runtime_release',
2527
+ ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
2528
+ }, async () => {
2529
+ try {
2530
+ const staged = (dependencies.stage ?? stageRuntimeReleaseFromCandidateSource)({ controllerHome: config.controllerHome, sourceRoot });
2531
+ assertRuntimeReleaseFiles(staged);
2532
+ return { ok: true as const, staged };
2533
+ } catch (error) {
2534
+ return { ok: false as const, detail: error instanceof Error ? error.message : 'Runtime release staging failed' };
2535
+ }
2536
+ });
2537
+ if (!stagedAttempt.acquired) {
2538
+ return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(stagedAttempt.owner) };
1907
2539
  }
2540
+ if (!stagedAttempt.value.ok) {
2541
+ audit(config, 'runtime_release_stage_failed', { sourceRoot, detail: stagedAttempt.value.detail });
2542
+ return { ok: false, attempted: false, noOp: true, detail: stagedAttempt.value.detail };
2543
+ }
2544
+ const staged: StagedRuntimeRelease = stagedAttempt.value.staged;
1908
2545
  const activationGuard: RuntimeReleaseActivationGuard = {
1909
2546
  ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
1910
2547
  expectedAuthorityRevision: expectedAuthority?.revision ?? null,
@@ -1933,8 +2570,7 @@ export async function stageAndActivateConfiguredRuntimeRelease(
1933
2570
  };
1934
2571
  }
1935
2572
 
1936
- function recoveryTunnelService(config: RecoveryConfig, uid: number): LaunchdService | undefined {
1937
- const configured = configuredRecoveryTunnel(config);
2573
+ function tunnelLaunchdService(configured: PublicTunnelServiceConfig | undefined, uid: number): LaunchdService | undefined {
1938
2574
  if (!configured || configured.platform !== 'launchd') return undefined;
1939
2575
  if (!/^com\.[A-Za-z0-9._-]{1,180}$/.test(configured.label)) return undefined;
1940
2576
  const plistPath = configured.plistPath;
@@ -1948,6 +2584,14 @@ function recoveryTunnelService(config: RecoveryConfig, uid: number): LaunchdServ
1948
2584
  };
1949
2585
  }
1950
2586
 
2587
+ function recoveryTunnelService(config: RecoveryConfig, uid: number): LaunchdService | undefined {
2588
+ return tunnelLaunchdService(configuredRecoveryTunnel(config), uid);
2589
+ }
2590
+
2591
+ function primaryPublicTunnelService(config: RecoveryConfig, uid: number): LaunchdService | undefined {
2592
+ return tunnelLaunchdService(configuredPrimaryPublicTunnel(config), uid);
2593
+ }
2594
+
1951
2595
  function tunnelRepairAllowed(config: RecoveryConfig, now: number): boolean {
1952
2596
  const cooldownMs = configuredRecoveryTunnel(config)?.cooldownMs ?? 60_000;
1953
2597
  const prior = json<{ lastAttemptAt?: unknown }>(publicTunnelRepairStatePath(config));
@@ -2015,17 +2659,32 @@ export interface RecoveryGatewayRestartResult {
2015
2659
  serviceTarget?: string;
2016
2660
  }
2017
2661
 
2018
- export interface RecoveryGatewayRestartDependencies {
2662
+ export interface RecoveryWatchdogRestartResult {
2663
+ ok: boolean;
2664
+ attempted: boolean;
2665
+ noOp?: boolean;
2666
+ detail: string;
2667
+ serviceTarget?: string;
2668
+ }
2669
+
2670
+ interface RecoveryRoleRestartDependencies {
2019
2671
  platform?: NodeJS.Platform;
2020
2672
  currentUid?: () => Promise<number | undefined>;
2021
2673
  runCommand?: CommandRunner;
2022
- probeGateway?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
2023
2674
  now?: () => number;
2024
2675
  sleep?: (ms: number) => Promise<void>;
2025
2676
  }
2026
2677
 
2027
- function recoveryGatewayLaunchdService(config: RecoveryConfig, uid: number): LaunchdService {
2028
- const label = 'com.moretea.forge-recovery-gateway';
2678
+ export interface RecoveryGatewayRestartDependencies extends RecoveryRoleRestartDependencies {
2679
+ probeGateway?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
2680
+ }
2681
+
2682
+ export interface RecoveryWatchdogRestartDependencies extends RecoveryRoleRestartDependencies {
2683
+ probeWatchdog?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
2684
+ }
2685
+
2686
+ function recoveryRoleLaunchdService(config: RecoveryConfig, uid: number, role: 'gateway' | 'watchdog'): LaunchdService {
2687
+ const label = role === 'gateway' ? RECOVERY_GATEWAY_LABEL : RECOVERY_WATCHDOG_LABEL;
2029
2688
  const generated = join(recoveryRoot(config), 'launchd', `${label}.plist`);
2030
2689
  const installed = join(homedir(), 'Library', 'LaunchAgents', `${label}.plist`);
2031
2690
  return {
@@ -2042,46 +2701,81 @@ async function probeRecoveryGateway(config: RecoveryConfig): Promise<{ ok: boole
2042
2701
  return probe(createRecoveryHttpTransport(config.controllerHome), `http://${config.gateway.host}:${config.gateway.port}/health`);
2043
2702
  }
2044
2703
 
2045
- export async function restartRecoveryGateway(
2046
- config: RecoveryConfig,
2047
- dependencies: RecoveryGatewayRestartDependencies = {},
2048
- ): Promise<RecoveryGatewayRestartResult> {
2049
- if (!config.gateway) return { ok: false, attempted: false, noOp: true, detail: 'Recovery Gateway is not configured' };
2050
- if ((dependencies.platform ?? process.platform) !== 'darwin') return { ok: false, attempted: false, noOp: true, detail: 'Recovery Gateway launchd restart is only supported on macOS' };
2051
- const check = dependencies.probeGateway ?? probeRecoveryGateway;
2052
- const initial = await check(config);
2053
- if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: 'Recovery Gateway is already healthy' };
2054
- const uid = await (dependencies.currentUid ?? currentUid)();
2055
- if (uid === undefined) return { ok: false, attempted: false, noOp: true, detail: 'Recovery Gateway launchd UID is unavailable' };
2056
- const service = recoveryGatewayLaunchdService(config, uid);
2057
- if (!existsSync(service.plistPath)) return { ok: false, attempted: false, noOp: true, detail: `Recovery Gateway launchd plist is missing: ${service.plistPath}`, serviceTarget: service.target };
2058
- const locked = await withLock(config, { action: 'restart_recovery_gateway' }, async () => {
2059
- const before = await check(config);
2060
- if (before.ok) return { ok: true, attempted: false, noOp: true, detail: 'Recovery Gateway recovered before restart', serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
2061
- const started = await ensureLaunchdServiceStarted(service, dependencies.runCommand ?? command);
2704
+ async function probeRecoveryWatchdog(config: RecoveryConfig): Promise<{ ok: boolean; detail: string }> {
2705
+ const health = observeRecoveryWatchdogHealth(config.controllerHome);
2706
+ return { ok: health.ok, detail: health.detail };
2707
+ }
2708
+
2709
+ async function restartRecoveryRole(input: {
2710
+ config: RecoveryConfig;
2711
+ role: 'gateway' | 'watchdog';
2712
+ action: 'restart_recovery_gateway' | 'restart_recovery_watchdog';
2713
+ check: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
2714
+ dependencies: RecoveryRoleRestartDependencies;
2715
+ }): Promise<RecoveryGatewayRestartResult> {
2716
+ const display = input.role === 'gateway' ? 'Recovery Gateway' : 'Recovery Watchdog';
2717
+ if ((input.dependencies.platform ?? process.platform) !== 'darwin') return { ok: false, attempted: false, noOp: true, detail: `${display} launchd restart is only supported on macOS` };
2718
+ const initial = await input.check(input.config);
2719
+ if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: `${display} is already healthy` };
2720
+ const uid = await (input.dependencies.currentUid ?? currentUid)();
2721
+ if (uid === undefined) return { ok: false, attempted: false, noOp: true, detail: `${display} launchd UID is unavailable` };
2722
+ const service = recoveryRoleLaunchdService(input.config, uid, input.role);
2723
+ if (!existsSync(service.plistPath)) return { ok: false, attempted: false, noOp: true, detail: `${display} launchd plist is missing: ${service.plistPath}`, serviceTarget: service.target };
2724
+ const eventPrefix = input.role === 'gateway' ? 'recovery_gateway' : 'recovery_watchdog';
2725
+ const locked = await withLock(input.config, { action: input.action }, async () => {
2726
+ const before = await input.check(input.config);
2727
+ if (before.ok) return { ok: true, attempted: false, noOp: true, detail: `${display} recovered before restart`, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
2728
+ const started = await ensureLaunchdServiceStarted(service, input.dependencies.runCommand ?? command);
2062
2729
  if (!started.ok) {
2063
- audit(config, 'recovery_gateway_restart_failed', { serviceTarget: service.target, detail: started.detail });
2730
+ audit(input.config, `${eventPrefix}_restart_failed`, { serviceTarget: service.target, detail: started.detail });
2064
2731
  return { ok: false, attempted: true, detail: started.detail, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
2065
2732
  }
2066
- const now = dependencies.now ?? Date.now;
2067
- const wait = dependencies.sleep ?? sleep;
2733
+ const now = input.dependencies.now ?? Date.now;
2734
+ const wait = input.dependencies.sleep ?? sleep;
2068
2735
  const deadline = now() + 20_000;
2069
2736
  let observed = before;
2070
2737
  while (now() < deadline) {
2071
2738
  await wait(1_000);
2072
- observed = await check(config);
2739
+ observed = await input.check(input.config);
2073
2740
  if (observed.ok) {
2074
- audit(config, 'recovery_gateway_restart_succeeded', { serviceTarget: service.target });
2075
- return { ok: true, attempted: true, detail: 'Recovery Gateway restarted and its local health endpoint recovered', serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
2741
+ audit(input.config, `${eventPrefix}_restart_succeeded`, { serviceTarget: service.target });
2742
+ return { ok: true, attempted: true, detail: `${display} restarted and passed local health verification`, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
2076
2743
  }
2077
2744
  }
2078
- audit(config, 'recovery_gateway_restart_unverified', { serviceTarget: service.target, detail: observed.detail });
2079
- return { ok: false, attempted: true, detail: 'Recovery Gateway restarted but did not pass local health verification before timeout', serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
2745
+ audit(input.config, `${eventPrefix}_restart_unverified`, { serviceTarget: service.target, detail: observed.detail });
2746
+ return { ok: false, attempted: true, detail: `${display} restarted but did not pass local health verification before timeout`, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
2080
2747
  });
2081
2748
  if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), serviceTarget: service.target };
2082
2749
  return locked.value;
2083
2750
  }
2084
2751
 
2752
+ export async function restartRecoveryGateway(
2753
+ config: RecoveryConfig,
2754
+ dependencies: RecoveryGatewayRestartDependencies = {},
2755
+ ): Promise<RecoveryGatewayRestartResult> {
2756
+ if (!config.gateway) return { ok: false, attempted: false, noOp: true, detail: 'Recovery Gateway is not configured' };
2757
+ return restartRecoveryRole({
2758
+ config,
2759
+ role: 'gateway',
2760
+ action: 'restart_recovery_gateway',
2761
+ check: dependencies.probeGateway ?? probeRecoveryGateway,
2762
+ dependencies,
2763
+ });
2764
+ }
2765
+
2766
+ export async function restartRecoveryWatchdog(
2767
+ config: RecoveryConfig,
2768
+ dependencies: RecoveryWatchdogRestartDependencies = {},
2769
+ ): Promise<RecoveryWatchdogRestartResult> {
2770
+ return restartRecoveryRole({
2771
+ config,
2772
+ role: 'watchdog',
2773
+ action: 'restart_recovery_watchdog',
2774
+ check: dependencies.probeWatchdog ?? probeRecoveryWatchdog,
2775
+ dependencies,
2776
+ });
2777
+ }
2778
+
2085
2779
  function pidAlive(pid: number | undefined): boolean {
2086
2780
  if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 0) return false;
2087
2781
  try { process.kill(pid, 0); return true; } catch { return false; }
@@ -2101,7 +2795,7 @@ export function gatewayToken(config: RecoveryConfig): string | undefined {
2101
2795
  export function initializeStandaloneRecovery(
2102
2796
  controllerHome: string,
2103
2797
  port = 8787,
2104
- extensions: Partial<Pick<RecoveryConfig, 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
2798
+ extensions: Partial<Pick<RecoveryConfig, 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryPublicTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
2105
2799
  ): RecoveryConfig {
2106
2800
  const root = resolve(controllerHome);
2107
2801
  const tokenPath = join(root, 'recovery', 'config', 'gateway-token.json');
@@ -2128,6 +2822,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
2128
2822
  rollback?: RollbackResult;
2129
2823
  publicTunnelRepair?: PublicTunnelRepairResult;
2130
2824
  recoveryGatewayRestart?: RecoveryGatewayRestartResult;
2825
+ primaryConnectorRestart?: PrimaryConnectorRestartResult;
2131
2826
  primaryRuntimeRestart?: PrimaryRuntimeRestartResult;
2132
2827
  primaryRuntimeRecovery?: PrimaryRuntimeRecoveryResult;
2133
2828
  }> {
@@ -2160,6 +2855,35 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
2160
2855
  fullVerificationPerformed = true;
2161
2856
  }
2162
2857
  const lastFullVerifyAt = fullVerificationPerformed ? now : scopedPrior.lastFullVerifyAt;
2858
+ const activeMutation = liveRecoveryMutationLock(config);
2859
+ if (activeMutation) {
2860
+ const attributable = recoveryLockOwnerAttributable(activeMutation);
2861
+ if (!attributable) {
2862
+ audit(config, 'recovery_operation_lock_identity_uncertain', {
2863
+ pid: activeMutation.pid,
2864
+ instanceId: activeMutation.instanceId,
2865
+ action: activeMutation.action ?? null,
2866
+ requestId: activeMutation.requestId ?? null,
2867
+ acquiredAt: activeMutation.acquiredAt,
2868
+ });
2869
+ }
2870
+ const reason = attributable
2871
+ ? `Recovery mutation in progress: action=${activeMutation.action} request=${activeMutation.requestId}; watchdog repair escalation suppressed`
2872
+ : 'Recovery mutation lock identity is uncertain; watchdog repair escalation suppressed';
2873
+ const state: WatchdogState = {
2874
+ ...scopedPrior,
2875
+ failures: 0,
2876
+ firstFailureAt: undefined,
2877
+ publicTunnelFailures: 0,
2878
+ publicTunnelFirstFailureAt: undefined,
2879
+ primaryConnectorFailures: 0,
2880
+ primaryConnectorFirstFailureAt: undefined,
2881
+ lastFullVerifyAt,
2882
+ lastDecision: 'degraded',
2883
+ lastReason: reason,
2884
+ };
2885
+ return { state, decision: { action: 'degraded', reason }, verify: verified };
2886
+ }
2163
2887
  if (runtimeStartupGrace && !localVerify.ok) {
2164
2888
  const state: WatchdogState = {
2165
2889
  ...scopedPrior,
@@ -2168,6 +2892,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
2168
2892
  runtimeHealthySince: undefined,
2169
2893
  lastFullVerifyAt,
2170
2894
  lastDecision: 'degraded',
2895
+ lastReason: 'canonical Runtime is within startup grace; restart escalation suppressed',
2171
2896
  };
2172
2897
  return {
2173
2898
  state,
@@ -2177,8 +2902,46 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
2177
2902
  }
2178
2903
  const recoveryHealthy = verified.probes.recovery_gateway?.ok !== false
2179
2904
  && verified.probes.recovery_external_http?.ok !== false;
2180
- const primaryRuntimeHealthy = verified.ok && localVerify.ok;
2181
- if (primaryRuntimeHealthy && recoveryHealthy) {
2905
+ const primaryRuntimeHealthy = localVerify.runtime.ok
2906
+ && localVerify.runtime.running
2907
+ && localVerify.runtime.ready
2908
+ && !localVerify.runtime.stale
2909
+ && localVerify.probes.active_gateway?.ok === true;
2910
+ const primaryConnectorConfigured = Boolean(config.primaryConnectorService);
2911
+ const primaryConnectorLocalFailed = verified.probes.primary_connector_local?.ok === false;
2912
+ const primaryConnectorCapacityFailed = connectorCapacityRecoveryRecommended(verified);
2913
+ const primaryPublicTransportManaged = Boolean(configuredPrimaryPublicTunnel(config));
2914
+ const primaryPublicTransportFailed = Boolean(
2915
+ config.publicMcpUrl
2916
+ && primaryRuntimeHealthy
2917
+ && !primaryConnectorCapacityFailed
2918
+ && (
2919
+ verified.probes.external_mcp_http?.ok === false
2920
+ || verified.probes.primary_connector_ready?.ok === false
2921
+ ),
2922
+ );
2923
+ const primaryConnectorFailed = Boolean(
2924
+ primaryConnectorConfigured
2925
+ && primaryRuntimeHealthy
2926
+ && (
2927
+ primaryConnectorLocalFailed
2928
+ || primaryConnectorCapacityFailed
2929
+ || (primaryPublicTransportManaged && primaryPublicTransportFailed)
2930
+ ),
2931
+ );
2932
+ if (primaryRuntimeHealthy && !primaryConnectorFailed && !primaryPublicTransportFailed && recoveryHealthy) {
2933
+ if (fullVerificationPerformed && verified.ok && !matchingKnownGood(config, verified.releases.active)) {
2934
+ try {
2935
+ const attestation = await withLock(config, { action: 'attest_known_good' }, async () => persistVerifiedKnownGood(config, verified));
2936
+ if (!attestation.acquired) {
2937
+ audit(config, 'known_good_attestation_deferred', { reason: recoveryBusyDetail(attestation.owner) });
2938
+ }
2939
+ } catch (error) {
2940
+ audit(config, 'known_good_attestation_deferred', {
2941
+ reason: error instanceof Error ? error.message : 'watchdog known-good attestation failed',
2942
+ });
2943
+ }
2944
+ }
2182
2945
  const stable = recordWatchdogRuntimeHealthy(
2183
2946
  scopeWatchdogStateToRuntimeRelease(scopedPrior, verified.releases.active),
2184
2947
  now,
@@ -2193,12 +2956,38 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
2193
2956
  publicTunnelFailures: 0,
2194
2957
  publicTunnelFirstFailureAt: undefined,
2195
2958
  publicTunnelRepairFailures: 0,
2959
+ primaryConnectorFailures: 0,
2960
+ primaryConnectorFirstFailureAt: undefined,
2961
+ primaryConnectorRestartAttempts: 0,
2962
+ primaryConnectorRestartFailures: 0,
2963
+ primaryConnectorRestartLastAttemptAt: undefined,
2196
2964
  recoveryGatewayRestartUsed: false,
2197
2965
  lastFullVerifyAt,
2198
2966
  lastDecision: 'healthy',
2967
+ lastReason: 'primary runtime and standalone Recovery verification passed',
2199
2968
  };
2200
2969
  return { state, decision: { action: 'healthy', reason: 'primary runtime and standalone Recovery verification passed' }, verify: verified };
2201
2970
  }
2971
+ if (primaryRuntimeHealthy && !primaryConnectorFailed && primaryPublicTransportFailed && recoveryHealthy) {
2972
+ const reason = primaryPublicTransportManaged
2973
+ ? 'primary public transport is unavailable but no safely repairable primary Connector path was established'
2974
+ : 'primary public transport is unavailable while the local Connector is healthy, but no managed primary public tunnel is configured; external transport repair is required';
2975
+ const state: WatchdogState = {
2976
+ ...recordWatchdogRuntimeHealthy(
2977
+ scopeWatchdogStateToRuntimeRelease(scopedPrior, verified.releases.active),
2978
+ now,
2979
+ watchdogRuntimeRestartBudgetStableMs(config),
2980
+ ),
2981
+ failures: scopedPrior.failures + 1,
2982
+ firstFailureAt: scopedPrior.firstFailureAt ?? now,
2983
+ primaryConnectorFailures: 0,
2984
+ primaryConnectorFirstFailureAt: undefined,
2985
+ lastFullVerifyAt,
2986
+ lastDecision: 'degraded',
2987
+ lastReason: reason,
2988
+ };
2989
+ return { state, decision: { action: 'degraded', reason }, verify: verified };
2990
+ }
2202
2991
  const publicTunnelFailed = isExternalTunnelFailure(config, verified, localVerify);
2203
2992
  const evidenceClasses = Object.entries(localVerify.probes)
2204
2993
  .filter(([name, value]) => !name.startsWith('recovery_') && !value.ok)
@@ -2221,16 +3010,31 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
2221
3010
  firstFailureAt: budgetPrior.firstFailureAt ?? now,
2222
3011
  publicTunnelFailures: (budgetPrior.publicTunnelFailures ?? 0) + 1,
2223
3012
  publicTunnelFirstFailureAt: budgetPrior.publicTunnelFirstFailureAt ?? now,
3013
+ primaryConnectorFailures: 0,
3014
+ primaryConnectorFirstFailureAt: undefined,
2224
3015
  }
2225
- : {
2226
- ...budgetPrior,
2227
- lastFullVerifyAt,
2228
- failures: budgetPrior.failures + 1,
2229
- firstFailureAt: budgetPrior.firstFailureAt ?? now,
2230
- publicTunnelFailures: 0,
2231
- publicTunnelFirstFailureAt: undefined,
2232
- };
2233
- if (!localVerify.ok) state.runtimeHealthySince = undefined;
3016
+ : primaryConnectorFailed
3017
+ ? {
3018
+ ...budgetPrior,
3019
+ lastFullVerifyAt,
3020
+ failures: budgetPrior.failures + 1,
3021
+ firstFailureAt: budgetPrior.firstFailureAt ?? now,
3022
+ publicTunnelFailures: 0,
3023
+ publicTunnelFirstFailureAt: undefined,
3024
+ primaryConnectorFailures: (budgetPrior.primaryConnectorFailures ?? 0) + 1,
3025
+ primaryConnectorFirstFailureAt: budgetPrior.primaryConnectorFirstFailureAt ?? now,
3026
+ }
3027
+ : {
3028
+ ...budgetPrior,
3029
+ lastFullVerifyAt,
3030
+ failures: budgetPrior.failures + 1,
3031
+ firstFailureAt: budgetPrior.firstFailureAt ?? now,
3032
+ publicTunnelFailures: 0,
3033
+ publicTunnelFirstFailureAt: undefined,
3034
+ primaryConnectorFailures: 0,
3035
+ primaryConnectorFirstFailureAt: undefined,
3036
+ };
3037
+ if (!primaryRuntimeHealthy) state.runtimeHealthySince = undefined;
2234
3038
  const primaryConfig = configuredPrimaryRuntimeService(config);
2235
3039
  const decision = decideWatchdog({
2236
3040
  ...state,
@@ -2238,7 +3042,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
2238
3042
  evidenceClasses,
2239
3043
  activeKnownGood,
2240
3044
  previousKnownGood,
2241
- primaryRuntimeFailed: !localVerify.ok,
3045
+ primaryRuntimeFailed: !primaryRuntimeHealthy,
2242
3046
  runtimeMaximumRestartAttempts: primaryConfig.maximumRestartAttempts,
2243
3047
  runtimeRestartCooldownMs: primaryConfig.restartCooldownMs,
2244
3048
  runtimeMinimumFailures: primaryConfig.minimumFailures,
@@ -2246,12 +3050,23 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
2246
3050
  runtimeRecoveryLastAttemptAt: state.runtimeRecoveryLastAttemptAt,
2247
3051
  runtimeRecoveryCooldownMs: primaryConfig.recoveryCooldownMs,
2248
3052
  recoveryGatewayFailed: verified.probes.recovery_gateway?.ok === false,
3053
+ primaryConnectorConfigured,
3054
+ primaryConnectorFailed,
3055
+ primaryConnectorFailures: state.primaryConnectorFailures,
3056
+ primaryConnectorFirstFailureAt: state.primaryConnectorFirstFailureAt,
3057
+ primaryConnectorRestartAttempts: state.primaryConnectorRestartAttempts,
3058
+ primaryConnectorMaximumRestartAttempts: config.primaryConnectorService?.maximumRestartAttempts,
3059
+ primaryConnectorRestartLastAttemptAt: state.primaryConnectorRestartLastAttemptAt,
3060
+ primaryConnectorRestartCooldownMs: config.primaryConnectorService?.restartCooldownMs,
3061
+ primaryConnectorMinimumFailures: config.primaryConnectorService?.minimumFailures,
3062
+ primaryConnectorMinimumFailureDurationMs: config.primaryConnectorService?.minimumFailureDurationMs,
2249
3063
  publicTunnelConfigured: Boolean(configuredRecoveryTunnel(config)),
2250
3064
  publicTunnelFailed,
2251
3065
  publicTunnelMinimumFailures: configuredRecoveryTunnel(config)?.minimumFailures,
2252
3066
  publicTunnelMinimumFailureDurationMs: configuredRecoveryTunnel(config)?.minimumFailureDurationMs,
2253
3067
  });
2254
3068
  state.lastDecision = decision.action;
3069
+ state.lastReason = decision.reason;
2255
3070
  if (decision.action === 'repair_public_tunnel') {
2256
3071
  const publicTunnelRepair = await repairPublicTunnel(config);
2257
3072
  const nextState = publicTunnelRepair.ok
@@ -2263,6 +3078,30 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
2263
3078
  const recoveryGatewayRestart = await restartRecoveryGateway(config);
2264
3079
  return { state: { ...state, recoveryGatewayRestartUsed: true }, decision, verify: verified, recoveryGatewayRestart };
2265
3080
  }
3081
+ if (decision.action === 'restart_primary_connector') {
3082
+ const primaryConnectorRestart = await restartPrimaryConnector(config, {
3083
+ requestId: `watchdog:restart_primary_connector:${now}`,
3084
+ });
3085
+ const attempts = (state.primaryConnectorRestartAttempts ?? 0) + (primaryConnectorRestart.attempted ? 1 : 0);
3086
+ const nextState: WatchdogState = primaryConnectorRestart.ok
3087
+ ? {
3088
+ ...state,
3089
+ failures: 0,
3090
+ firstFailureAt: undefined,
3091
+ primaryConnectorFailures: 0,
3092
+ primaryConnectorFirstFailureAt: undefined,
3093
+ primaryConnectorRestartAttempts: 0,
3094
+ primaryConnectorRestartFailures: 0,
3095
+ primaryConnectorRestartLastAttemptAt: now,
3096
+ }
3097
+ : {
3098
+ ...state,
3099
+ primaryConnectorRestartAttempts: attempts,
3100
+ primaryConnectorRestartFailures: (state.primaryConnectorRestartFailures ?? 0) + (primaryConnectorRestart.attempted ? 1 : 0),
3101
+ primaryConnectorRestartLastAttemptAt: now,
3102
+ };
3103
+ return { state: nextState, decision, verify: verified, primaryConnectorRestart };
3104
+ }
2266
3105
  if (decision.action === 'restart_primary_runtime') {
2267
3106
  const primaryRuntimeRestart = await restartPrimaryRuntime(config);
2268
3107
  const attempts = (state.runtimeRestartAttempts ?? 0) + (primaryRuntimeRestart.attempted ? 1 : 0);