@moretea-labs/forge 1.8.0 → 1.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (528) hide show
  1. package/AGENTS.md +70 -7
  2. package/CHANGELOG.md +14 -0
  3. package/CLAUDE.md +17 -2
  4. package/CONTRIBUTING.md +3 -2
  5. package/THIRD_PARTY_NOTICES.md +1 -1
  6. package/adapters/chatgpt/browser-delivery-host.ts +7 -1
  7. package/adapters/chatgpt/browser-delivery-runtime.ts +529 -56
  8. package/adapters/chatgpt/controller-binding-store.ts +101 -22
  9. package/adapters/chatgpt/controller-round-host.ts +15 -25
  10. package/adapters/chatgpt/controller-round-settlement-store.ts +58 -4
  11. package/adapters/chatgpt/provider-delivery.ts +228 -1
  12. package/adapters/chatgpt/work-conversation-binding-store.ts +79 -53
  13. package/adapters/computer/chatgpt-extension-target.ts +463 -0
  14. package/adapters/computer/chatgpt-target-router.ts +103 -0
  15. package/adapters/computer/chrome-extension/background.js +385 -0
  16. package/adapters/computer/chrome-extension/content.js +378 -0
  17. package/adapters/computer/chrome-extension/core.js +24 -0
  18. package/adapters/computer/chrome-extension/manifest.json +30 -0
  19. package/adapters/computer/desktop-operator-discovery.ts +11 -31
  20. package/adapters/computer/interaction-target-authority.ts +142 -18
  21. package/adapters/mcp/auth.ts +0 -1
  22. package/adapters/mcp/instructions.ts +5 -4
  23. package/adapters/mcp/multi-repository.ts +3 -19
  24. package/adapters/mcp/oauth.ts +73 -0
  25. package/adapters/mcp/paths.ts +4 -1
  26. package/adapters/mcp/runtime-gateway/automation-receipt-adapter.ts +87 -0
  27. package/adapters/mcp/runtime-gateway/bounded-result-runtime.ts +62 -0
  28. package/adapters/mcp/runtime-gateway/cognition-settlement.ts +201 -0
  29. package/adapters/mcp/runtime-gateway/context-adapter.ts +411 -73
  30. package/adapters/mcp/runtime-gateway/context-pack-sidecar.ts +91 -0
  31. package/adapters/mcp/runtime-gateway/controller-authority-adapter.ts +20 -5
  32. package/adapters/mcp/runtime-gateway/core-capability-adapter.ts +258 -0
  33. package/adapters/mcp/runtime-gateway/engineering-preconditions.ts +10 -27
  34. package/adapters/mcp/runtime-gateway/engineering-tool-contract.ts +15 -0
  35. package/adapters/mcp/runtime-gateway/execution-tools.ts +8 -42
  36. package/adapters/mcp/runtime-gateway/gateway-execution-adapter.ts +2 -1
  37. package/adapters/mcp/runtime-gateway/plugin-adapter.ts +117 -45
  38. package/adapters/mcp/runtime-gateway/process-tools.ts +190 -22
  39. package/adapters/mcp/runtime-gateway/protected-computer-adapter.ts +16 -4
  40. package/adapters/mcp/runtime-gateway/recovery-adapter.ts +69 -52
  41. package/adapters/mcp/runtime-gateway/repository-compat-adapter.ts +4 -35
  42. package/adapters/mcp/runtime-gateway/result-adapter.ts +10 -4
  43. package/adapters/mcp/runtime-gateway/router.ts +18 -504
  44. package/adapters/mcp/runtime-gateway/routing-policy.ts +499 -0
  45. package/adapters/mcp/runtime-gateway/runtime-observation-adapter.ts +14 -65
  46. package/adapters/mcp/runtime-gateway/runtime-readiness-observation.ts +218 -0
  47. package/adapters/mcp/runtime-gateway/runtime-tool-definitions.ts +113 -173
  48. package/adapters/mcp/runtime-gateway/runtime-tools.ts +15 -5
  49. package/adapters/mcp/runtime-gateway/scheduler-adapter.ts +1 -0
  50. package/adapters/mcp/runtime-gateway/shared-adapter.ts +20 -0
  51. package/adapters/mcp/runtime-gateway/status-inbox-adapter.ts +358 -266
  52. package/adapters/mcp/runtime-gateway/typescript-navigation-sidecar.ts +136 -0
  53. package/adapters/mcp/runtime-gateway/work-adapter.ts +261 -3122
  54. package/adapters/mcp/runtime-gateway/work-controller-operations.ts +257 -0
  55. package/adapters/mcp/runtime-gateway/work-detail-projection.ts +60 -0
  56. package/adapters/mcp/runtime-gateway/work-input-wire-migration.ts +246 -0
  57. package/adapters/mcp/runtime-gateway/work-learning-operations.ts +145 -0
  58. package/adapters/mcp/runtime-gateway/work-list-projection.ts +38 -0
  59. package/adapters/mcp/runtime-gateway/work-plan-operations.ts +382 -0
  60. package/adapters/mcp/runtime-gateway/work-repair-adapter.ts +308 -0
  61. package/adapters/mcp/runtime-gateway/work-requirement-operations.ts +148 -0
  62. package/adapters/mcp/runtime-gateway/work-semantic-operations.ts +263 -0
  63. package/adapters/mcp/runtime-gateway/work-workflow-operations.ts +113 -0
  64. package/adapters/mcp/runtime-gateway/workflow-supervisor-adapter.ts +156 -0
  65. package/adapters/mcp/server.ts +67 -23
  66. package/adapters/mcp/setup.ts +1 -3
  67. package/adapters/mcp/tool-mapping/legacy-tool-service.ts +44 -170
  68. package/adapters/mcp/tool-mapping/post-finalize-work-attribution.ts +4 -9
  69. package/adapters/mcp/tool-mapping/repository-tools.ts +133 -142
  70. package/adapters/mcp/toolset-names.ts +5 -0
  71. package/adapters/mcp/toolset.ts +6 -5
  72. package/adapters/mcp/transports/http-observation.ts +365 -0
  73. package/adapters/mcp/transports/http.ts +154 -677
  74. package/adapters/mcp/transports/oauth-http.ts +322 -0
  75. package/adapters/mcp/transports/session-registry.ts +6 -0
  76. package/adapters/mcp/tunnels/openai-secure-tunnel.ts +89 -5
  77. package/assets/plugin-registry.v1.json +6 -4
  78. package/assets/reference-configs/agentic-development-flow.md +25 -0
  79. package/assets/reference-configs/global-working-rules.md +19 -0
  80. package/assets/skill-version.json +2 -2
  81. package/docs/architecture/CURRENT.md +111 -41
  82. package/docs/architecture/decisions/20260803-release-identity-binding-and-exit-policy.md +14 -5
  83. package/docs/architecture/decisions/20260823-immediate-controller-relay.md +6 -3
  84. package/docs/architecture/decisions/20260907-assistant-learning-loop.md +2 -0
  85. package/docs/architecture/decisions/20260907-controller-round-transition-policy.md +19 -4
  86. package/docs/architecture/decisions/20260916-workflow-supervisor-authority.md +249 -0
  87. package/docs/architecture/decisions/20260917-cognitive-memory-foundation.md +82 -0
  88. package/docs/architecture/decisions/20260918-stable-candidate-release-lifecycle.md +221 -0
  89. package/docs/architecture/decisions/20260919-autonomous-work-liveness.md +62 -0
  90. package/docs/architecture/decisions/20260924-thin-semantic-working-context.md +72 -0
  91. package/docs/architecture/decisions/20261001-supervisor-turn-authority.md +237 -0
  92. package/docs/architecture/decisions/20261005-forge-v3-desktop-client-boundary.md +77 -0
  93. package/docs/architecture/domains/verification.md +1 -0
  94. package/docs/architecture/domains/workflow-engine.md +6 -0
  95. package/docs/architecture/modules/controller-runtime/transport-lifecycle.md +1 -1
  96. package/docs/forge-plugin-management.md +2 -2
  97. package/docs/operations/controller-performance-and-502.md +17 -0
  98. package/docs/operations/controller-reliability-runbook.md +2 -2
  99. package/docs/operations/features.md +1 -1
  100. package/docs/operations/features.zh-CN.md +1 -1
  101. package/docs/operations/homebrew.md +2 -2
  102. package/docs/operations/mcp-tool-exposure.md +2 -0
  103. package/docs/operations/releasing.md +2 -0
  104. package/docs/operations/standalone-disaster-recovery.md +14 -6
  105. package/docs/wiki/Implementation.md +1 -1
  106. package/package.json +25 -14
  107. package/packages/kernel/cognition/api/index.ts +6 -0
  108. package/packages/kernel/cognition/application/activation-service.ts +451 -0
  109. package/packages/kernel/cognition/application/consolidation-service.ts +185 -0
  110. package/packages/kernel/cognition/application/legacy-experience-adapter.ts +38 -0
  111. package/packages/kernel/cognition/application/memory-service.ts +64 -0
  112. package/packages/kernel/cognition/domain/learning.ts +106 -0
  113. package/packages/kernel/cognition/domain/memory.ts +254 -0
  114. package/packages/kernel/controller/api/index.ts +1 -0
  115. package/packages/kernel/controller/application/continuation-service.ts +281 -0
  116. package/packages/kernel/controller/application/controller-round-service.ts +12 -0
  117. package/packages/kernel/controller/domain/controller-round-transition-policy.ts +242 -21
  118. package/packages/kernel/controller/domain/controller-round.ts +4 -1
  119. package/packages/kernel/controller/domain/execution-quality.ts +3 -1
  120. package/packages/kernel/controller/infrastructure/controller-round-store.ts +689 -103
  121. package/packages/kernel/controller/infrastructure/controller-session-store.ts +297 -71
  122. package/packages/kernel/controller/ports/controller-host.ts +2 -0
  123. package/packages/kernel/identity/api/index.ts +10 -0
  124. package/packages/kernel/identity/domain/activity-projection.ts +43 -0
  125. package/packages/kernel/identity/domain/capability-broker.ts +60 -0
  126. package/packages/kernel/identity/domain/grant.ts +72 -0
  127. package/packages/kernel/identity/domain/owned-resource.ts +116 -0
  128. package/packages/kernel/identity/domain/user-request.ts +93 -0
  129. package/packages/kernel/identity/infrastructure/activity-projection-store.ts +77 -0
  130. package/packages/kernel/identity/infrastructure/capability-broker.ts +81 -0
  131. package/packages/kernel/identity/infrastructure/grant-store.ts +284 -0
  132. package/packages/kernel/identity/infrastructure/owned-resource-store.ts +259 -0
  133. package/packages/kernel/identity/infrastructure/user-request-store.ts +182 -0
  134. package/packages/kernel/scheduler/api/index.ts +0 -2
  135. package/packages/kernel/scheduler/application/schedule-service.ts +3 -0
  136. package/packages/kernel/scheduler/application/settlement.ts +0 -15
  137. package/packages/kernel/scheduler/domain/schedule.ts +87 -0
  138. package/packages/kernel/scheduler/infrastructure/schedule-store.ts +153 -133
  139. package/packages/kernel/work/api/index.ts +3 -2
  140. package/packages/kernel/work/application/work-service.ts +17 -42
  141. package/packages/kernel/work/domain/current-task-lineage.ts +104 -0
  142. package/packages/kernel/work/domain/engineering-design.ts +10 -5
  143. package/packages/kernel/work/domain/engineering-profile.ts +8 -12
  144. package/packages/kernel/work/domain/evaluation-promotion-receipt.ts +186 -0
  145. package/packages/kernel/work/domain/execution-concurrency.ts +19 -26
  146. package/packages/kernel/work/domain/execution-snapshot.ts +0 -31
  147. package/packages/kernel/work/domain/implementation-review.ts +14 -331
  148. package/packages/kernel/work/domain/objective-graph.ts +119 -0
  149. package/packages/kernel/work/domain/project-engineering-contract.ts +23 -1
  150. package/packages/kernel/work/domain/repository-completion-receipt.ts +1 -1
  151. package/packages/kernel/work/domain/state-machine.ts +25 -63
  152. package/packages/kernel/work/domain/types.ts +57 -51
  153. package/packages/kernel/work/infrastructure/work-contract-store.ts +819 -728
  154. package/packages/kernel/work/ports/work-contract-store.ts +5 -2
  155. package/packages/plugin-runtime/browser/session-authority.ts +2 -1
  156. package/packages/plugin-runtime/computer/chatgpt-identity.ts +24 -0
  157. package/packages/plugin-runtime/computer/conversation-target.ts +143 -0
  158. package/packages/plugin-runtime/computer/index.ts +2 -0
  159. package/packages/plugin-runtime/computer/target-authority.ts +26 -1
  160. package/packages/plugin-runtime/external/unix-jsonl-transport.ts +142 -18
  161. package/packages/protocols/computer/contract.ts +27 -1
  162. package/scripts/activate-source-baseline.ts +7 -9
  163. package/scripts/canary-controller-semantic-wait.ts +12 -24
  164. package/scripts/check-mcp-compatibility.ts +48 -136
  165. package/scripts/check-runtime-architecture.mjs +622 -217
  166. package/scripts/forge-local-recovery-helper.mjs +93 -15
  167. package/scripts/lib/project-init-lib.sh +3 -0
  168. package/scripts/open-source-audit-allowlist.txt +1 -2
  169. package/scripts/run-bounded-command.mjs +5 -0
  170. package/scripts/run-governed-gate.ts +0 -1
  171. package/scripts/run-mobile-monitor-bridge.ts +0 -1
  172. package/scripts/smoke-control-plane-failure-domain.ts +94 -0
  173. package/scripts/smoke-mcp-http-runtime.ts +5 -0
  174. package/scripts/smoke-runtime-recovery.ts +12 -43
  175. package/scripts/smoke-schedule-engine.ts +23 -15
  176. package/scripts/smoke-unattended-transport-recovery.ts +223 -0
  177. package/scripts/smoke-user-request-continuation-restart.ts +232 -0
  178. package/scripts/smoke-workflow-supervisor-chrome-adapter.ts +92 -0
  179. package/scripts/smoke-workflow-supervisor-process-restart.ts +276 -0
  180. package/scripts/smoke-workflow-supervisor-restart-recovery.ts +150 -0
  181. package/scripts/smoke-workflow-supervisor.ts +103 -0
  182. package/scripts/verify-forge-runtime.sh +2 -1
  183. package/src/cli/agent-jobs/executor-health.ts +11 -34
  184. package/src/cli/agent-jobs/integration.ts +18 -49
  185. package/src/cli/agent-jobs/job-manager.ts +7 -3
  186. package/src/cli/commands/brain-assistant.ts +27 -13
  187. package/src/cli/commands/brain.ts +13 -15
  188. package/src/cli/commands/chatgpt.ts +1 -163
  189. package/src/cli/commands/recovery.ts +135 -121
  190. package/src/cli/commands/runtime.ts +3 -3
  191. package/src/cli/commands/setup-profile.ts +8 -0
  192. package/src/cli/controller/check-runner.ts +13 -2
  193. package/src/cli/controller/completion-backlog.ts +3 -3
  194. package/src/cli/controller/completion-orchestrator.ts +4 -43
  195. package/src/cli/controller/completion-target.ts +3 -72
  196. package/src/cli/controller/context/source-materializer.ts +81 -100
  197. package/src/cli/controller/context-pack.ts +10 -8
  198. package/src/cli/controller/execution-completion.ts +4 -7
  199. package/src/cli/controller/execution-policy.ts +36 -153
  200. package/src/cli/controller/governance.ts +2 -3
  201. package/src/cli/controller/issue-store.ts +4 -99
  202. package/src/cli/controller/launch-agents.ts +59 -18
  203. package/src/cli/controller/operational-plan.ts +24 -51
  204. package/src/cli/controller/runtime-config.ts +5 -1
  205. package/src/cli/controller/stuck-state-migration.ts +2 -3
  206. package/src/cli/controller/systemd-user.ts +39 -1
  207. package/src/cli/editing/edit-session.ts +1 -7
  208. package/src/cli/local-bridge/console-automations.ts +18 -13
  209. package/src/cli/local-bridge/console-view-models.ts +3 -12
  210. package/src/cli/local-bridge/facade-api.ts +64 -191
  211. package/src/cli/local-bridge/server.ts +24 -147
  212. package/src/cli/local-bridge/types.ts +0 -1
  213. package/src/cli/mcp/oauth-client-retirement.ts +74 -0
  214. package/src/cli/repositories/command-executor.ts +29 -23
  215. package/src/cli/repositories/command-scope.ts +35 -16
  216. package/src/cli/repositories/controller-home.ts +48 -0
  217. package/src/cli/repositories/ephemeral-workspace.ts +20 -30
  218. package/src/cli/repositories/git-command-executor.ts +2 -2
  219. package/src/cli/repositories/registry.ts +77 -11
  220. package/src/cli/repositories/structured-git.ts +62 -2
  221. package/src/cli/repository/session-cache.ts +92 -6
  222. package/src/runtime/context/assistant-context.ts +20 -7
  223. package/src/runtime/context/assistant-work-context.ts +89 -27
  224. package/src/runtime/context/automatic-learning.ts +565 -0
  225. package/src/runtime/context/cognitive-attention.ts +173 -0
  226. package/src/runtime/context/cognitive-skill-canary.ts +158 -0
  227. package/src/runtime/context/context-closure.ts +46 -9
  228. package/src/runtime/context/context-pack-process.ts +103 -0
  229. package/src/runtime/context/context-plane.ts +1 -9
  230. package/src/runtime/context/direct-controller-learning.ts +298 -0
  231. package/src/runtime/context/semantic-navigation-contract.ts +9 -0
  232. package/src/runtime/context/semantic-navigation.ts +6 -34
  233. package/src/runtime/context/typescript-navigation-process.ts +145 -0
  234. package/src/runtime/context/typescript-navigation.ts +89 -3
  235. package/src/runtime/control-plane/cleanup-artifact-retention.ts +212 -9
  236. package/src/runtime/control-plane/concurrency/work-execution-concurrency.ts +9 -47
  237. package/src/runtime/control-plane/execution/controller-authority-recovery.ts +34 -4
  238. package/src/runtime/control-plane/execution/edit-validation-coordinator.ts +29 -20
  239. package/src/runtime/control-plane/execution/execution-identity.ts +23 -3
  240. package/src/runtime/control-plane/execution/repository-work-attribution.ts +7 -30
  241. package/src/runtime/control-plane/execution/retained-work-resume.ts +206 -199
  242. package/src/runtime/control-plane/execution/session-store.ts +31 -1
  243. package/src/runtime/control-plane/execution/validation.ts +6 -8
  244. package/src/runtime/control-plane/execution/work-authority-reconciler.ts +13 -42
  245. package/src/runtime/control-plane/execution/work-check-evidence.ts +137 -0
  246. package/src/runtime/control-plane/execution/work-completion-authority.ts +30 -85
  247. package/src/runtime/control-plane/execution/work-delivery-service.ts +497 -0
  248. package/src/runtime/control-plane/execution/work-evidence-policy.ts +12 -50
  249. package/src/runtime/control-plane/execution/work-execution-support.ts +50 -149
  250. package/src/runtime/control-plane/execution/work-handle-authority.ts +188 -109
  251. package/src/runtime/control-plane/execution/work-handle-store.ts +19 -43
  252. package/src/runtime/control-plane/execution/work-head-settlement.ts +53 -20
  253. package/src/runtime/control-plane/execution/work-lifecycle-audit.ts +52 -29
  254. package/src/runtime/control-plane/execution/work-operation-service.ts +43 -27
  255. package/src/runtime/control-plane/execution/work-preparation-service.ts +126 -266
  256. package/src/runtime/control-plane/execution/work-remote-delivery.ts +2 -3
  257. package/src/runtime/control-plane/execution/work-revision-diff.ts +39 -0
  258. package/src/runtime/control-plane/execution/work-terminal-cleanup.ts +173 -186
  259. package/src/runtime/control-plane/execution/work-validation-reconciler.ts +28 -84
  260. package/src/runtime/control-plane/execution/work-verification-service.ts +406 -50
  261. package/src/runtime/control-plane/execution/work-verification-snapshot.ts +5 -1
  262. package/src/runtime/control-plane/facade/capability-registry.ts +362 -37
  263. package/src/runtime/control-plane/facade/check-normalization.ts +3 -9
  264. package/src/runtime/control-plane/facade/codex-delegation.ts +10 -11
  265. package/src/runtime/control-plane/facade/controller-connection.ts +135 -0
  266. package/src/runtime/control-plane/facade/goal-workloop-access.ts +1 -20
  267. package/src/runtime/control-plane/facade/goal-workloop.ts +281 -1753
  268. package/src/runtime/control-plane/facade/handoff-inbox-application.ts +12 -25
  269. package/src/runtime/control-plane/facade/handoff-inbox-store.ts +256 -52
  270. package/src/runtime/control-plane/facade/index.ts +0 -1
  271. package/src/runtime/control-plane/facade/operation-digest.ts +12 -15
  272. package/src/runtime/control-plane/facade/plan-contract-store.ts +451 -991
  273. package/src/runtime/control-plane/facade/policy-gate.ts +0 -23
  274. package/src/runtime/control-plane/facade/repository-work-admission.ts +12 -22
  275. package/src/runtime/control-plane/facade/requirement-authority.ts +102 -199
  276. package/src/runtime/control-plane/facade/requirement-board.ts +3 -3
  277. package/src/runtime/control-plane/facade/rh-work-operation-contract.ts +51 -14
  278. package/src/runtime/control-plane/facade/self-healing-loop.ts +18 -53
  279. package/src/runtime/control-plane/facade/suggested-actions.ts +15 -18
  280. package/src/runtime/control-plane/facade/types.ts +49 -108
  281. package/src/runtime/control-plane/facade/work-continuation.ts +8 -8
  282. package/src/runtime/control-plane/generated-cache-retention.ts +97 -1
  283. package/src/runtime/control-plane/global-scheduler/autonomous-continuation.ts +248 -0
  284. package/src/runtime/control-plane/global-scheduler/durable-admission.ts +1 -1
  285. package/src/runtime/control-plane/global-scheduler/maintenance.ts +239 -112
  286. package/src/runtime/control-plane/global-scheduler/reconciliation.ts +2 -1
  287. package/src/runtime/control-plane/global-scheduler/scheduler.ts +66 -40
  288. package/src/runtime/control-plane/global-scheduler/wake-signal.ts +42 -2
  289. package/src/runtime/control-plane/governance/access-policy.ts +19 -2
  290. package/src/runtime/control-plane/governance/authorization.ts +20 -3
  291. package/src/runtime/control-plane/launcher/chatgpt-round-continuation.ts +63 -300
  292. package/src/runtime/control-plane/launcher/chatgpt-work-continuation.ts +28 -300
  293. package/src/runtime/control-plane/launcher/local-conversation-controller.ts +238 -0
  294. package/src/runtime/control-plane/launcher/provider-mcp-bootstrap.ts +5 -2
  295. package/src/runtime/control-plane/launcher/thin-launcher.ts +96 -240
  296. package/src/runtime/control-plane/persistence/cognition-store.ts +820 -0
  297. package/src/runtime/control-plane/persistence/experience-store.ts +162 -29
  298. package/src/runtime/control-plane/persistence/requirement-store.ts +216 -74
  299. package/src/runtime/control-plane/persistence/sqlite-store.ts +113 -6
  300. package/src/runtime/control-plane/release-retention.ts +135 -47
  301. package/src/runtime/control-plane/repository-namespace-retention.ts +2 -2
  302. package/src/runtime/control-plane/runtime-cleanup.ts +290 -21
  303. package/src/runtime/control-plane/workspace/project-onboarding.ts +80 -0
  304. package/src/runtime/control-plane/workspace/semantic-sync-service.ts +19 -5
  305. package/src/runtime/control-plane/workspace/workspace-store.ts +169 -0
  306. package/src/runtime/diagnostics/incident-repair.ts +313 -146
  307. package/src/runtime/diagnostics/mcp-timing.ts +114 -0
  308. package/src/runtime/evidence/artifact-store.ts +158 -40
  309. package/src/runtime/evidence/evidence-store.ts +112 -8
  310. package/src/runtime/evidence/output-handle-index.ts +225 -0
  311. package/src/runtime/evidence/result-store.ts +158 -44
  312. package/src/runtime/execution/jobs/store.ts +3 -4
  313. package/src/runtime/execution/managed-workspace.ts +57 -0
  314. package/src/runtime/execution/process-runtime/check-facade.ts +1 -1
  315. package/src/runtime/execution/process-runtime/check-result-retention.ts +2 -2
  316. package/src/runtime/execution/process-runtime/check-result.ts +105 -1
  317. package/src/runtime/execution/process-runtime/check-runner-sidecar.ts +18 -8
  318. package/src/runtime/execution/process-runtime/command-facade.ts +79 -27
  319. package/src/runtime/execution/process-runtime/gc.ts +15 -2
  320. package/src/runtime/execution/process-runtime/handle-index.ts +176 -0
  321. package/src/runtime/execution/process-runtime/host-command-lane.ts +139 -0
  322. package/src/runtime/execution/process-runtime/index.ts +3 -0
  323. package/src/runtime/execution/process-runtime/lightweight-managed.ts +251 -72
  324. package/src/runtime/execution/process-runtime/persisted-check.ts +3 -3
  325. package/src/runtime/execution/process-runtime/process-runner-entry.ts +47 -5
  326. package/src/runtime/execution/process-runtime/process-scope.ts +160 -0
  327. package/src/runtime/execution/process-runtime/resource-claims.ts +65 -61
  328. package/src/runtime/execution/process-runtime/runtime.ts +35 -2
  329. package/src/runtime/execution/process-runtime/store.ts +30 -2
  330. package/src/runtime/execution/process-runtime/types.ts +6 -1
  331. package/src/runtime/execution/workers/executor.ts +2 -6
  332. package/src/runtime/platform/service-inventory.ts +130 -0
  333. package/src/runtime/platform/service-manager.ts +8 -0
  334. package/src/runtime/plugins/action-application.ts +85 -28
  335. package/src/runtime/plugins/app-store-connect-adapter.ts +38 -14
  336. package/src/runtime/plugins/browser-adapter.ts +757 -34
  337. package/src/runtime/plugins/browser-automation-service.ts +23 -3
  338. package/src/runtime/plugins/browser-macos-bridge.ts +321 -29
  339. package/src/runtime/plugins/browser-manifest-surface.ts +45 -11
  340. package/src/runtime/plugins/browser-native-session-service.ts +63 -13
  341. package/src/runtime/plugins/browser-registration.ts +16 -0
  342. package/src/runtime/plugins/browser-runtime.ts +14 -1
  343. package/src/runtime/plugins/browser-session-authority.ts +2 -2
  344. package/src/runtime/plugins/browser-session-store.ts +46 -76
  345. package/src/runtime/plugins/capability-authorization-grants.ts +258 -62
  346. package/src/runtime/plugins/computer-chatgpt-browser-adapter-installer.ts +219 -0
  347. package/src/runtime/plugins/computer-chatgpt-macos-target.ts +891 -0
  348. package/src/runtime/plugins/computer-registration.ts +222 -11
  349. package/src/runtime/plugins/desktop-operator-external-policy.ts +36 -4
  350. package/src/runtime/plugins/desktop-operator-registration.ts +12 -4
  351. package/src/runtime/plugins/external-adapter.ts +42 -6
  352. package/src/runtime/plugins/external-registration.ts +21 -1
  353. package/src/runtime/plugins/gmail-adapter.ts +23 -2
  354. package/src/runtime/plugins/google-calendar-adapter.ts +2 -2
  355. package/src/runtime/plugins/google-shared.ts +14 -7
  356. package/src/runtime/plugins/google-tasks-adapter.ts +2 -2
  357. package/src/runtime/plugins/ios-adapter.ts +1 -1
  358. package/src/runtime/plugins/lightweight-action.ts +208 -2
  359. package/src/runtime/plugins/local-system-adapter.ts +12 -2
  360. package/src/runtime/plugins/managed-process-adapter.ts +12 -5
  361. package/src/runtime/plugins/plugin-action-sidecar.ts +17 -13
  362. package/src/runtime/plugins/plugin-management-adapter.ts +1 -1
  363. package/src/runtime/plugins/store.ts +602 -428
  364. package/src/runtime/plugins/types.ts +20 -0
  365. package/src/runtime/plugins/xiaohongshu-publish.ts +16 -0
  366. package/src/runtime/projections/invalidation.ts +7 -3
  367. package/src/runtime/projections/materialized-view.ts +80 -36
  368. package/src/runtime/recovery/actions.ts +3 -3
  369. package/src/runtime/recovery/capability-action-application.ts +46 -45
  370. package/src/runtime/recovery/lifecycle-inventory.ts +29 -29
  371. package/src/runtime/recovery/maintenance-executor.ts +374 -123
  372. package/src/runtime/recovery/store.ts +29 -11
  373. package/src/runtime/release/release-coordinator.ts +296 -0
  374. package/src/runtime/release/release-session.ts +716 -0
  375. package/src/runtime/release/release-state-migration.ts +25 -0
  376. package/src/runtime/resources/leases/store.ts +56 -25
  377. package/src/runtime/root/computer-composition.ts +191 -6
  378. package/src/runtime/root/controller-progression-composition.ts +192 -0
  379. package/src/runtime/root/controller-round-composition.ts +20 -1
  380. package/src/runtime/root/deployment-topology.ts +106 -0
  381. package/src/runtime/root/entry.ts +4 -3
  382. package/src/runtime/root/known-good-recovery.ts +184 -0
  383. package/src/runtime/root/local-bridge.ts +0 -1
  384. package/src/runtime/root/mcp-transport.ts +111 -18
  385. package/src/runtime/root/ownership.ts +261 -9
  386. package/src/runtime/root/package-connector-service.ts +105 -19
  387. package/src/runtime/root/package-runtime-release.ts +117 -18
  388. package/src/runtime/root/package-runtime-service.ts +17 -11
  389. package/src/runtime/root/release-execution-canary.ts +44 -12
  390. package/src/runtime/root/release-loader.ts +90 -8
  391. package/src/runtime/root/release-manifest.ts +80 -6
  392. package/src/runtime/root/release-materialize.ts +372 -28
  393. package/src/runtime/root/release-store.ts +395 -36
  394. package/src/runtime/root/runtime-lane.ts +191 -0
  395. package/src/runtime/root/runtime.ts +159 -27
  396. package/src/runtime/root/scheduled-controller-composition.ts +28 -3
  397. package/src/runtime/root/service-runner.ts +2 -2
  398. package/src/runtime/root/service.ts +36 -33
  399. package/src/runtime/root/status.ts +48 -0
  400. package/src/runtime/root/types.ts +32 -3
  401. package/src/runtime/root/workflow-supervisor-composition.ts +493 -0
  402. package/src/runtime/root/workflow-supervisor-runtime.ts +209 -0
  403. package/src/runtime/root/write-fence.ts +50 -2
  404. package/src/runtime/safe-tooling/google-credential-store.ts +76 -21
  405. package/src/runtime/safe-tooling/google-oauth-broker.ts +25 -7
  406. package/src/runtime/safe-tooling/workspace-auth.ts +1 -1
  407. package/src/runtime/shared/bounded-child-supervisor.ts +9 -1
  408. package/src/runtime/shared/process-environment.ts +20 -9
  409. package/src/runtime/shared/response-budget.ts +1 -3
  410. package/src/runtime/standalone-recovery/controller-home-migration.ts +16 -24
  411. package/src/runtime/standalone-recovery/core.ts +3702 -342
  412. package/src/runtime/standalone-recovery/entry.ts +469 -39
  413. package/src/runtime/standalone-recovery/installer.ts +262 -148
  414. package/src/runtime/standalone-recovery/mcp-server.ts +29 -193
  415. package/src/runtime/standalone-recovery/operation-lock.ts +1 -1
  416. package/src/runtime/standalone-recovery/performance.ts +54 -16
  417. package/src/runtime/standalone-recovery/release.ts +53 -5
  418. package/src/runtime/standalone-recovery/service-labels.ts +5 -2
  419. package/src/runtime/standalone-recovery/watchdog-heartbeat.ts +5 -2
  420. package/src/runtime/workflow/schedules/browser-probe.ts +17 -6
  421. package/src/runtime/workflow/schedules/chatgpt-automation-migration.ts +10 -3
  422. package/src/runtime/workflow/schedules/engine.ts +138 -172
  423. package/src/runtime/workflow/schedules/work-continuation.ts +44 -5
  424. package/src/runtime/workflows/runtime.ts +15 -19
  425. package/src/testing/test-governance.ts +5 -1
  426. package/supervisor/chatgpt-conversation.ts +23 -0
  427. package/supervisor/client.ts +201 -0
  428. package/supervisor/control-plane.ts +868 -0
  429. package/supervisor/entry.ts +15 -0
  430. package/supervisor/forge-validators.ts +80 -0
  431. package/supervisor/index.ts +5 -0
  432. package/supervisor/native-browser-adapter.ts +786 -0
  433. package/supervisor/native-messaging/host.ts +39 -0
  434. package/supervisor/native-messaging/manifest.ts +26 -0
  435. package/supervisor/native-messaging/protocol.ts +27 -0
  436. package/supervisor/paths.ts +18 -0
  437. package/supervisor/protocol.ts +202 -0
  438. package/supervisor/server.ts +404 -0
  439. package/supervisor/store.ts +1451 -0
  440. package/supervisor/tsconfig.json +8 -0
  441. package/supervisor/types.ts +255 -0
  442. package/adapters/chatgpt/controller-host.ts +0 -119
  443. package/adapters/mcp/controller-round-compatibility.ts +0 -184
  444. package/adapters/mcp/frozen-client-semantic-compatibility.ts +0 -292
  445. package/adapters/mcp/runtime-gateway/work-compat-adapter.ts +0 -502
  446. package/packages/kernel/progression/api/index.ts +0 -2
  447. package/packages/kernel/progression/application/projector.ts +0 -211
  448. package/packages/kernel/progression/domain/types.ts +0 -116
  449. package/packages/kernel/progression/index.ts +0 -1
  450. package/packages/kernel/scheduler/application/continuation-service.ts +0 -212
  451. package/packages/kernel/scheduler/domain/continuation.ts +0 -38
  452. package/packages/kernel/scheduler/infrastructure/continuation-dispatch-store.ts +0 -38
  453. package/packages/kernel/work/domain/admission-policy.ts +0 -34
  454. package/scripts/benchmark-background-check-overlap.ts +0 -228
  455. package/scripts/benchmark-computer-hotpath.ts +0 -255
  456. package/scripts/benchmark-controller-hotpath.ts +0 -212
  457. package/scripts/benchmark-controller-round2.ts +0 -191
  458. package/scripts/benchmark-ios-agent-device.ts +0 -511
  459. package/scripts/benchmark-mcp-host.sh +0 -117
  460. package/scripts/benchmark-mcp-transport.ts +0 -169
  461. package/scripts/benchmark-operational-memory-activation.ts +0 -235
  462. package/scripts/benchmark-operational-shadow.ts +0 -28
  463. package/scripts/benchmark-quality-harness.ts +0 -146
  464. package/scripts/benchmark-route-session-concurrency.ts +0 -915
  465. package/scripts/benchmark-thin-harness-gateway-ab.ts +0 -632
  466. package/scripts/benchmark-thin-harness-mcp-e2e.ts +0 -498
  467. package/scripts/benchmark-thin-harness.ts +0 -251
  468. package/scripts/benchmark-typescript-navigation.ts +0 -186
  469. package/scripts/benchmark-work-efficiency.ts +0 -334
  470. package/scripts/build-controller-ui.ts +0 -28
  471. package/scripts/check-npm-release.sh +0 -32
  472. package/scripts/check-public-release-surface.sh +0 -19
  473. package/scripts/check-release-published.sh +0 -57
  474. package/scripts/check-release-readiness.sh +0 -37
  475. package/scripts/check-release-version.mjs +0 -74
  476. package/scripts/check-tarball-install-smoke.sh +0 -150
  477. package/scripts/package-source-archive.sh +0 -147
  478. package/scripts/public-release-files.txt +0 -39
  479. package/scripts/public-release.ts +0 -174
  480. package/scripts/publish-release-tarball.sh +0 -27
  481. package/scripts/route-nl-vs-ts-eval.ts +0 -679
  482. package/scripts/stage-runtime-release.ts +0 -44
  483. package/src/cli/controller/progress.ts +0 -656
  484. package/src/cli/controller/work-mode.ts +0 -181
  485. package/src/cli/local-bridge/console-assets.ts +0 -17
  486. package/src/cli/local-bridge/dashboard.ts +0 -17
  487. package/src/cli/local-bridge/frontend/src/App.tsx +0 -10
  488. package/src/cli/local-bridge/frontend/src/components/Button.tsx +0 -1
  489. package/src/cli/local-bridge/frontend/src/components/DetailPane.tsx +0 -3
  490. package/src/cli/local-bridge/frontend/src/components/SearchField.tsx +0 -1
  491. package/src/cli/local-bridge/frontend/src/components/SectionHeader.tsx +0 -1
  492. package/src/cli/local-bridge/frontend/src/components/Segmented.tsx +0 -1
  493. package/src/cli/local-bridge/frontend/src/components/StatusText.tsx +0 -3
  494. package/src/cli/local-bridge/frontend/src/design/icons.tsx +0 -11
  495. package/src/cli/local-bridge/frontend/src/design/tokens.css +0 -8
  496. package/src/cli/local-bridge/frontend/src/features/automations/AutomationsView.tsx +0 -153
  497. package/src/cli/local-bridge/frontend/src/features/capabilities/CapabilitiesView.tsx +0 -37
  498. package/src/cli/local-bridge/frontend/src/features/overview/OverviewView.tsx +0 -183
  499. package/src/cli/local-bridge/frontend/src/features/repositories/RepositoriesView.tsx +0 -2
  500. package/src/cli/local-bridge/frontend/src/features/system/SystemView.tsx +0 -2
  501. package/src/cli/local-bridge/frontend/src/features/work/WorkView.tsx +0 -4
  502. package/src/cli/local-bridge/frontend/src/lib/api.ts +0 -30
  503. package/src/cli/local-bridge/frontend/src/lib/format.ts +0 -4
  504. package/src/cli/local-bridge/frontend/src/lib/routes.ts +0 -7
  505. package/src/cli/local-bridge/frontend/src/main.tsx +0 -1
  506. package/src/cli/local-bridge/frontend/src/shell/AppShell.tsx +0 -1
  507. package/src/cli/local-bridge/frontend/src/shell/CommandBar.tsx +0 -2
  508. package/src/cli/local-bridge/frontend/src/shell/Sidebar.tsx +0 -4
  509. package/src/cli/local-bridge/frontend/src/styles.css +0 -28
  510. package/src/cli/local-bridge/frontend/src/types.ts +0 -15
  511. package/src/cli/local-bridge/frontend/src/vite-env.d.ts +0 -1
  512. package/src/cli/local-bridge/frontend/vite.config.ts +0 -27
  513. package/src/cli/local-bridge/ui-dist/app.css +0 -1
  514. package/src/cli/local-bridge/ui-dist/app.js +0 -11228
  515. package/src/runtime/control-plane/execution/content-equivalent-commit-authority.ts +0 -169
  516. package/src/runtime/control-plane/execution/direct-canonical-work-reconciliation.ts +0 -85
  517. package/src/runtime/control-plane/execution/direct-edit-work-completion.ts +0 -655
  518. package/src/runtime/control-plane/execution/work-finalization-service.ts +0 -2850
  519. package/src/runtime/control-plane/execution/work-path-scope.ts +0 -37
  520. package/src/runtime/control-plane/execution/work-task-receipt.ts +0 -468
  521. package/src/runtime/control-plane/facade/controller-round-relay.ts +0 -3
  522. package/src/runtime/control-plane/facade/work-admission-policy.ts +0 -142
  523. package/src/runtime/control-plane/facade/work-implementation-review.ts +0 -2
  524. package/src/runtime/control-plane/facade/work-state-machine.ts +0 -2
  525. package/src/runtime/control-plane/facade/work-task-projection.ts +0 -66
  526. package/src/runtime/control-plane/routing/index.ts +0 -1
  527. package/src/runtime/control-plane/routing/route-policy.ts +0 -401
  528. package/src/runtime/control-plane/routing/workspace-admission.ts +0 -70
@@ -1,20 +1,37 @@
1
1
  import { assertRuntimePerformanceEvidence, measureRuntimePerformance, samePerformanceIdentity, type RuntimePerformanceDependencies, type RuntimePerformanceEvidence, type RuntimePerformanceIdentity } from './performance';
2
+ import { runBoundedChild } from '../shared/bounded-child-supervisor';
2
3
  import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'crypto';
3
4
  import { spawn, spawnSync } from 'child_process';
4
- import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from 'fs';
5
+ import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, renameSync, rmSync, statSync, writeFileSync } from 'fs';
5
6
  import { homedir, hostname } from 'os';
7
+ import { createServer as createNetServer } from 'net';
6
8
  import { basename, delimiter, dirname, isAbsolute, join, resolve } from 'path';
7
9
  import { assertStorageHeadroom } from '../shared/storage-capacity';
8
- import { resolveBunExecutable } from '../shared/process-environment';
9
- import { observeRuntimeStatus } from '../root/status';
10
+ import { resolveBunExecutable, runtimeAuthorityFreeEnvironment } from '../shared/process-environment';
11
+ import { backupControlPlaneDatabase } from '../control-plane/persistence/sqlite-store';
12
+ import { measureReclaimablePath } from '../control-plane/lifecycle-retention-metrics';
13
+ import { cleanupControllerReleaseHistory } from '../control-plane/release-retention';
14
+ import { observeRuntimeStatus, readRuntimeStartupFailureEvidence } from '../root/status';
15
+ import { reconcileStoppedRuntimeOwnership, terminateVerifiedRuntimeOwner, readRuntimeOwner } from '../root/ownership';
16
+ import type { RuntimeOwnerRecord } from '../root/ownership';
10
17
  import {
11
18
  activeRuntimeEntrypoint,
12
19
  activeRuntimeLaunchSpec,
13
20
  ensureForgeRuntimeLaunchAgentContract,
14
21
  forgeRuntimeServicePaths,
15
22
  inspectForgeRuntimeLaunchAgentContract,
23
+ installForgeRuntimeService,
16
24
  readForgeRuntimeServiceConfig,
25
+ syncForgeRuntimeActiveEntrypoint,
26
+ uninstallForgeRuntimeService,
17
27
  } from '../root/service';
28
+ import { removeRetiredCandidateExecutionLane } from '../root/runtime-lane';
29
+ import {
30
+ inspectKnownGoodRecoveryBundle,
31
+ knownGoodRecoveryBundleRoot,
32
+ sha256FileBounded,
33
+ type KnownGoodRecoveryBundle,
34
+ } from '../root/known-good-recovery';
18
35
  import { systemdUserUnitName, systemdUserUnitPath } from '../../cli/controller/systemd-user';
19
36
  import {
20
37
  openAiSecureTunnelConnectArgs,
@@ -24,14 +41,18 @@ import {
24
41
  } from '../../../adapters/mcp/tunnels/openai-secure-tunnel';
25
42
  import {
26
43
  renderPackageRuntimeSystemdUserService,
44
+ systemdRuntimeInstallCommands,
27
45
  writePackageRuntimeSystemdUserService,
28
46
  } from '../root/package-runtime-service';
29
47
  import { loadRuntimeReleaseManifest } from '../root/release-manifest';
30
- import { assertRuntimeReleaseExecutionCanaries, assertRuntimeReleaseFiles, stageRuntimeReleaseFromCandidateSource, type RuntimeReleaseExecutionCanaryDependencies, type StagedRuntimeRelease } from '../root/release-materialize';
48
+ import { assertRuntimeReleaseExecutionCanaries, assertRuntimeReleaseFiles, promotePortableRuntimeRelease, runtimeReleaseTreeSha256, stageRuntimeReleaseFromCandidateSource, withRuntimeReleaseSourceSnapshot, type RuntimeReleaseExecutionCanaryDependencies, type StagedRuntimeRelease } from '../root/release-materialize';
31
49
  import {
50
+ prepareRuntimeReleaseDatabaseBackup,
32
51
  publishRuntimeRelease,
33
52
  readRuntimeReleaseAuthority,
34
- rollbackRuntimeRelease,
53
+ reconcileFailedRuntimeReleaseActivationAuthority,
54
+ rollbackRuntimeReleaseWithResult,
55
+ type RuntimeDatabaseRollbackDisposition,
35
56
  type RuntimePublishedRelease,
36
57
  type RuntimeReleaseAuthority,
37
58
  } from '../root/release-store';
@@ -39,16 +60,34 @@ import type { RuntimeReleaseManifest } from '../root/types';
39
60
  import { ensurePackageConnectorService, packageConnectorAuthMode, packageConnectorServicePaths, type PackageConnectorReleaseBinding } from '../root/package-connector-service';
40
61
  import { createRecoveryHttpTransport, type RecoveryHttpTransport } from './http-transport';
41
62
  import { observeRecoveryWatchdogHealth } from './watchdog-heartbeat';
42
- import { RECOVERY_GATEWAY_LABEL, RECOVERY_WATCHDOG_LABEL } from './service-labels';
63
+ import { RECOVERY_DAEMON_LABEL } from './service-labels';
64
+ import { resolveWorkflowSupervisorForgeHome, workflowSupervisorSocketPath } from '../../../supervisor/paths';
65
+ import { getWorkflowSupervisorContinuationProof } from '../../../supervisor/client';
66
+ import type { WorkflowSupervisorContinuationProof } from '../../../supervisor/types';
67
+ import { reconcileStoppedWorkflowSupervisorSocket } from '../../../supervisor/server';
43
68
  import {
44
69
  migrateStoppedRepoLocalControllerHomeStorage,
45
70
  repoLocalControllerHomeStorageNeedsMigration,
46
71
  rollbackStoppedRepoLocalControllerHomeStorage,
47
72
  type ControllerHomeStorageMigration,
48
73
  } from '../../cli/repositories/controller-home';
49
- import { readCurrentRecoveryRelease } from './release';
74
+ import { readCurrentRecoveryRelease, type RecoveryRuntimeRole } from './release';
50
75
  import { recoveryOperationLockPath } from './operation-lock';
51
76
  import { RECOVERY_MUTATION_IDENTITY_FIELDS, type RecoveryMutationIdentityArguments } from './mutation-identity-contract';
77
+ import { createCandidateExecutionLane, readStableExecutionLane } from '../root/runtime-lane';
78
+ import {
79
+ advanceReleaseSession,
80
+ createReleaseSession,
81
+ listCompleteReleaseSessions,
82
+ readReleaseSession,
83
+ recordReleaseSessionTransaction,
84
+ releaseSessionCandidateIsRetired,
85
+ releaseSessionIsSoakingPredecessorOfStable,
86
+ releaseSessionIsSoakingSupersededByStableAuthority,
87
+ type ReleaseSession,
88
+ type ReleaseSessionCandidateRelease,
89
+ type ReleaseSessionStableRelease,
90
+ } from '../release/release-session';
52
91
 
53
92
  /** Standalone recovery reads only canonical Runtime observation and whole-release authority. */
54
93
  interface PublicTunnelRecoveryPolicy {
@@ -125,9 +164,24 @@ export interface SystemdPrimaryConnectorServiceConfig extends PrimaryConnectorRe
125
164
 
126
165
  export type PrimaryConnectorServiceConfig = LaunchdPrimaryConnectorServiceConfig | SystemdPrimaryConnectorServiceConfig;
127
166
 
167
+ export type RecoveryInstallProfile = 'manual' | 'gateway' | 'self-healing';
168
+
169
+ export function normalizeRecoveryInstallProfile(value: unknown, fallback: RecoveryInstallProfile = 'self-healing'): RecoveryInstallProfile {
170
+ if (value === undefined || value === null || value === '') return fallback;
171
+ if (value === 'manual' || value === 'gateway' || value === 'self-healing') return value;
172
+ throw new Error(`RECOVERY_INSTALL_PROFILE_INVALID: ${String(value)}`);
173
+ }
174
+
175
+ export function recoveryInstallProfileRoles(profile: RecoveryInstallProfile): RecoveryRuntimeRole[] {
176
+ if (profile === 'manual') return [];
177
+ return ['daemon'];
178
+ }
179
+
128
180
  export interface RecoveryConfig {
129
181
  schemaVersion: 1;
130
182
  controllerHome: string;
183
+ /** Persistent Recovery role selection. Legacy configs default to self-healing. */
184
+ installProfile: RecoveryInstallProfile;
131
185
  publicMcpUrl?: string;
132
186
  recoveryPublicUrl?: string;
133
187
  recoveryTunnelService?: RecoveryTunnelServiceConfig;
@@ -220,28 +274,46 @@ interface ReleaseEvidence {
220
274
  manifestSha256: string;
221
275
  workerProtocolVersion: number;
222
276
  controllerHome?: string;
277
+ sourceRepositoryId?: string;
223
278
  releaseAuthorityRevision?: number;
224
279
  releaseFencingTokenSha256?: string;
225
280
  attestedAt?: string;
281
+ pinnedAt?: string;
226
282
  performance?: RuntimePerformanceEvidence;
283
+ recoveryBundle?: KnownGoodRecoveryBundle;
227
284
  }
228
285
 
229
286
  interface KnownGoodStore {
230
- schemaVersion: 1;
287
+ schemaVersion: 1 | 2;
231
288
  releases: ReleaseEvidence[];
232
289
  updatedAt: string;
233
290
  }
234
291
 
292
+ interface RuntimePinStore {
293
+ schemaVersion: 1;
294
+ release: ReleaseEvidence;
295
+ updatedAt: string;
296
+ }
297
+
235
298
  export type RecoveryMutationAction =
236
299
  | 'attest_known_good'
237
300
  | 'rollback_previous'
238
301
  | 'restart_primary_runtime'
239
302
  | 'recover_primary_runtime'
240
303
  | 'activate_runtime_release'
304
+ | 'pin_runtime_release'
305
+ | 'unpin_runtime_release'
306
+ | 'activate_pinned_runtime_release'
241
307
  | 'stage_and_activate_runtime_release'
308
+ | 'release_session_prepare'
309
+ | 'release_session_static_verify'
310
+ | 'release_session_candidate_boot'
311
+ | 'release_session_cutover'
312
+ | 'release_session_cancel'
313
+ | 'release_session_rollback'
314
+ | 'release_session_known_good'
242
315
  | 'restart_primary_connector'
243
316
  | 'restart_recovery_gateway'
244
- | 'restart_recovery_watchdog'
245
317
  | 'repair_public_tunnel';
246
318
 
247
319
  interface RecoveryLock {
@@ -275,6 +347,26 @@ export interface VerifyStableRuntimeOptions extends RuntimeReleaseExecutionCanar
275
347
  probeMcpProtocol?: boolean;
276
348
  }
277
349
 
350
+ /**
351
+ * Remote tunnel control-plane lookup is weaker evidence than the live data plane.
352
+ * Once the configured tunnel is locally healthy/ready with matching identities and
353
+ * the public MCP protocol succeeds end-to-end, a degraded/unknown control-plane
354
+ * lookup remains diagnostic instead of vetoing whole-Runtime verification.
355
+ * Any local health, readiness, identity, endpoint, or MCP failure still gates.
356
+ */
357
+ export function isPrimaryTunnelControlPlaneOnlyDegradation(
358
+ probe: VerifyResult['probes'][string] | undefined,
359
+ mcpProtocolHealthy: boolean,
360
+ ): boolean {
361
+ if (!mcpProtocolHealthy || !probe || probe.ok) return false;
362
+ const observed = probe.value as Partial<OpenAiSecureTunnelRuntimeObservation> | undefined;
363
+ return observed?.healthy === true
364
+ && observed.ready === true
365
+ && observed.tunnelMatches === true
366
+ && observed.endpointMatches === true
367
+ && (observed.controlPlaneState === 'degraded' || observed.controlPlaneState === 'unknown');
368
+ }
369
+
278
370
  /** Bounded, release-scoped explanation for a failed watchdog verification. */
279
371
  export interface RecoveryWatchdogDiagnosticEvidence {
280
372
  fingerprint: string;
@@ -370,6 +462,8 @@ export interface ConfiguredRuntimeActivationResult {
370
462
  noOp?: boolean;
371
463
  detail: string;
372
464
  staged?: StagedRuntimeRelease;
465
+ releaseSession?: ReleaseSession;
466
+ /** Legacy/home-bound recovery activation only. Fresh portable source candidates never set this during preparation. */
373
467
  activation?: RuntimeReleaseActivationResult;
374
468
  }
375
469
 
@@ -415,6 +509,7 @@ export function defaultPrimaryRuntimeServiceConfig(platform: NodeJS.Platform = p
415
509
 
416
510
  const DEFAULT_CONFIG: Omit<RecoveryConfig, 'controllerHome'> = {
417
511
  schemaVersion: 1,
512
+ installProfile: 'self-healing',
418
513
  readOnlyTool: { name: STABLE_RECOVERY_READ_ONLY_TOOL.name, arguments: { ...STABLE_RECOVERY_READ_ONLY_TOOL.arguments } },
419
514
  primaryRuntimeService: defaultPrimaryRuntimeServiceConfig(),
420
515
  };
@@ -439,6 +534,7 @@ function writeJson(path: string, value: unknown): void {
439
534
 
440
535
  function recoveryRoot(config: RecoveryConfig): string { return join(resolve(config.controllerHome), 'recovery'); }
441
536
  function statePath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'state', 'known-good.json'); }
537
+ function runtimePinPath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'state', 'runtime-pin.json'); }
442
538
  function lockPath(config: RecoveryConfig): string { return recoveryOperationLockPath(config.controllerHome); }
443
539
  function auditPath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'audit', 'recovery.jsonl'); }
444
540
  function watchdogDiagnosticPath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'state', 'watchdog-diagnostics.json'); }
@@ -488,6 +584,7 @@ export function loadRecoveryConfig(controllerHome: string, explicit?: string): R
488
584
  ...DEFAULT_CONFIG,
489
585
  schemaVersion: 1,
490
586
  controllerHome: resolve(typeof loaded.controllerHome === 'string' ? loaded.controllerHome : controllerHome),
587
+ installProfile: normalizeRecoveryInstallProfile(loaded.installProfile, 'self-healing'),
491
588
  ...(typeof loaded.publicMcpUrl === 'string' ? { publicMcpUrl: loaded.publicMcpUrl } : {}),
492
589
  ...(typeof loaded.recoveryPublicUrl === 'string' ? { recoveryPublicUrl: loaded.recoveryPublicUrl } : {}),
493
590
  ...(loaded.recoveryTunnelService ? { recoveryTunnelService: loaded.recoveryTunnelService } : {}),
@@ -550,33 +647,193 @@ function previousAuthorityRelease(config: RecoveryConfig): ReleaseEvidence | und
550
647
  return releaseEvidence(config.controllerHome, authority?.previous, authority);
551
648
  }
552
649
 
553
- function knownGoodEvidence(config: RecoveryConfig, entry: ReleaseEvidence | undefined): ReleaseEvidence | undefined {
650
+ function sameReleaseIdentity(left: ReleaseEvidence, right: ReleaseEvidence): boolean {
651
+ return left.path === right.path
652
+ && left.revision === right.revision
653
+ && left.artifactIdentity === right.artifactIdentity
654
+ && left.manifestSha256 === right.manifestSha256
655
+ && left.workerProtocolVersion === right.workerProtocolVersion;
656
+ }
657
+
658
+ function knownGoodAttestationEvidence(config: RecoveryConfig, entry: ReleaseEvidence | undefined): ReleaseEvidence | undefined {
554
659
  if (!entry || !entry.attestedAt || !entry.releaseFencingTokenSha256 || typeof entry.controllerHome !== 'string') return undefined;
660
+ // Legacy metadata-only attestations are audit history, not recoverable authority.
661
+ // The cheap tier requires durable bundle identity but never reads bundle contents.
662
+ if (!entry.recoveryBundle || entry.recoveryBundle.schemaVersion !== 1) return undefined;
555
663
  if (resolve(entry.controllerHome) !== resolve(config.controllerHome)) return undefined;
556
664
  const authority = releaseAuthority(config);
557
665
  const candidates = [
558
666
  releaseEvidence(config.controllerHome, authority?.active, authority),
559
667
  releaseEvidence(config.controllerHome, authority?.previous, authority),
560
668
  ].filter((item): item is ReleaseEvidence => Boolean(item));
561
- const matched = candidates.find((release) =>
562
- release.path === entry.path
563
- && release.revision === entry.revision
564
- && release.artifactIdentity === entry.artifactIdentity
565
- && release.manifestSha256 === entry.manifestSha256
566
- && release.workerProtocolVersion === entry.workerProtocolVersion,
567
- );
568
- return matched ? entry : undefined;
669
+ return candidates.some((release) => sameReleaseIdentity(release, entry)) ? entry : undefined;
670
+ }
671
+
672
+ function knownGoodEvidence(config: RecoveryConfig, entry: ReleaseEvidence | undefined): ReleaseEvidence | undefined {
673
+ const attested = knownGoodAttestationEvidence(config, entry);
674
+ if (!attested) return undefined;
675
+ try { inspectKnownGoodRecoveryBundle(config.controllerHome, attested); }
676
+ catch { return undefined; }
677
+ return attested;
569
678
  }
570
679
 
571
680
  function knownGood(config: RecoveryConfig): KnownGoodStore {
572
681
  return json<KnownGoodStore>(statePath(config)) ?? { schemaVersion: 1, releases: [], updatedAt: new Date(0).toISOString() };
573
682
  }
574
683
 
684
+ function inspectKnownGoodRecoverability(config: RecoveryConfig): {
685
+ available: ReleaseEvidence[];
686
+ unavailable: Array<{ revision: string; path: string; reason: string }>;
687
+ } {
688
+ const available: ReleaseEvidence[] = [];
689
+ const unavailable: Array<{ revision: string; path: string; reason: string }> = [];
690
+ const store = knownGood(config);
691
+ // Pre-bundle records remain audit history only. They are deliberately
692
+ // neither usable Recovery authority nor a readiness failure for an
693
+ // otherwise healthy Runtime, because no action can make them restorable.
694
+ if (store.schemaVersion !== 2) return { available, unavailable };
695
+ const releasesRoot = resolve(config.controllerHome, 'runtime', 'releases');
696
+ const authority = releaseAuthority(config);
697
+ const rollbackReleasePaths = new Set<string>();
698
+ if (authority) {
699
+ rollbackReleasePaths.add(resolve(dirname(authority.active.manifestPath)));
700
+ if (authority.previous) rollbackReleasePaths.add(resolve(dirname(authority.previous.manifestPath)));
701
+ }
702
+ for (const entry of store.releases) {
703
+ if (!rollbackReleasePaths.has(resolve(dirname(entry.path)))) continue;
704
+ try {
705
+ const inspected = inspectKnownGoodRecoveryBundle(config.controllerHome, entry);
706
+ if (dirname(inspected.releaseRoot) !== releasesRoot) throw new Error('known-good path is outside Runtime release authority');
707
+ available.push(entry);
708
+ } catch (error) {
709
+ unavailable.push({
710
+ revision: entry.revision,
711
+ path: entry.path,
712
+ reason: error instanceof Error ? error.message : String(error),
713
+ });
714
+ }
715
+ }
716
+ return { available, unavailable };
717
+ }
718
+
719
+ function matchingKnownGoodAttestation(config: RecoveryConfig, release: ReleaseEvidence | undefined): ReleaseEvidence | undefined {
720
+ if (!release) return undefined;
721
+ return knownGoodAttestationEvidence(config, knownGood(config).releases.find((entry) => sameReleaseIdentity(entry, release)));
722
+ }
723
+
575
724
  function matchingKnownGood(config: RecoveryConfig, release: ReleaseEvidence | undefined): ReleaseEvidence | undefined {
576
725
  if (!release) return undefined;
577
- return knownGoodEvidence(config, knownGood(config).releases.find((entry) =>
578
- entry.revision === release.revision && entry.manifestSha256 === release.manifestSha256 && entry.path === release.path,
579
- ));
726
+ return knownGoodEvidence(config, knownGood(config).releases.find((entry) => sameReleaseIdentity(entry, release)));
727
+ }
728
+
729
+ function sameReleaseEvidenceIdentity(left: ReleaseEvidence | undefined, right: ReleaseEvidence | undefined): boolean {
730
+ return Boolean(
731
+ left
732
+ && right
733
+ && left.path === right.path
734
+ && left.revision === right.revision
735
+ && left.artifactIdentity === right.artifactIdentity
736
+ && left.manifestSha256 === right.manifestSha256
737
+ && left.workerProtocolVersion === right.workerProtocolVersion,
738
+ );
739
+ }
740
+
741
+ function liveRuntimeOwnsRelease(config: RecoveryConfig, release: ReleaseEvidence): boolean {
742
+ const active = activeAuthorityRelease(config);
743
+ const observed = observeRuntimeStatus(config.controllerHome);
744
+ const snapshot = observed.snapshot;
745
+ return Boolean(
746
+ sameReleaseEvidenceIdentity(active, release)
747
+ && observed.running
748
+ && observed.ready
749
+ && !observed.stale
750
+ && snapshot
751
+ && snapshot.releaseId === release.revision
752
+ && snapshot.artifactIdentity === release.artifactIdentity,
753
+ );
754
+ }
755
+
756
+ function runtimePin(config: RecoveryConfig): RuntimePinStore | undefined {
757
+ const path = runtimePinPath(config);
758
+ if (!existsSync(path)) return undefined;
759
+ let parsed: RuntimePinStore;
760
+ try { parsed = JSON.parse(readFileSync(path, 'utf8')) as RuntimePinStore; }
761
+ catch (error) { throw new Error(`RUNTIME_PIN_AUTHORITY_INVALID: ${error instanceof Error ? error.message : String(error)}`); }
762
+ const release = parsed?.release;
763
+ if (parsed?.schemaVersion !== 1 || !release || !release.revision?.trim() || !release.path?.trim() || !release.artifactIdentity?.trim() || !release.manifestSha256?.trim() || !Number.isInteger(release.workerProtocolVersion)) {
764
+ throw new Error('RUNTIME_PIN_AUTHORITY_INVALID');
765
+ }
766
+ const releasesRoot = resolve(config.controllerHome, 'runtime', 'releases');
767
+ const manifestPath = resolve(release.path);
768
+ const releaseRoot = dirname(manifestPath);
769
+ if (basename(manifestPath) !== 'manifest.json' || basename(releaseRoot) !== release.revision || dirname(releaseRoot) !== releasesRoot) {
770
+ throw new Error('RUNTIME_PIN_AUTHORITY_PATH_INVALID');
771
+ }
772
+ return parsed;
773
+ }
774
+
775
+ function releaseEvidenceForPin(
776
+ config: RecoveryConfig,
777
+ candidate: { manifest: RuntimeReleaseManifest; manifestPath: string },
778
+ ): ReleaseEvidence {
779
+ const releasesRoot = resolve(config.controllerHome, 'runtime', 'releases');
780
+ const releaseRoot = dirname(resolve(candidate.manifestPath));
781
+ if (!existsSync(releasesRoot) || dirname(realpathSync(releaseRoot)) !== realpathSync(releasesRoot)) {
782
+ throw new Error('RUNTIME_PIN_RELEASE_OUTSIDE_CONTROLLER_RUNTIME_ROOT');
783
+ }
784
+ const expectedRepositoryId = config.primaryRuntimeSourceRepositoryId?.trim();
785
+ if (expectedRepositoryId && candidate.manifest.sourceRepositoryId !== expectedRepositoryId) {
786
+ throw new Error(`RUNTIME_PIN_SOURCE_REPOSITORY_MISMATCH: expected ${expectedRepositoryId}, got ${candidate.manifest.sourceRepositoryId ?? 'missing'}`);
787
+ }
788
+ return {
789
+ path: candidate.manifestPath,
790
+ revision: candidate.manifest.releaseId,
791
+ artifactIdentity: candidate.manifest.artifactIdentity,
792
+ manifestSha256: createHash('sha256').update(readFileSync(candidate.manifestPath)).digest('hex'),
793
+ workerProtocolVersion: candidate.manifest.workerProtocolVersion,
794
+ controllerHome: resolve(config.controllerHome),
795
+ ...(candidate.manifest.sourceRepositoryId ? { sourceRepositoryId: candidate.manifest.sourceRepositoryId } : {}),
796
+ pinnedAt: new Date().toISOString(),
797
+ };
798
+ }
799
+
800
+ export async function pinRuntimeRelease(config: RecoveryConfig, candidateManifestPath: string, requestId?: string): Promise<Record<string, unknown>> {
801
+ let candidate: { manifest: RuntimeReleaseManifest; releaseRoot: string; manifestPath: string };
802
+ try { candidate = validateRuntimeReleaseCandidate(config, candidateManifestPath); }
803
+ catch (error) { return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) }; }
804
+ if (candidate.manifest.deploymentScope === 'portable') {
805
+ return {
806
+ ok: false,
807
+ attempted: false,
808
+ noOp: true,
809
+ detail: 'RUNTIME_PIN_PORTABLE_RELEASE_FORBIDDEN: portable source candidates require a verified Recovery ReleaseSession',
810
+ };
811
+ }
812
+ let evidence: ReleaseEvidence;
813
+ try { evidence = releaseEvidenceForPin(config, candidate); }
814
+ catch (error) { return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) }; }
815
+ const locked = await withLock(config, { action: 'pin_runtime_release', requestId }, async () => {
816
+ const current = runtimePin(config);
817
+ if (current?.release.revision === evidence.revision && current.release.manifestSha256 === evidence.manifestSha256) {
818
+ return { ok: true, attempted: false, noOp: true, detail: 'requested Runtime release is already pinned', pinned: current.release };
819
+ }
820
+ const store: RuntimePinStore = { schemaVersion: 1, release: evidence, updatedAt: new Date().toISOString() };
821
+ writeJson(runtimePinPath(config), store);
822
+ audit(config, 'runtime_release_pinned', { requestId, revision: evidence.revision, artifactIdentity: evidence.artifactIdentity });
823
+ return { ok: true, attempted: true, detail: 'Runtime release pinned for retention and explicit Runtime-only activation', pinned: evidence };
824
+ });
825
+ return locked.acquired ? locked.value : { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
826
+ }
827
+
828
+ export async function unpinRuntimeRelease(config: RecoveryConfig, requestId?: string): Promise<Record<string, unknown>> {
829
+ const locked = await withLock(config, { action: 'unpin_runtime_release', requestId }, async () => {
830
+ const current = runtimePin(config);
831
+ if (!current) return { ok: true, attempted: false, noOp: true, detail: 'no Runtime release is pinned' };
832
+ rmSync(runtimePinPath(config), { force: true });
833
+ audit(config, 'runtime_release_unpinned', { requestId, revision: current.release.revision, artifactIdentity: current.release.artifactIdentity });
834
+ return { ok: true, attempted: true, detail: 'Runtime release pin removed', unpinned: current.release };
835
+ });
836
+ return locked.acquired ? locked.value : { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
580
837
  }
581
838
 
582
839
 
@@ -586,6 +843,75 @@ function audit(config: RecoveryConfig, event: string, detail: Record<string, unk
586
843
  writeFileSync(auditPath(config), `${line}\n`, { encoding: 'utf8', mode: 0o600, flag: 'a' });
587
844
  }
588
845
 
846
+ export interface RecoveryHistoryEvent {
847
+ at: string;
848
+ event: string;
849
+ ok?: boolean;
850
+ }
851
+
852
+ const RECOVERY_HISTORY_MAX_EVENTS = 50;
853
+ const RECOVERY_HISTORY_MAX_BYTES = 256 * 1024;
854
+
855
+ /**
856
+ * Bounded read-only projection of the canonical Recovery audit. The projection
857
+ * intentionally omits arbitrary detail payloads, filesystem paths, request ids,
858
+ * service targets, and other mechanics that the desktop does not need.
859
+ */
860
+ export function listRecoveryHistory(
861
+ config: RecoveryConfig,
862
+ requestedLimit = 20,
863
+ ): { events: RecoveryHistoryEvent[]; bounded: true; truncated: boolean } {
864
+ const limit = Number.isFinite(requestedLimit)
865
+ ? Math.max(1, Math.min(Math.trunc(requestedLimit), RECOVERY_HISTORY_MAX_EVENTS))
866
+ : 20;
867
+ const path = auditPath(config);
868
+ if (!existsSync(path)) return { events: [], bounded: true, truncated: false };
869
+
870
+ const size = statSync(path).size;
871
+ if (size <= 0) return { events: [], bounded: true, truncated: false };
872
+ const start = Math.max(0, size - RECOVERY_HISTORY_MAX_BYTES);
873
+ const length = size - start;
874
+ const buffer = Buffer.alloc(length);
875
+ const fd = openSync(path, 'r');
876
+ let read = 0;
877
+ try {
878
+ while (read < length) {
879
+ const count = readSync(fd, buffer, read, length - read, start + read);
880
+ if (count <= 0) break;
881
+ read += count;
882
+ }
883
+ } finally {
884
+ closeSync(fd);
885
+ }
886
+
887
+ let text = buffer.subarray(0, read).toString('utf8');
888
+ if (start > 0) {
889
+ const firstNewline = text.indexOf('\n');
890
+ text = firstNewline >= 0 ? text.slice(firstNewline + 1) : '';
891
+ }
892
+ const parsed: RecoveryHistoryEvent[] = [];
893
+ for (const line of text.split('\n')) {
894
+ const trimmed = line.trim();
895
+ if (!trimmed) continue;
896
+ try {
897
+ const value = JSON.parse(trimmed) as { at?: unknown; event?: unknown; detail?: unknown };
898
+ if (typeof value.at !== 'string' || typeof value.event !== 'string') continue;
899
+ const detail = value.detail && typeof value.detail === 'object' && !Array.isArray(value.detail)
900
+ ? value.detail as Record<string, unknown>
901
+ : undefined;
902
+ parsed.push({
903
+ at: value.at,
904
+ event: value.event,
905
+ ...(typeof detail?.ok === 'boolean' ? { ok: detail.ok } : {}),
906
+ });
907
+ } catch {
908
+ // A partially written final line is not a durable audit event yet.
909
+ }
910
+ }
911
+ const truncated = start > 0 || parsed.length > limit;
912
+ return { events: parsed.slice(-limit).reverse(), bounded: true, truncated };
913
+ }
914
+
589
915
  const WATCHDOG_DIAGNOSTIC_LIMIT = 32;
590
916
 
591
917
  function watchdogProbeComponent(name: string): RecoveryWatchdogDiagnosticEvidence['components'][number] {
@@ -649,7 +975,7 @@ function recoveryLockOwnerAlive(lock: RecoveryLock): boolean {
649
975
  if (!pidAlive(lock.pid)) return false;
650
976
  if (!lock.processStartTime) return true;
651
977
  const observed = processStartTime(lock.pid);
652
- return observed === undefined || observed === lock.processStartTime;
978
+ return observed === lock.processStartTime;
653
979
  }
654
980
 
655
981
  function recoveryLockOwnerAttributable(owner: RecoveryLock): boolean {
@@ -907,7 +1233,16 @@ function canonicalRuntimeSafeForTargetedConnectorRecovery(verified: VerifyResult
907
1233
  }
908
1234
 
909
1235
  function mainToken(config: RecoveryConfig): string | undefined {
910
- const candidate = config.mainMcpTokenFile ?? join(config.controllerHome, 'mcp', 'mcp.tokens.json');
1236
+ const explicit = config.mainMcpTokenFile?.trim();
1237
+ const candidate = explicit || join(config.controllerHome, 'mcp', 'mcp.tokens.json');
1238
+ if (explicit) {
1239
+ try {
1240
+ const raw = readFileSync(candidate, 'utf8').trim();
1241
+ if (raw.length >= 24 && !raw.startsWith('{')) return raw;
1242
+ } catch {
1243
+ return undefined;
1244
+ }
1245
+ }
911
1246
  const parsed = json<{ bearerToken?: unknown }>(candidate);
912
1247
  return typeof parsed?.bearerToken === 'string' && parsed.bearerToken.length >= 24 ? parsed.bearerToken : undefined;
913
1248
  }
@@ -1003,6 +1338,20 @@ function runtimeHealthEndpoint(endpoint: string): string {
1003
1338
  return url.toString();
1004
1339
  }
1005
1340
 
1341
+ /**
1342
+ * tunnel-client materialises its own state directory relative to the process
1343
+ * working directory. Persistent Forge services run under launchd/systemd with a
1344
+ * read-only working directory, so a Forge-owned tunnel reconnect must run from a
1345
+ * writable Forge-managed directory. Otherwise the reconnect fails with
1346
+ * `mkdir .tunnel-client: read-only file system`, the dedicated Recovery
1347
+ * transport can never be repaired, and the Recovery connector stays dark.
1348
+ */
1349
+ function tunnelClientWorkingDirectory(controllerHome: string): string {
1350
+ const directory = join(resolve(controllerHome), 'tmp', 'tunnel-client');
1351
+ mkdirSync(directory, { recursive: true, mode: 0o700 });
1352
+ return directory;
1353
+ }
1354
+
1006
1355
  async function observeOpenAiTunnelRuntime(
1007
1356
  configured: OpenAiSecureTunnelServiceConfig,
1008
1357
  runCommand: CommandRunner = command,
@@ -1013,6 +1362,8 @@ async function observeOpenAiTunnelRuntime(
1013
1362
  } catch (error) {
1014
1363
  return {
1015
1364
  ok: false, running: false, healthy: false, ready: false, tunnelMatches: false, endpointMatches: false,
1365
+ controlPlaneState: 'unknown',
1366
+ controlPlaneDetail: 'tunnel runtime configuration could not be observed',
1016
1367
  alias: configured.alias, tunnelId: configured.tunnelId,
1017
1368
  detail: error instanceof Error ? error.message : 'OpenAI tunnel runtime configuration is invalid',
1018
1369
  };
@@ -1023,6 +1374,8 @@ async function observeOpenAiTunnelRuntime(
1023
1374
  if (local) return local;
1024
1375
  return {
1025
1376
  ok: false, running: false, healthy: false, ready: false, tunnelMatches: false, endpointMatches: false,
1377
+ controlPlaneState: 'unknown',
1378
+ controlPlaneDetail: 'tunnel-client status produced no remote control-plane evidence',
1026
1379
  alias: configured.alias, tunnelId: configured.tunnelId,
1027
1380
  detail: status.stderr.trim() || 'OpenAI tunnel runtime status is unavailable',
1028
1381
  };
@@ -1069,7 +1422,11 @@ export async function observeOpenAiTunnelLocalHealthFallback(
1069
1422
  try { return (await request(url.toString(), { signal: controller.signal })).ok; } catch { return false; } finally { clearTimeout(timer); }
1070
1423
  };
1071
1424
  const [healthy, ready] = await Promise.all([probe('/healthz'), probe('/readyz')]);
1072
- const ok = healthy && ready && identityMatches && endpointMatches;
1425
+ const localReady = healthy && ready && identityMatches && endpointMatches;
1426
+ // Loopback health proves only that the local tunnel process and MCP binding are
1427
+ // alive. It must never stand in for the remote OpenAI control plane when the
1428
+ // status command itself could not establish that path.
1429
+ const ok = false;
1073
1430
  const failures: string[] = [];
1074
1431
  if (!identityMatches) failures.push('profile tunnel id does not match');
1075
1432
  if (!endpointMatches) failures.push('profile MCP endpoint does not match');
@@ -1082,11 +1439,13 @@ export async function observeOpenAiTunnelLocalHealthFallback(
1082
1439
  ready,
1083
1440
  tunnelMatches: identityMatches,
1084
1441
  endpointMatches,
1442
+ controlPlaneState: 'unknown',
1443
+ controlPlaneDetail: 'tunnel-client status was unavailable; only loopback health was observed',
1085
1444
  alias: configured.alias,
1086
1445
  tunnelId: configured.tunnelId,
1087
1446
  ...(identityMatches ? { observedTunnelId: configured.tunnelId } : {}),
1088
- detail: ok
1089
- ? `managed runtime ${configured.alias} is locally healthy and ready for ${configured.tunnelId} (status command unavailable)`
1447
+ detail: localReady
1448
+ ? `managed runtime ${configured.alias} is locally healthy and ready for ${configured.tunnelId}, but the OpenAI control plane is unverified because status is unavailable`
1090
1449
  : failures.join('; ') || 'OpenAI tunnel local health fallback failed',
1091
1450
  profilePath,
1092
1451
  };
@@ -1104,11 +1463,22 @@ async function observeOpenAiRecoveryTunnel(
1104
1463
  async function ensureOpenAiTunnelRuntimeStarted(
1105
1464
  configured: OpenAiSecureTunnelServiceConfig,
1106
1465
  runCommand: CommandRunner = command,
1466
+ workingDirectory?: string,
1107
1467
  ): Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string; serviceLabel: string; serviceTarget: string }> {
1108
1468
  const serviceLabel = configured.alias;
1109
1469
  const serviceTarget = `tunnel-client:${configured.alias}`;
1110
1470
  const observed = await observeOpenAiTunnelRuntime(configured, runCommand);
1111
1471
  if (observed.ok) return { ok: true, attempted: false, noOp: true, detail: observed.detail, serviceLabel, serviceTarget };
1472
+ if (observed.healthy && observed.ready && observed.tunnelMatches && observed.endpointMatches && observed.controlPlaneState !== 'ready') {
1473
+ return {
1474
+ ok: false,
1475
+ attempted: false,
1476
+ noOp: true,
1477
+ detail: `OpenAI Secure MCP Tunnel runtime ${configured.alias} is locally ready but its remote control plane is ${observed.controlPlaneState}; refusing a local reconnect for an upstream transport failure: ${observed.controlPlaneDetail}`,
1478
+ serviceLabel,
1479
+ serviceTarget,
1480
+ };
1481
+ }
1112
1482
  if (observed.observedTunnelId && !observed.tunnelMatches) {
1113
1483
  return { ok: false, attempted: false, noOp: true, detail: `OpenAI tunnel alias ${configured.alias} is already bound to a different tunnel id`, serviceLabel, serviceTarget };
1114
1484
  }
@@ -1129,7 +1499,7 @@ async function ensureOpenAiTunnelRuntimeStarted(
1129
1499
  } catch (error) {
1130
1500
  return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : 'OpenAI tunnel runtime configuration is invalid', serviceLabel, serviceTarget };
1131
1501
  }
1132
- const connected = await runCommand('tunnel-client', args, Math.max(30_000, configured.postRestartVerifyTimeoutMs ?? 20_000));
1502
+ const connected = await runCommand('tunnel-client', args, Math.max(30_000, configured.postRestartVerifyTimeoutMs ?? 20_000), workingDirectory ? { cwd: workingDirectory } : {});
1133
1503
  return connected.ok
1134
1504
  ? { ok: true, attempted: true, detail: `OpenAI Secure MCP Tunnel runtime ${configured.alias} connect dispatched`, serviceLabel, serviceTarget }
1135
1505
  : { ok: false, attempted: true, detail: `OpenAI Secure MCP Tunnel runtime connect failed: ${connected.stderr || connected.stdout || connected.status}`, serviceLabel, serviceTarget };
@@ -1143,6 +1513,140 @@ async function probeOpenAiRecoveryTunnel(
1143
1513
  return observed ? { ok: observed.ok, detail: observed.detail, value: observed } : undefined;
1144
1514
  }
1145
1515
 
1516
+ function openAiTunnelObservationFromProbe(
1517
+ probe: { value?: unknown } | undefined,
1518
+ ): OpenAiSecureTunnelRuntimeObservation | undefined {
1519
+ const value = probe?.value;
1520
+ if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
1521
+ const observed = value as Partial<OpenAiSecureTunnelRuntimeObservation>;
1522
+ if (
1523
+ typeof observed.alias !== 'string'
1524
+ || typeof observed.tunnelId !== 'string'
1525
+ || typeof observed.healthy !== 'boolean'
1526
+ || typeof observed.ready !== 'boolean'
1527
+ || typeof observed.tunnelMatches !== 'boolean'
1528
+ || typeof observed.endpointMatches !== 'boolean'
1529
+ || !['ready', 'degraded', 'unknown'].includes(String(observed.controlPlaneState))
1530
+ ) return undefined;
1531
+ return observed as OpenAiSecureTunnelRuntimeObservation;
1532
+ }
1533
+
1534
+ function isUpstreamOpenAiControlPlaneIncident(observed: OpenAiSecureTunnelRuntimeObservation | undefined): boolean {
1535
+ return Boolean(
1536
+ observed
1537
+ && observed.healthy
1538
+ && observed.ready
1539
+ && observed.tunnelMatches
1540
+ && observed.endpointMatches
1541
+ && observed.controlPlaneState !== 'ready'
1542
+ );
1543
+ }
1544
+
1545
+ async function probeOpenAiApiReachability(): Promise<{ ok: boolean; detail: string; status?: number }> {
1546
+ const controller = new AbortController();
1547
+ const timer = setTimeout(() => controller.abort('RECOVERY_OPENAI_API_REACHABILITY_TIMEOUT'), 4_000);
1548
+ try {
1549
+ const response = await fetch('https://api.openai.com/v1/models', {
1550
+ method: 'GET',
1551
+ headers: { accept: 'application/json' },
1552
+ signal: controller.signal,
1553
+ });
1554
+ return {
1555
+ ok: true,
1556
+ status: response.status,
1557
+ detail: `api.openai.com returned HTTP ${response.status}; DNS/TCP/TLS/HTTP path is reachable`,
1558
+ };
1559
+ } catch (error) {
1560
+ const cause = error instanceof Error && error.cause instanceof Error ? `; cause=${error.cause.message}` : '';
1561
+ return { ok: false, detail: `${error instanceof Error ? error.message : String(error)}${cause}`.slice(0, 500) };
1562
+ } finally {
1563
+ clearTimeout(timer);
1564
+ }
1565
+ }
1566
+
1567
+ async function observeRecentMacNetworkExtensionChanges(
1568
+ runCommand: CommandRunner = command,
1569
+ ): Promise<{ ok: boolean; detail: string } | undefined> {
1570
+ if (process.platform !== 'darwin') return undefined;
1571
+ const result = await runCommand('/usr/bin/log', [
1572
+ 'show', '--last', '90s', '--style', 'compact', '--predicate',
1573
+ 'process == "nesessionmanager" AND (eventMessage CONTAINS[c] "configuration changed" OR eventMessage CONTAINS[c] "neconfigurationchanged")',
1574
+ ], 3_000, { maxOutputBytes: 12_000 });
1575
+ const output = (result.stdout || result.stderr).trim();
1576
+ return {
1577
+ ok: result.ok,
1578
+ detail: output ? output.slice(-4_000) : (result.ok ? 'no recent Network Extension configuration changes observed' : 'Network Extension event query failed'),
1579
+ };
1580
+ }
1581
+
1582
+ async function persistOpenAiTransportIncidentEvidence(input: {
1583
+ config: RecoveryConfig;
1584
+ verify: VerifyResult;
1585
+ runtimeObservation: ReturnType<typeof observeRuntimeStatus>;
1586
+ recoveryTunnel: OpenAiSecureTunnelRuntimeObservation;
1587
+ }): Promise<void> {
1588
+ const primaryTunnel = configuredPrimaryPublicTunnel(input.config);
1589
+ const [openAiApi, connectorActivity, macNetworkExtension, primaryTunnelObservation] = await Promise.all([
1590
+ probeOpenAiApiReachability(),
1591
+ probeReleaseCutoverReadiness(input.config, { timeoutMs: 1_000 }),
1592
+ observeRecentMacNetworkExtensionChanges(),
1593
+ primaryTunnel?.platform === 'openai-secure-tunnel'
1594
+ ? observeOpenAiTunnelRuntime(primaryTunnel)
1595
+ : Promise.resolve(undefined),
1596
+ ]);
1597
+ const activeReleaseSessions = listCompleteReleaseSessions(input.config.controllerHome).sessions
1598
+ .filter((session) => !['known_good', 'rolled_back', 'failed'].includes(session.phase))
1599
+ .sort((left, right) => Date.parse(right.updatedAt) - Date.parse(left.updatedAt))
1600
+ .slice(0, 3)
1601
+ .map((session) => ({ sessionId: session.sessionId, phase: session.phase, revision: session.revision, updatedAt: session.updatedAt }));
1602
+ const incidentDetail = [
1603
+ `alias=${input.recoveryTunnel.alias}`,
1604
+ `tunnel=${input.recoveryTunnel.tunnelId}`,
1605
+ `client=${input.recoveryTunnel.clientInstanceId ?? 'unknown'}`,
1606
+ `controlPlane=${input.recoveryTunnel.controlPlaneState}`,
1607
+ input.recoveryTunnel.controlPlaneDetail,
1608
+ ].join('; ');
1609
+ const diagnostic = persistWatchdogDiagnosticEvidence(input.config, {
1610
+ ...input.verify,
1611
+ ok: false,
1612
+ probes: {
1613
+ recovery_tunnel_runtime: { ok: false, detail: incidentDetail },
1614
+ },
1615
+ });
1616
+ audit(input.config, 'network_transport_incident_observed', {
1617
+ diagnosticFingerprint: diagnostic?.fingerprint,
1618
+ recoveryTunnel: {
1619
+ alias: input.recoveryTunnel.alias,
1620
+ tunnelId: input.recoveryTunnel.tunnelId,
1621
+ clientInstanceId: input.recoveryTunnel.clientInstanceId,
1622
+ controlPlaneState: input.recoveryTunnel.controlPlaneState,
1623
+ controlPlaneDetail: input.recoveryTunnel.controlPlaneDetail,
1624
+ localHealthy: input.recoveryTunnel.healthy,
1625
+ localReady: input.recoveryTunnel.ready,
1626
+ },
1627
+ primaryTunnel: primaryTunnelObservation ? {
1628
+ alias: primaryTunnelObservation.alias,
1629
+ tunnelId: primaryTunnelObservation.tunnelId,
1630
+ clientInstanceId: primaryTunnelObservation.clientInstanceId,
1631
+ controlPlaneState: primaryTunnelObservation.controlPlaneState,
1632
+ controlPlaneDetail: primaryTunnelObservation.controlPlaneDetail,
1633
+ localHealthy: primaryTunnelObservation.healthy,
1634
+ localReady: primaryTunnelObservation.ready,
1635
+ } : undefined,
1636
+ openAiApi,
1637
+ connectorActivity,
1638
+ runtime: {
1639
+ releaseId: input.runtimeObservation.snapshot?.releaseId,
1640
+ runtimeInstanceId: input.runtimeObservation.snapshot?.runtimeInstanceId,
1641
+ pid: input.runtimeObservation.snapshot?.pid,
1642
+ ready: input.runtimeObservation.ready,
1643
+ stale: input.runtimeObservation.stale,
1644
+ },
1645
+ releaseSessions: activeReleaseSessions,
1646
+ macNetworkExtension,
1647
+ });
1648
+ }
1649
+
1146
1650
  async function probeMcp(config: RecoveryConfig, transport: RecoveryHttpTransport): Promise<Record<string, { ok: boolean; detail: string; value?: unknown }>> {
1147
1651
  const url = config.publicMcpUrl ?? runtimeEndpoint(config);
1148
1652
  if (!url) return { mcp_initialize: { ok: false, detail: 'canonical Runtime MCP endpoint is unavailable' } };
@@ -1230,6 +1734,24 @@ export async function verifyStableRuntime(
1230
1734
  : 'canonical Runtime is not running',
1231
1735
  },
1232
1736
  };
1737
+ const knownGoodRecoverability = inspectKnownGoodRecoverability(config);
1738
+ probes.recovery_known_good_recoverability = {
1739
+ ok: knownGoodRecoverability.unavailable.length === 0,
1740
+ detail: knownGoodRecoverability.unavailable.length === 0
1741
+ ? `${knownGoodRecoverability.available.length} bounded known-good Runtime release(s) are physically recoverable`
1742
+ : `${knownGoodRecoverability.unavailable.length} known-good attestation(s) no longer have a matching immutable Runtime release`,
1743
+ value: knownGoodRecoverability,
1744
+ };
1745
+ if (!observation.running || !observation.ready) {
1746
+ const startupFailure = readRuntimeStartupFailureEvidence(config.controllerHome);
1747
+ if (startupFailure && (!active || !startupFailure.releaseId || startupFailure.releaseId === active.revision)) {
1748
+ probes.runtime_startup_failure = {
1749
+ ok: false,
1750
+ detail: `latest Runtime startup failed during ${startupFailure.stage}: ${startupFailure.reasonCode}${startupFailure.message ? ` — ${startupFailure.message}` : ''}`,
1751
+ value: startupFailure,
1752
+ };
1753
+ }
1754
+ }
1233
1755
  const endpoint = observation.snapshot?.endpoint;
1234
1756
  probes.active_gateway = endpoint
1235
1757
  ? await probe(transport, runtimeHealthEndpoint(endpoint))
@@ -1250,7 +1772,10 @@ export async function verifyStableRuntime(
1250
1772
  const primaryConnectorLocal = await probePrimaryConnectorLocal(config, transport);
1251
1773
  if (primaryConnectorLocal) probes.primary_connector_local = primaryConnectorLocal;
1252
1774
  if (config.gateway) probes.recovery_gateway = await probe(transport, `http://${config.gateway.host}:${config.gateway.port}/health`);
1253
- const watchdogHealth = observeRecoveryWatchdogHealth(config.controllerHome);
1775
+ const watchdogHealth = observeRecoveryWatchdogHealth(
1776
+ config.controllerHome,
1777
+ config.installProfile === 'self-healing',
1778
+ );
1254
1779
  probes.recovery_watchdog = {
1255
1780
  ok: watchdogHealth.ok,
1256
1781
  detail: watchdogHealth.detail,
@@ -1267,8 +1792,24 @@ export async function verifyStableRuntime(
1267
1792
  const recoveryTunnelRuntime = await probeOpenAiRecoveryTunnel(config);
1268
1793
  if (recoveryTunnelRuntime) probes.recovery_tunnel_runtime = recoveryTunnelRuntime;
1269
1794
  if (options.probeMcpProtocol !== false) Object.assign(probes, await probeMcp(config, transport));
1795
+ const mcpProtocolProbeNames = ['mcp_initialize', 'mcp_initialized_notification', 'mcp_tools_list', 'mcp_read_only_call', 'mcp_session_close'] as const;
1796
+ const mcpProtocolHealthy = options.probeMcpProtocol !== false
1797
+ && mcpProtocolProbeNames.every((name) => probes[name]?.ok === true);
1270
1798
  const coreChecks = Object.entries(probes)
1271
1799
  .filter(([name]) => !name.startsWith('recovery_'))
1800
+ // The unauthenticated public initialize probe remains diagnostic and is the
1801
+ // authority for dedicated external/tunnel verification. Once the stronger
1802
+ // authenticated MCP session completes end-to-end, that raw reachability
1803
+ // probe cannot independently veto full stable Runtime verification.
1804
+ .filter(([name]) => !(name === 'external_mcp_http' && mcpProtocolHealthy))
1805
+ // The OpenAI tunnel control-plane lookup is likewise diagnostic when the
1806
+ // local tunnel identity/readiness is exact and the stronger public MCP data
1807
+ // plane has already completed end-to-end. Real tunnel/data-plane failures
1808
+ // still remain in coreChecks and fail closed.
1809
+ .filter(([name, entry]) => !(
1810
+ name === 'primary_tunnel_runtime'
1811
+ && isPrimaryTunnelControlPlaneOnlyDegradation(entry, mcpProtocolHealthy)
1812
+ ))
1272
1813
  .every(([, entry]) => entry.ok);
1273
1814
  const runtimeHealthy = observation.running && observation.ready && !observation.stale;
1274
1815
  const coherent = Boolean(
@@ -1309,6 +1850,139 @@ export async function verifyStableRuntime(
1309
1850
  });
1310
1851
  return result;
1311
1852
  }
1853
+ function recoveryBundleId(): string {
1854
+ return `attestation-${Date.now()}-${randomUUID().replaceAll('-', '').slice(0, 16)}`;
1855
+ }
1856
+
1857
+ /**
1858
+ * Build a self-contained, Recovery-owned restore point before publishing its
1859
+ * attestation. The copy is intentionally made while the full Runtime is
1860
+ * still verified and fenced; a later crash cannot turn a metadata-only record
1861
+ * into a false known-good authority.
1862
+ */
1863
+ function createKnownGoodRecoveryBundle(config: RecoveryConfig): KnownGoodRecoveryBundle {
1864
+ const attestationId = recoveryBundleId();
1865
+ const root = join(knownGoodRecoveryBundleRoot(config.controllerHome), attestationId);
1866
+ const databasePath = join(root, 'controller.sqlite');
1867
+ const serviceContractPath = join(root, 'service-contract.json');
1868
+ try {
1869
+ const database = backupControlPlaneDatabase(config.controllerHome, databasePath);
1870
+ const servicePaths = forgeRuntimeServicePaths(config.controllerHome);
1871
+ const serviceConfig = readForgeRuntimeServiceConfig(servicePaths.configPath);
1872
+ const serviceContract = {
1873
+ schemaVersion: 1 as const,
1874
+ configPath: servicePaths.configPath,
1875
+ serviceConfig,
1876
+ // The label and fixed executable path are derived, never copied from a
1877
+ // mutable launchd plist. Recovery can rebuild that contract from this
1878
+ // declarative service configuration plus the immutable release.
1879
+ label: servicePaths.label,
1880
+ activeEntrypointPath: servicePaths.activeEntrypointPath,
1881
+ createdAt: new Date().toISOString(),
1882
+ };
1883
+ writeJson(serviceContractPath, serviceContract);
1884
+ return {
1885
+ schemaVersion: 1,
1886
+ attestationId,
1887
+ root,
1888
+ database: {
1889
+ path: databasePath,
1890
+ sha256: sha256FileBounded(databasePath),
1891
+ schemaVersion: database.schemaVersion,
1892
+ recordCount: database.recordCount,
1893
+ auditEventCount: database.auditEventCount,
1894
+ },
1895
+ serviceContract: { path: serviceContractPath, sha256: sha256FileBounded(serviceContractPath) },
1896
+ createdAt: new Date().toISOString(),
1897
+ };
1898
+ } catch (error) {
1899
+ rmSync(root, { recursive: true, force: true });
1900
+ throw error;
1901
+ }
1902
+ }
1903
+
1904
+ function pruneRetiredKnownGoodRecoveryBundles(config: RecoveryConfig, releases: readonly ReleaseEvidence[]): void {
1905
+ const root = knownGoodRecoveryBundleRoot(config.controllerHome);
1906
+ if (!existsSync(root)) return;
1907
+ const retained = new Set(releases.flatMap((release) => release.recoveryBundle ? [resolve(release.recoveryBundle.root)] : []));
1908
+ for (const entry of readdirSync(root, { withFileTypes: true })) {
1909
+ if (!entry.isDirectory()) continue;
1910
+ const path = resolve(root, entry.name);
1911
+ if (dirname(path) !== resolve(root) || retained.has(path)) continue;
1912
+ rmSync(path, { recursive: true, force: true });
1913
+ }
1914
+ }
1915
+
1916
+ /**
1917
+ * Reconcile Recovery's restore-point ledger with the only releases that the
1918
+ * committed Runtime authority can actually roll back to. Historical
1919
+ * attestations are evidence, not a second rollback authority, and must not
1920
+ * keep full SQLite bundles and immutable Runtime trees alive indefinitely.
1921
+ */
1922
+ function reconcileKnownGoodRecoveryRetention(config: RecoveryConfig): void {
1923
+ const store = knownGood(config);
1924
+ if (store.schemaVersion !== 2 || store.releases.length === 0) return;
1925
+ const authority = releaseAuthority(config);
1926
+ if (!authority) throw new Error('KNOWN_GOOD_RETENTION_RUNTIME_AUTHORITY_MISSING');
1927
+ const rollbackReleases = [
1928
+ releaseEvidence(config.controllerHome, authority.active, authority),
1929
+ ...(authority.previous ? [releaseEvidence(config.controllerHome, authority.previous, authority)] : []),
1930
+ ].filter((release): release is ReleaseEvidence => Boolean(release));
1931
+ const retained: ReleaseEvidence[] = [];
1932
+ for (const entry of store.releases) {
1933
+ const rollbackEntry = rollbackReleases.find((release) => sameReleaseIdentity(release, entry));
1934
+ if (!rollbackEntry) continue;
1935
+ try { inspectKnownGoodRecoveryBundle(config.controllerHome, entry); }
1936
+ catch (error) {
1937
+ throw new Error(`KNOWN_GOOD_RETENTION_AUTHORITY_INVALID: ${error instanceof Error ? error.message : String(error)}`);
1938
+ }
1939
+ retained.push(entry);
1940
+ }
1941
+ if (retained.length === store.releases.length) return;
1942
+ writeJson(statePath(config), { schemaVersion: 2, releases: retained, updatedAt: new Date().toISOString() } satisfies KnownGoodStore);
1943
+ pruneRetiredKnownGoodRecoveryBundles(config, retained);
1944
+ }
1945
+
1946
+ interface ReleaseSessionStorageEstimate {
1947
+ requiredBytes: number;
1948
+ reserveBytes: number;
1949
+ }
1950
+
1951
+ function estimateReleaseSessionCandidateBytes(config: RecoveryConfig, manifestPath: string): ReleaseSessionStorageEstimate {
1952
+ const releaseRoot = dirname(resolve(manifestPath));
1953
+ const releaseMeasurement = measureReclaimablePath(releaseRoot, 100_000);
1954
+ const sourceRoot = config.primaryRuntimeSourceRoot?.trim();
1955
+ if (!sourceRoot) throw new Error('RELEASE_SESSION_STORAGE_SOURCE_UNAVAILABLE');
1956
+ const resolvedSourceRoot = resolve(sourceRoot);
1957
+ const sourceMeasurement = measureReclaimablePath(resolvedSourceRoot, 100_000);
1958
+ const dependencyRoot = join(resolvedSourceRoot, 'node_modules');
1959
+ const dependencyMeasurement = existsSync(dependencyRoot)
1960
+ ? measureReclaimablePath(realpathSync(dependencyRoot), 1_000_000)
1961
+ : { bytes: 0, entries: 0, complete: true };
1962
+ const databasePath = join(resolve(config.controllerHome), 'control-plane.sqlite');
1963
+ let databaseBytes: number;
1964
+ try { databaseBytes = Math.max(0, statSync(databasePath).size); }
1965
+ catch (error) { throw new Error(`RELEASE_SESSION_STORAGE_DATABASE_UNAVAILABLE: ${error instanceof Error ? error.message : String(error)}`); }
1966
+ if (!releaseMeasurement.complete) throw new Error('RELEASE_SESSION_STORAGE_RELEASE_ESTIMATE_INCOMPLETE');
1967
+ if (!sourceMeasurement.complete) throw new Error('RELEASE_SESSION_STORAGE_SOURCE_ESTIMATE_INCOMPLETE');
1968
+ if (!dependencyMeasurement.complete) throw new Error('RELEASE_SESSION_STORAGE_DEPENDENCY_ESTIMATE_INCOMPLETE');
1969
+ // The candidate is rebuilt from the frozen checkout, not copied from the
1970
+ // active release. It creates a detached worktree, a dependency-inclusive
1971
+ // package snapshot, compiled artifacts, and a consistent SQLite snapshot.
1972
+ // Account for each independently rather than treating the active tree as a
1973
+ // proxy for the build's incremental footprint.
1974
+ const sourceBytes = releaseMeasurement.bytes
1975
+ + sourceMeasurement.bytes
1976
+ + dependencyMeasurement.bytes
1977
+ + databaseBytes;
1978
+ if (!Number.isFinite(sourceBytes) || sourceBytes < 1) throw new Error('RELEASE_SESSION_STORAGE_ESTIMATE_INVALID');
1979
+ // Keep an equally sized reserve for compiler/materialization variance and
1980
+ // atomic promotion, rather than treating the global warning threshold as
1981
+ // rollback capacity.
1982
+ const measuredBytes = Math.max(1, Math.ceil(sourceBytes));
1983
+ return { requiredBytes: measuredBytes, reserveBytes: measuredBytes };
1984
+ }
1985
+
1312
1986
  /** Explicitly records evidence only after the full independent verification passed. */
1313
1987
  function persistVerifiedKnownGood(config: RecoveryConfig, verified: VerifyResult, performance: RuntimePerformanceEvidence): ReleaseEvidence {
1314
1988
  const authority = releaseAuthority(config);
@@ -1332,13 +2006,19 @@ function persistVerifiedKnownGood(config: RecoveryConfig, verified: VerifyResult
1332
2006
  releaseAuthorityRevision: authority.revision,
1333
2007
  releaseFencingTokenSha256: createHash('sha256').update(authority.fencingToken).digest('hex'),
1334
2008
  attestedAt: new Date().toISOString(),
2009
+ recoveryBundle: createKnownGoodRecoveryBundle(config),
1335
2010
  };
1336
2011
  const store = knownGood(config);
2012
+ const previous = authority.previous ? releaseEvidence(config.controllerHome, authority.previous, authority) : undefined;
1337
2013
  const releases = [
1338
2014
  attested,
1339
- ...store.releases.filter((entry) => entry.path !== active.path && existsSync(entry.path)),
1340
- ].slice(0, 8);
1341
- writeJson(statePath(config), { schemaVersion: 1, releases, updatedAt: new Date().toISOString() } satisfies KnownGoodStore);
2015
+ ...(previous ? store.releases.filter((entry) => sameReleaseIdentity(entry, previous)) : []),
2016
+ ];
2017
+ writeJson(statePath(config), { schemaVersion: 2, releases, updatedAt: new Date().toISOString() } satisfies KnownGoodStore);
2018
+ // State is durable before cleanup. A crash between these lines leaves an
2019
+ // orphaned bundle, which this same owner removes on the next attestation;
2020
+ // it never invalidates a published recovery point.
2021
+ pruneRetiredKnownGoodRecoveryBundles(config, releases);
1342
2022
  audit(config, 'known_good_attested', {
1343
2023
  revision: active.revision,
1344
2024
  artifactIdentity: active.artifactIdentity,
@@ -1363,6 +2043,82 @@ function runtimePerformanceIdentity(config: RecoveryConfig): RuntimePerformanceI
1363
2043
  };
1364
2044
  }
1365
2045
 
2046
+ export const RECOVERY_INTERNAL_PERFORMANCE_COMMAND = '__measure-runtime-performance';
2047
+
2048
+ export async function measureConfiguredRuntimePerformance(
2049
+ config: RecoveryConfig,
2050
+ dependencies: RuntimePerformanceDependencies = {},
2051
+ ): Promise<RuntimePerformanceEvidence> {
2052
+ return await measureRuntimePerformance(() => runtimePerformanceIdentity(config), dependencies);
2053
+ }
2054
+
2055
+ function hasRuntimePerformanceTestSeam(dependencies: RuntimePerformanceDependencies): boolean {
2056
+ return dependencies.readCpu !== undefined
2057
+ || dependencies.monotonicNow !== undefined
2058
+ || dependencies.wallNow !== undefined
2059
+ || dependencies.sleep !== undefined;
2060
+ }
2061
+
2062
+ async function measureRuntimePerformanceIsolated(
2063
+ config: RecoveryConfig,
2064
+ expectedIdentity: RuntimePerformanceIdentity,
2065
+ ): Promise<RuntimePerformanceEvidence> {
2066
+ const recoveryBefore = readCurrentRecoveryRelease(config.controllerHome);
2067
+ if (!recoveryBefore) throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: current immutable Recovery release unavailable');
2068
+ const executable = join(recoveryBefore.releasePath, 'forge-recovery');
2069
+ if (!existsSync(executable)) throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: immutable Recovery sampler executable unavailable');
2070
+
2071
+ const result = await runBoundedChild(
2072
+ executable,
2073
+ [RECOVERY_INTERNAL_PERFORMANCE_COMMAND, '--controller-home', config.controllerHome],
2074
+ {
2075
+ timeoutMs: 100_000,
2076
+ maxOutputBytes: 32 * 1024,
2077
+ forwardSignals: false,
2078
+ env: runtimeAuthorityFreeEnvironment(process.env),
2079
+ },
2080
+ );
2081
+ if (result.status !== 0 || result.failureCode || result.timedOut) {
2082
+ const detail = result.failureCode ?? result.error ?? (result.stderr.trim() || `exit=${result.status}`);
2083
+ throw new Error(`RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler failed (${detail.slice(0, 240)})`);
2084
+ }
2085
+
2086
+ let envelope: unknown;
2087
+ try {
2088
+ envelope = JSON.parse(result.stdout);
2089
+ } catch {
2090
+ throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler returned malformed evidence');
2091
+ }
2092
+ if (!envelope || typeof envelope !== 'object' || Array.isArray(envelope)) {
2093
+ throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler returned invalid evidence envelope');
2094
+ }
2095
+ const parsed = envelope as { schemaVersion?: unknown; ok?: unknown; evidence?: unknown; error?: unknown };
2096
+ if (parsed.schemaVersion !== 1 || typeof parsed.ok !== 'boolean') {
2097
+ throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler returned invalid evidence envelope');
2098
+ }
2099
+ if (!parsed.ok) {
2100
+ const detail = typeof parsed.error === 'string' ? parsed.error : '';
2101
+ if (/^RECOVERY_PERFORMANCE_(?:UNKNOWN|REJECTED):/.test(detail)) throw new Error(detail);
2102
+ throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler returned an unclassified failure');
2103
+ }
2104
+ if (!parsed.evidence || typeof parsed.evidence !== 'object' || Array.isArray(parsed.evidence)) {
2105
+ throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler omitted performance evidence');
2106
+ }
2107
+ const performance = parsed.evidence as RuntimePerformanceEvidence;
2108
+ assertRuntimePerformanceEvidence(performance, expectedIdentity);
2109
+
2110
+ const recoveryAfter = readCurrentRecoveryRelease(config.controllerHome);
2111
+ if (!recoveryAfter
2112
+ || recoveryAfter.releaseRevision !== recoveryBefore.releaseRevision
2113
+ || recoveryAfter.manifestSha256 !== recoveryBefore.manifestSha256) {
2114
+ throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: Recovery release changed during performance observation');
2115
+ }
2116
+ if (!samePerformanceIdentity(expectedIdentity, runtimePerformanceIdentity(config))) {
2117
+ throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: Runtime or release authority changed after performance observation');
2118
+ }
2119
+ return performance;
2120
+ }
2121
+
1366
2122
  export async function attestKnownGood(
1367
2123
  config: RecoveryConfig,
1368
2124
  dependencies: RuntimePerformanceDependencies = {},
@@ -1376,17 +2132,20 @@ export async function attestKnownGood(
1376
2132
  throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: Runtime or release authority changed during functional verification');
1377
2133
  }
1378
2134
 
1379
- // Performance observation is read-only and deliberately stays outside the
1380
- // single Recovery mutation lock. Runtime/release identity is fenced on every
1381
- // sample; a concurrent restart/rollback/activation therefore invalidates the
1382
- // evidence instead of being blocked for the six-minute observation window.
1383
- const performance = await measureRuntimePerformance(() => {
1384
- const current = runtimePerformanceIdentity(config);
1385
- if (!samePerformanceIdentity(verifiedIdentity, current)) {
1386
- throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: Runtime or release authority changed after functional verification');
1387
- }
1388
- return current;
1389
- }, dependencies);
2135
+ // Production observation executes in one short-lived child of the exact
2136
+ // immutable Recovery release so Gateway/watchdog work in the persistent
2137
+ // daemon cannot stretch the sampler's monotonic windows. Explicit dependency
2138
+ // hooks remain the in-process test seam only. Both paths stay read-only and
2139
+ // outside the Recovery mutation lock; exact Runtime identity remains fenced.
2140
+ const performance = hasRuntimePerformanceTestSeam(dependencies)
2141
+ ? await measureRuntimePerformance(() => {
2142
+ const current = runtimePerformanceIdentity(config);
2143
+ if (!samePerformanceIdentity(verifiedIdentity, current)) {
2144
+ throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: Runtime or release authority changed after functional verification');
2145
+ }
2146
+ return current;
2147
+ }, dependencies)
2148
+ : await measureRuntimePerformanceIsolated(config, verifiedIdentity);
1390
2149
 
1391
2150
  const locked = await withLock(config, { action: 'attest_known_good' }, async () => {
1392
2151
  const verified = await verifyStableRuntime(config);
@@ -1442,7 +2201,8 @@ async function rollbackPreviousLocked(config: RecoveryConfig, reason: string): P
1442
2201
  }
1443
2202
  const operationId = `recovery-rollback-${Date.now()}-${randomUUID().slice(0, 8)}`;
1444
2203
  try {
1445
- const committed = rollbackRuntimeRelease(config.controllerHome, operationId);
2204
+ const rollbackResult = rollbackRuntimeReleaseWithResult(config.controllerHome, operationId);
2205
+ const committed = rollbackResult.authority;
1446
2206
  if (
1447
2207
  committed.active.releaseId !== target.revision
1448
2208
  || committed.active.artifactIdentity !== target.artifactIdentity
@@ -1455,11 +2215,16 @@ async function rollbackPreviousLocked(config: RecoveryConfig, reason: string): P
1455
2215
  activeRevision: active.revision,
1456
2216
  restoredRevision: target.revision,
1457
2217
  restoredManifestSha256: target.manifestSha256,
2218
+ databaseDisposition: rollbackResult.databaseDisposition,
2219
+ liveAuditEventCount: rollbackResult.liveAuditEventCount,
2220
+ rollbackAuditEventCount: rollbackResult.rollbackAuditEventCount,
1458
2221
  });
1459
2222
  return {
1460
2223
  ok: true,
1461
2224
  operationId,
1462
- detail: 'whole-Runtime release and SQLite backup restored; Canonical Runtime remains stopped until its sole launcher starts it',
2225
+ detail: rollbackResult.databaseDisposition === 'restored_backup'
2226
+ ? 'whole-Runtime release and unchanged SQLite backup restored; Canonical Runtime remains stopped until its sole launcher starts it'
2227
+ : 'whole-Runtime release restored while newer live SQLite state was preserved; Canonical Runtime remains stopped until its sole launcher starts it',
1463
2228
  };
1464
2229
  } catch (error) {
1465
2230
  const detail = error instanceof Error ? error.message : 'whole-Runtime rollback failed';
@@ -1508,65 +2273,226 @@ async function verifyLocalRuntime(
1508
2273
  }, createRecoveryHttpTransport(config.controllerHome), options);
1509
2274
  }
1510
2275
 
1511
- function isExternalTunnelFailure(config: RecoveryConfig, verified: VerifyResult, localVerify: VerifyResult): boolean {
1512
- const configured = configuredRecoveryTunnel(config);
1513
- const localRecoveryGatewayHealthy = localVerify.probes.recovery_gateway?.ok ?? true;
1514
- if (!configured || !localVerify.ok || !localRecoveryGatewayHealthy) return false;
1515
- if (configured.platform === 'openai-secure-tunnel') return verified.probes.recovery_tunnel_runtime?.ok !== true;
1516
- const external = verified.probes.recovery_external_http ?? verified.probes.external_mcp_http;
1517
- return Boolean(configuredRecoveryPublicUrl(config) && external?.ok !== true);
1518
- }
1519
-
1520
- export const WATCHDOG_RUNTIME_STARTUP_GRACE_MS = 60_000;
1521
- export const WATCHDOG_RUNTIME_RESTART_BUDGET_STABLE_MS = 5 * 60_000;
1522
-
1523
- export function watchdogRuntimeStartupGraceMs(
1524
- config: Pick<RecoveryConfig, 'primaryRuntimeService'>,
1525
- ): number {
1526
- const configuredVerifyTimeout = config.primaryRuntimeService?.postRestartVerifyTimeoutMs;
1527
- return Math.max(
1528
- WATCHDOG_RUNTIME_STARTUP_GRACE_MS,
1529
- Number.isFinite(configuredVerifyTimeout) ? Math.max(0, configuredVerifyTimeout ?? 0) : 0,
1530
- );
1531
- }
1532
-
1533
- export function runtimeWithinWatchdogStartupGrace(
1534
- input: { running: boolean; stale: boolean; snapshot?: { startedAt?: string } },
1535
- nowMs = Date.now(),
1536
- graceMs = WATCHDOG_RUNTIME_STARTUP_GRACE_MS,
1537
- ): boolean {
1538
- const startedAtMs = input.snapshot?.startedAt ? Date.parse(input.snapshot.startedAt) : Number.NaN;
1539
- return Boolean(
1540
- input.running
1541
- && !input.stale
1542
- && Number.isFinite(startedAtMs)
1543
- && nowMs >= startedAtMs
1544
- && nowMs - startedAtMs < Math.max(0, graceMs),
1545
- );
1546
- }
1547
-
1548
- export function runtimeRestartBudgetIdentity(release: Pick<ReleaseEvidence, 'revision' | 'artifactIdentity' | 'manifestSha256'> | undefined): string | undefined {
1549
- if (!release?.revision || !release.artifactIdentity || !release.manifestSha256) return undefined;
1550
- return `${release.revision}:${release.artifactIdentity}:${release.manifestSha256}`;
2276
+ /**
2277
+ * Release cutover must not create fresh sessions on the Connector whose quiet
2278
+ * window fences the activation. The Runtime/Recovery surfaces are sufficient
2279
+ * to verify the frozen Stable A and Candidate B identities; probing the
2280
+ * primary Connector here would refresh its activity timestamp and make the
2281
+ * cutover gate self-starve on every automatic retry.
2282
+ */
2283
+ async function verifyLocalRuntimeForReleaseCutover(config: RecoveryConfig): Promise<VerifyResult> {
2284
+ return verifyLocalRuntime({ ...config, primaryConnectorService: undefined }, { probeMcpProtocol: false });
1551
2285
  }
1552
2286
 
1553
2287
  /**
1554
- * Bind primary Runtime restart accounting to the exact immutable Runtime release.
1555
- * A missing identity is a legacy-state migration: retain its counters and bind
1556
- * them to the currently observed release so a watchdog upgrade cannot mint a
1557
- * fresh restart budget for an already failing Runtime.
2288
+ * Bounded observation owns health, not release verification. Keep this path
2289
+ * deliberately bounded to already-published Runtime authority/status plus local
2290
+ * HTTP transport checks needed for prompt targeted repair. Expensive execution
2291
+ * canaries, known-good bundle inspection, external transport probes and MCP
2292
+ * initialize/list/call stay in verifyStableRuntime/verifyLocalRuntime and run
2293
+ * only on the periodic verification deadline or an explicit strict boundary.
1558
2294
  */
1559
- export function scopeWatchdogStateToRuntimeRelease(
1560
- state: WatchdogState,
1561
- release: Pick<ReleaseEvidence, 'revision' | 'artifactIdentity' | 'manifestSha256'> | undefined,
1562
- ): WatchdogState {
1563
- const identity = runtimeRestartBudgetIdentity(release);
1564
- if (!identity || state.runtimeRestartBudgetIdentity === identity) return state;
1565
- if (!state.runtimeRestartBudgetIdentity) return { ...state, runtimeRestartBudgetIdentity: identity };
1566
- return {
1567
- ...state,
1568
- failures: 0,
1569
- firstFailureAt: undefined,
2295
+ async function observeBoundedRuntimeHealth(
2296
+ config: RecoveryConfig,
2297
+ transport: RecoveryHttpTransport,
2298
+ options: {
2299
+ /**
2300
+ * Primary Connector probing is excluded from repair surfaces that must keep
2301
+ * working while the primary transport is the broken thing.
2302
+ */
2303
+ includePrimaryConnectorLocal?: boolean;
2304
+ /**
2305
+ * Public/Connector transport probes are lightweight health evidence but
2306
+ * create Connector activity. Disable them while a ReleaseSession is waiting
2307
+ * for the cutover quiet window so watchdog observation cannot self-starve it.
2308
+ */
2309
+ includePrimaryTransport?: boolean;
2310
+ } = {},
2311
+ ): Promise<VerifyResult> {
2312
+ const observation = observeRuntimeStatus(config.controllerHome);
2313
+ const authority = releaseAuthority(config);
2314
+ const active = activeAuthorityRelease(config);
2315
+ const previous = previousAuthorityRelease(config);
2316
+ const runtimeHealthy = observation.running && observation.ready && !observation.stale;
2317
+ const coherent = Boolean(
2318
+ authority
2319
+ && active
2320
+ && observation.snapshot?.releaseId === active.revision
2321
+ && observation.snapshot?.artifactIdentity === active.artifactIdentity
2322
+ && authority.active.workerProtocolVersion === active.workerProtocolVersion
2323
+ );
2324
+ const probes: VerifyResult['probes'] = {
2325
+ runtime_status: {
2326
+ ok: observation.running && !observation.stale,
2327
+ detail: observation.running
2328
+ ? observation.stale ? 'canonical Runtime status is stale' : 'canonical Runtime owner is live'
2329
+ : 'canonical Runtime is not running',
2330
+ },
2331
+ };
2332
+ const endpoint = observation.snapshot?.endpoint;
2333
+ probes.active_gateway = endpoint
2334
+ ? await probe(transport, runtimeHealthEndpoint(endpoint))
2335
+ : { ok: false, detail: 'canonical Runtime endpoint is unavailable' };
2336
+ if (options.includePrimaryTransport !== false && config.publicMcpUrl) {
2337
+ probes.external_mcp_http = await probeExternalMcp(transport, config.publicMcpUrl);
2338
+ const connectorReadinessEndpoint = config.primaryConnectorService?.localMcpUrl?.trim() || config.publicMcpUrl;
2339
+ probes.primary_connector_ready = await probePublicGatewayReadiness(transport, connectorReadinessEndpoint, {
2340
+ acceptOAuthChallenge: packageConnectorAuthMode(config.controllerHome) === 'oauth',
2341
+ });
2342
+ }
2343
+ if (options.includePrimaryTransport !== false) {
2344
+ const primaryTunnel = configuredPrimaryPublicTunnel(config);
2345
+ if (primaryTunnel?.platform === 'openai-secure-tunnel') {
2346
+ const observed = await observeOpenAiTunnelRuntime(primaryTunnel);
2347
+ probes.primary_tunnel_runtime = { ok: observed.ok, detail: observed.detail, value: observed };
2348
+ }
2349
+ }
2350
+ if (options.includePrimaryConnectorLocal !== false) {
2351
+ const primaryConnectorLocal = await probePrimaryConnectorLocal(config, transport);
2352
+ if (primaryConnectorLocal) probes.primary_connector_local = primaryConnectorLocal;
2353
+ }
2354
+ if (config.gateway) probes.recovery_gateway = await probe(transport, `http://${config.gateway.host}:${config.gateway.port}/health`);
2355
+ const watchdogHealth = observeRecoveryWatchdogHealth(
2356
+ config.controllerHome,
2357
+ config.installProfile === 'self-healing',
2358
+ );
2359
+ probes.recovery_watchdog = {
2360
+ ok: watchdogHealth.ok,
2361
+ detail: watchdogHealth.detail,
2362
+ value: {
2363
+ pulseAgeMs: watchdogHealth.pulseAgeMs,
2364
+ tickAgeMs: watchdogHealth.tickAgeMs,
2365
+ currentReleaseRevision: watchdogHealth.currentReleaseRevision,
2366
+ watchdogReleaseRevision: watchdogHealth.runtimeIdentity?.releaseRevision,
2367
+ watchdogPid: watchdogHealth.runtimeIdentity?.pid,
2368
+ },
2369
+ };
2370
+ // The dedicated Recovery transport is the only channel that can repair a
2371
+ // broken primary transport. It must be observed on every liveness tick, or a
2372
+ // dead Recovery tunnel stays invisible while the watchdog keeps reporting a
2373
+ // healthy system and never reaches the bounded tunnel-repair path.
2374
+ const recoveryTransport = await probeOpenAiRecoveryTunnel(config);
2375
+ if (recoveryTransport) probes.recovery_tunnel_runtime = recoveryTransport;
2376
+ const localChecks = Object.entries(probes)
2377
+ .filter(([name]) => !name.startsWith('recovery_'))
2378
+ .every(([, entry]) => entry.ok);
2379
+ return {
2380
+ ok: Boolean(runtimeHealthy && coherent && localChecks),
2381
+ at: new Date().toISOString(),
2382
+ runtime: {
2383
+ ok: runtimeHealthy,
2384
+ running: observation.running,
2385
+ ready: observation.ready,
2386
+ stale: observation.stale,
2387
+ reasonCodes: [...observation.reasonCodes],
2388
+ },
2389
+ releases: {
2390
+ active,
2391
+ previous,
2392
+ // This tier reports durable attestation identity only. Physical bundle
2393
+ // integrity is deliberately reserved for strict verification boundaries.
2394
+ knownGood: matchingKnownGoodAttestation(config, active),
2395
+ coherent,
2396
+ },
2397
+ probes,
2398
+ };
2399
+ }
2400
+
2401
+ async function observeWatchdogHealthTier(
2402
+ config: RecoveryConfig,
2403
+ transport = createRecoveryHttpTransport(config.controllerHome),
2404
+ ): Promise<VerifyResult> {
2405
+ // The primary Connector quiet window is a cutover fence. Once the same
2406
+ // ReleaseSession is eligible for cutover, Recovery's five-second health
2407
+ // probe must stop opening Connector MCP sessions or it will continuously
2408
+ // refresh the very activity timestamp that the fence is waiting to drain.
2409
+ // An incomplete inventory fails closed by retaining the normal probe.
2410
+ const inventory = listCompleteReleaseSessions(config.controllerHome);
2411
+ const cutoverQuietWindowRequired = inventory.invalidSessionFiles.length === 0
2412
+ && inventory.sessions.some((session) => (
2413
+ session.phase === 'cutover_eligible'
2414
+ || session.phase === 'cutover_attempting'
2415
+ || session.phase === 'cutover_committed'
2416
+ ));
2417
+ const primaryTransportAttributionRequired = Boolean(
2418
+ config.primaryConnectorService || configuredPrimaryPublicTunnel(config),
2419
+ );
2420
+ return observeBoundedRuntimeHealth(config, transport, {
2421
+ includePrimaryConnectorLocal: !cutoverQuietWindowRequired,
2422
+ // A plain public URL is strict-verification evidence, not a five-second
2423
+ // liveness dependency. Probe primary transport on the cheap tier only when
2424
+ // Recovery has a managed Connector/tunnel whose failure attribution can
2425
+ // drive a bounded repair decision.
2426
+ includePrimaryTransport: !cutoverQuietWindowRequired && primaryTransportAttributionRequired,
2427
+ });
2428
+ }
2429
+
2430
+ function isExternalTunnelFailure(config: RecoveryConfig, verified: VerifyResult, localVerify: VerifyResult): boolean {
2431
+ const configured = configuredRecoveryTunnel(config);
2432
+ const localRecoveryGatewayHealthy = localVerify.probes.recovery_gateway?.ok ?? true;
2433
+ if (!configured || !localVerify.ok || !localRecoveryGatewayHealthy) return false;
2434
+ if (configured.platform === 'openai-secure-tunnel') {
2435
+ const observed = openAiTunnelObservationFromProbe(verified.probes.recovery_tunnel_runtime);
2436
+ // A locally healthy tunnel with degraded/unknown OpenAI control-plane reachability
2437
+ // is an upstream transport incident, not evidence that reconnecting the local
2438
+ // tunnel process is a valid repair. Keep it degraded and collect evidence.
2439
+ if (isUpstreamOpenAiControlPlaneIncident(observed)) return false;
2440
+ return verified.probes.recovery_tunnel_runtime?.ok !== true;
2441
+ }
2442
+ const external = verified.probes.recovery_external_http ?? verified.probes.external_mcp_http;
2443
+ return Boolean(configuredRecoveryPublicUrl(config) && external?.ok !== true);
2444
+ }
2445
+
2446
+ export const WATCHDOG_RUNTIME_STARTUP_GRACE_MS = 60_000;
2447
+ export const WATCHDOG_RUNTIME_RESTART_BUDGET_STABLE_MS = 5 * 60_000;
2448
+
2449
+ export function watchdogRuntimeStartupGraceMs(
2450
+ config: Pick<RecoveryConfig, 'primaryRuntimeService'>,
2451
+ ): number {
2452
+ const configuredVerifyTimeout = config.primaryRuntimeService?.postRestartVerifyTimeoutMs;
2453
+ return Math.max(
2454
+ WATCHDOG_RUNTIME_STARTUP_GRACE_MS,
2455
+ Number.isFinite(configuredVerifyTimeout) ? Math.max(0, configuredVerifyTimeout ?? 0) : 0,
2456
+ );
2457
+ }
2458
+
2459
+ export function runtimeWithinWatchdogStartupGrace(
2460
+ input: { running: boolean; stale: boolean; snapshot?: { startedAt?: string } },
2461
+ nowMs = Date.now(),
2462
+ graceMs = WATCHDOG_RUNTIME_STARTUP_GRACE_MS,
2463
+ ): boolean {
2464
+ const startedAtMs = input.snapshot?.startedAt ? Date.parse(input.snapshot.startedAt) : Number.NaN;
2465
+ return Boolean(
2466
+ input.running
2467
+ && !input.stale
2468
+ && Number.isFinite(startedAtMs)
2469
+ && nowMs >= startedAtMs
2470
+ && nowMs - startedAtMs < Math.max(0, graceMs),
2471
+ );
2472
+ }
2473
+
2474
+ export function runtimeRestartBudgetIdentity(release: Pick<ReleaseEvidence, 'revision' | 'artifactIdentity' | 'manifestSha256'> | undefined): string | undefined {
2475
+ if (!release?.revision || !release.artifactIdentity || !release.manifestSha256) return undefined;
2476
+ return `${release.revision}:${release.artifactIdentity}:${release.manifestSha256}`;
2477
+ }
2478
+
2479
+ /**
2480
+ * Bind primary Runtime restart accounting to the exact immutable Runtime release.
2481
+ * A missing identity is a legacy-state migration: retain its counters and bind
2482
+ * them to the currently observed release so a watchdog upgrade cannot mint a
2483
+ * fresh restart budget for an already failing Runtime.
2484
+ */
2485
+ export function scopeWatchdogStateToRuntimeRelease(
2486
+ state: WatchdogState,
2487
+ release: Pick<ReleaseEvidence, 'revision' | 'artifactIdentity' | 'manifestSha256'> | undefined,
2488
+ ): WatchdogState {
2489
+ const identity = runtimeRestartBudgetIdentity(release);
2490
+ if (!identity || state.runtimeRestartBudgetIdentity === identity) return state;
2491
+ if (!state.runtimeRestartBudgetIdentity) return { ...state, runtimeRestartBudgetIdentity: identity };
2492
+ return {
2493
+ ...state,
2494
+ failures: 0,
2495
+ firstFailureAt: undefined,
1570
2496
  rollbackUsed: false,
1571
2497
  runtimeRestartAttempts: 0,
1572
2498
  runtimeRestartFailures: 0,
@@ -1723,22 +2649,26 @@ export function decideWatchdog(input: {
1723
2649
 
1724
2650
  export async function diagnose(config: RecoveryConfig): Promise<Record<string, unknown>> {
1725
2651
  const verified = await verifyStableRuntime(config);
1726
- return { verified, knownGood: knownGood(config), quarantine: json(quarantinePath(config)) ?? { releases: [] } };
2652
+ return { verified, knownGood: inspectKnownGoodRecoverability(config), quarantine: json(quarantinePath(config)) ?? { releases: [] } };
1727
2653
  }
1728
2654
 
1729
2655
  export async function listReleases(config: RecoveryConfig): Promise<Record<string, unknown>> {
1730
2656
  const observation = observeRuntimeStatus(config.controllerHome);
2657
+ const knownGoodRecoverability = inspectKnownGoodRecoverability(config);
1731
2658
  return {
1732
2659
  runtimeRunning: observation.running,
1733
2660
  runtimeReady: observation.ready,
1734
2661
  active: activeAuthorityRelease(config),
1735
2662
  previous: previousAuthorityRelease(config),
1736
- knownGood: knownGood(config).releases,
2663
+ knownGood: knownGoodRecoverability.available,
2664
+ knownGoodUnavailable: knownGoodRecoverability.unavailable,
2665
+ pinned: runtimePin(config)?.release,
1737
2666
  };
1738
2667
  }
1739
2668
 
1740
2669
  interface CommandResult { ok: boolean; status: number | null; stdout: string; stderr: string; }
1741
- type CommandRunner = (commandName: string, args: string[], timeoutMs?: number) => Promise<CommandResult>;
2670
+ interface CommandOptions { cwd?: string; home?: string; maxOutputBytes?: number; }
2671
+ type CommandRunner = (commandName: string, args: string[], timeoutMs?: number, options?: CommandOptions) => Promise<CommandResult>;
1742
2672
  interface LaunchdService { uid: number; domain: string; target: string; label: string; plistPath: string; }
1743
2673
 
1744
2674
  /**
@@ -1758,6 +2688,28 @@ export function recoveryCommandPath(
1758
2688
  return entries.includes(userBin) ? inheritedPath : [userBin, ...entries].join(delimiter);
1759
2689
  }
1760
2690
 
2691
+ /**
2692
+ * Child environment for Recovery-owned commands.
2693
+ *
2694
+ * Keep the hosting Runtime's private FORGE_* authority out, but preserve the
2695
+ * account context: user-installed tools resolve their own per-account state from
2696
+ * HOME, and a persistent launchd/systemd service starts without an interactive
2697
+ * HOME to inherit. Without it, `tunnel-client` cannot see its own alias registry,
2698
+ * so Recovery both misreads a healthy dedicated tunnel as stopped and cannot
2699
+ * confirm the tunnel it just reconnected.
2700
+ */
2701
+ export function recoveryCommandEnvironment(
2702
+ env: NodeJS.ProcessEnv = process.env,
2703
+ accountHome = homedir(),
2704
+ ): NodeJS.ProcessEnv {
2705
+ const home = env.HOME?.trim() || accountHome;
2706
+ return {
2707
+ ...runtimeAuthorityFreeEnvironment(env),
2708
+ PATH: recoveryCommandPath(env.PATH ?? '', process.platform, home),
2709
+ HOME: home,
2710
+ };
2711
+ }
2712
+
1761
2713
  export interface PublicTunnelRepairDependencies {
1762
2714
  platform?: NodeJS.Platform;
1763
2715
  currentUid?: () => Promise<number | undefined>;
@@ -1768,14 +2720,14 @@ export interface PublicTunnelRepairDependencies {
1768
2720
  sleep?: (ms: number) => Promise<void>;
1769
2721
  }
1770
2722
 
1771
- function command(commandName: string, args: string[], timeoutMs = 10_000, options: { cwd?: string; maxOutputBytes?: number } = {}): Promise<CommandResult> {
2723
+ function command(commandName: string, args: string[], timeoutMs = 10_000, options: CommandOptions = {}): Promise<CommandResult> {
1772
2724
  return new Promise((resolveCommand) => {
1773
2725
  const child = spawn(commandName, args, {
1774
2726
  shell: false,
1775
2727
  stdio: ['ignore', 'pipe', 'pipe'],
1776
2728
  windowsHide: true,
1777
2729
  cwd: options.cwd,
1778
- env: { ...process.env, PATH: recoveryCommandPath() },
2730
+ env: recoveryCommandEnvironment(process.env, options.home),
1779
2731
  });
1780
2732
  const stdout: Buffer[] = [];
1781
2733
  const stderr: Buffer[] = [];
@@ -1853,10 +2805,14 @@ async function ensureLaunchdServiceStarted(service: LaunchdService, runCommand:
1853
2805
  }
1854
2806
 
1855
2807
  export interface PrimaryRuntimeRecoveryDependencies {
2808
+ /** Explicit operator restart may recycle an already-healthy Runtime; watchdog/internal recovery keeps healthy no-op semantics. */
2809
+ forceHealthyRestart?: boolean;
1856
2810
  platform?: NodeJS.Platform;
1857
2811
  currentUid?: () => Promise<number | undefined>;
1858
2812
  runCommand?: CommandRunner;
1859
2813
  verifyLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
2814
+ /** Cheap readiness observation used while a restarted Runtime is converging; strict verification remains a separate acceptance boundary. */
2815
+ observeLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
1860
2816
  /** Rebind the independently supervised OAuth Connector after a whole-Runtime release switch. */
1861
2817
  repairPrimaryConnectorBinding?: (config: RecoveryConfig) => Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string }>;
1862
2818
  runtimeRunning?: (config: RecoveryConfig) => boolean;
@@ -1865,13 +2821,31 @@ export interface PrimaryRuntimeRecoveryDependencies {
1865
2821
  sleep?: (ms: number) => Promise<void>;
1866
2822
  }
1867
2823
 
2824
+ export interface RuntimeReleaseKnownGoodDependencies extends RuntimePerformanceDependencies {
2825
+ /** Test/host seam for the existing whole-Runtime rollback transaction. */
2826
+ rollback?: PrimaryRuntimeRecoveryDependencies;
2827
+ /** Exact active-release live continuation proof; production defaults to the Supervisor single-writer ledger. */
2828
+ continuationProof?: (
2829
+ forgeHome: string,
2830
+ input: { repoId?: string; activeReleaseId: string; notBefore: string },
2831
+ ) => Promise<WorkflowSupervisorContinuationProof | undefined>;
2832
+ }
2833
+
1868
2834
  export interface RuntimeReleaseActivationGuard {
1869
2835
  /** External caller identity for audit/lock attribution. */
1870
2836
  requestId?: string;
2837
+ /** Internal Recovery-only mode: allow explicit activation of current.previous when it is durably pinned. */
2838
+ allowPreviousRelease?: boolean;
2839
+ /** Internal Recovery-only mode: restore Runtime authority without restoring an older SQLite backup on activation failure. */
2840
+ preserveDatabaseOnFailure?: boolean;
2841
+ /** Internal Recovery-only pin fence checked after acquiring the mutation lock. */
2842
+ requiredPinnedReleaseRevision?: string;
1871
2843
  /** Authority snapshot observed when the caller decided to activate; null means the caller observed no authority. */
1872
2844
  expectedAuthorityRevision?: number | null;
1873
2845
  /** Active release observed when the caller decided to activate; null means the caller observed no active release. */
1874
2846
  expectedActiveReleaseId?: string | null;
2847
+ /** Fresh portable source candidates may activate only after this exact Recovery ReleaseSession becomes cutover-eligible. */
2848
+ releaseSessionId?: string;
1875
2849
  }
1876
2850
 
1877
2851
  function configuredPrimaryRuntimeService(config: RecoveryConfig, platform: NodeJS.Platform = process.platform): PrimaryRuntimeServiceConfig {
@@ -2067,6 +3041,103 @@ async function probePrimaryConnectorLocal(
2067
3041
  });
2068
3042
  }
2069
3043
 
3044
+ export interface AutomaticReleaseCutoverReadiness {
3045
+ ready: boolean;
3046
+ activeSessions?: number;
3047
+ activePosts?: number;
3048
+ activeStreams?: number;
3049
+ initializing?: number;
3050
+ latestActivityAgeMs?: number;
3051
+ detail: string;
3052
+ }
3053
+
3054
+ const AUTOMATIC_RELEASE_CUTOVER_QUIET_MS = 30_000;
3055
+ const AUTOMATIC_RELEASE_CUTOVER_HEALTH_TIMEOUT_MS = 2_000;
3056
+
3057
+ function primaryConnectorHealthEndpoint(config: RecoveryConfig): string | undefined {
3058
+ const localMcpUrl = config.primaryConnectorService?.localMcpUrl?.trim();
3059
+ if (!localMcpUrl) return undefined;
3060
+ try {
3061
+ const url = new URL(localMcpUrl);
3062
+ url.pathname = '/health';
3063
+ url.search = '';
3064
+ url.hash = '';
3065
+ return url.toString();
3066
+ } catch {
3067
+ return undefined;
3068
+ }
3069
+ }
3070
+
3071
+ export async function probeReleaseCutoverReadiness(
3072
+ config: RecoveryConfig,
3073
+ options: { transport?: RecoveryHttpTransport; quietMs?: number; timeoutMs?: number } = {},
3074
+ ): Promise<AutomaticReleaseCutoverReadiness> {
3075
+ const healthUrl = primaryConnectorHealthEndpoint(config);
3076
+ if (!healthUrl) return { ready: false, detail: 'primary Connector health endpoint is not configured' };
3077
+ const quietMs = options.quietMs ?? AUTOMATIC_RELEASE_CUTOVER_QUIET_MS;
3078
+ const timeoutMs = options.timeoutMs ?? AUTOMATIC_RELEASE_CUTOVER_HEALTH_TIMEOUT_MS;
3079
+ const transport = options.transport ?? createRecoveryHttpTransport(config.controllerHome);
3080
+ const controller = new AbortController();
3081
+ const timer = setTimeout(() => controller.abort('RECOVERY_AUTOMATIC_RELEASE_CUTOVER_HEALTH_TIMEOUT'), timeoutMs);
3082
+ try {
3083
+ const response = await transport.request({
3084
+ url: healthUrl,
3085
+ headers: { accept: 'application/json' },
3086
+ timeoutMs,
3087
+ signal: controller.signal,
3088
+ });
3089
+ if (!response.ok) return { ready: false, detail: `primary Connector health returned HTTP ${response.status}` };
3090
+ let payload: { sessions?: { active?: unknown; activePosts?: unknown; activeStreams?: unknown; initializing?: unknown; latestActivityAgeMs?: unknown } };
3091
+ try {
3092
+ payload = JSON.parse(response.body) as typeof payload;
3093
+ } catch {
3094
+ return { ready: false, detail: 'primary Connector health returned invalid JSON' };
3095
+ }
3096
+ const activeSessions = Number(payload.sessions?.active);
3097
+ const activePosts = Number(payload.sessions?.activePosts);
3098
+ const activeStreams = Number(payload.sessions?.activeStreams);
3099
+ const initializing = Number(payload.sessions?.initializing);
3100
+ const latestActivityAgeMs = Number(payload.sessions?.latestActivityAgeMs);
3101
+ if (
3102
+ !Number.isFinite(activeSessions)
3103
+ || !Number.isFinite(activePosts)
3104
+ || !Number.isFinite(activeStreams)
3105
+ || !Number.isFinite(initializing)
3106
+ ) {
3107
+ return { ready: false, detail: 'primary Connector health omitted bounded session activity counters' };
3108
+ }
3109
+ const evidence = {
3110
+ activeSessions,
3111
+ activePosts,
3112
+ ...(Number.isFinite(activeStreams) ? { activeStreams } : {}),
3113
+ initializing,
3114
+ ...(Number.isFinite(latestActivityAgeMs) ? { latestActivityAgeMs } : {}),
3115
+ };
3116
+ if (activePosts > 0 || activeStreams > 0 || initializing > 0) {
3117
+ return {
3118
+ ready: false,
3119
+ ...evidence,
3120
+ detail: `primary Connector is busy: activePosts=${activePosts}; activeStreams=${activeStreams}; initializing=${initializing}`,
3121
+ };
3122
+ }
3123
+ if (activeSessions > 0 && (!Number.isFinite(latestActivityAgeMs) || latestActivityAgeMs < quietMs)) {
3124
+ const remainingMs = Number.isFinite(latestActivityAgeMs) ? Math.max(0, quietMs - latestActivityAgeMs) : quietMs;
3125
+ return { ready: false, ...evidence, detail: `waiting for ${remainingMs}ms more MCP quiet time before Runtime cutover` };
3126
+ }
3127
+ return {
3128
+ ready: true,
3129
+ ...evidence,
3130
+ detail: activeSessions === 0
3131
+ ? 'primary Connector has no live MCP sessions'
3132
+ : `primary Connector has been MCP-idle for ${latestActivityAgeMs}ms`,
3133
+ };
3134
+ } catch (error) {
3135
+ return { ready: false, detail: `primary Connector health is unavailable: ${error instanceof Error ? error.message : String(error)}` };
3136
+ } finally {
3137
+ clearTimeout(timer);
3138
+ }
3139
+ }
3140
+
2070
3141
  function activePackageConnectorReleaseBinding(config: RecoveryConfig): PackageConnectorReleaseBinding | undefined {
2071
3142
  const configured = config.primaryConnectorService;
2072
3143
  if (!configured || !configured.localMcpUrl?.trim()) return undefined;
@@ -2082,7 +3153,15 @@ function activePackageConnectorReleaseBinding(config: RecoveryConfig): PackageCo
2082
3153
  return undefined;
2083
3154
  }
2084
3155
  if (!existsSync(join(packageRoot, 'src', 'cli', 'index.ts')) || !existsSync(join(packageRoot, 'src', 'runtime', 'shared', 'node-ts-loader.mjs'))) return undefined;
2085
- return { releaseId: authority.active.releaseId, releaseRoot, packageRoot };
3156
+ const manifest = loadRuntimeReleaseManifest(authority.active.manifestPath, config.controllerHome);
3157
+ return {
3158
+ releaseId: authority.active.releaseId,
3159
+ releaseRoot,
3160
+ packageRoot,
3161
+ ...(manifest.connectorEntrypoint === 'forge-mcp-gateway' && manifest.connectorArtifactIdentity
3162
+ ? { connectorEntrypoint: manifest.connectorEntrypoint, connectorArtifactIdentity: manifest.connectorArtifactIdentity }
3163
+ : {}),
3164
+ };
2086
3165
  }
2087
3166
 
2088
3167
  export function resolveRecoveryPackageConnectorExecutable(
@@ -2263,7 +3342,7 @@ export async function restartPrimaryConnector(
2263
3342
  tunnelRestarted = { ok: result.ok, attempted: true, detail: result.detail, serviceTarget: tunnel.target };
2264
3343
  }
2265
3344
  } else {
2266
- tunnelRestarted = await ensureOpenAiTunnelRuntimeStarted(primaryTunnel, runCommand);
3345
+ tunnelRestarted = await ensureOpenAiTunnelRuntimeStarted(primaryTunnel, runCommand, tunnelClientWorkingDirectory(config.controllerHome));
2267
3346
  }
2268
3347
  if (!tunnelRestarted?.ok) {
2269
3348
  const detail = tunnelRestarted?.detail ?? 'primary public tunnel service configuration is invalid or unavailable';
@@ -2509,19 +3588,9 @@ async function stopPrimaryRuntimeForReleaseTransition(input: {
2509
3588
  runCommand: CommandRunner;
2510
3589
  runtimeRunning: (config: RecoveryConfig) => boolean;
2511
3590
  }): Promise<PrimaryRuntimeStoppedTransition> {
3591
+ const ownerBeforeStop: RuntimeOwnerRecord | null = readRuntimeOwner(input.config.controllerHome) ?? null;
2512
3592
  const stopped = await stopPrimaryRuntimeServiceOwner(input.service, input.runCommand);
2513
3593
  if (!stopped.ok) return { ok: false, detail: stopped.detail };
2514
- const runtimeStopped = await waitForPrimaryRuntimeState({
2515
- config: input.config,
2516
- expectedRunning: false,
2517
- timeoutMs: 20_000,
2518
- now: input.now,
2519
- wait: input.wait,
2520
- runtimeRunning: input.runtimeRunning,
2521
- });
2522
- if (!runtimeStopped) {
2523
- return { ok: false, detail: `primary Runtime owner remained live after bounded ${input.service.platform} stop` };
2524
- }
2525
3594
  const serviceStopped = await waitForPrimaryRuntimeServiceStopped({
2526
3595
  owner: input.service,
2527
3596
  timeoutMs: 20_000,
@@ -2538,6 +3607,35 @@ async function stopPrimaryRuntimeForReleaseTransition(input: {
2538
3607
  };
2539
3608
  }
2540
3609
 
3610
+ let runtimeStopped = await waitForPrimaryRuntimeState({
3611
+ config: input.config,
3612
+ expectedRunning: false,
3613
+ timeoutMs: 5_000,
3614
+ now: input.now,
3615
+ wait: input.wait,
3616
+ runtimeRunning: input.runtimeRunning,
3617
+ });
3618
+ if (!runtimeStopped) {
3619
+ const orphanTermination = await terminateVerifiedRuntimeOwner(input.config.controllerHome, ownerBeforeStop);
3620
+ if (!orphanTermination.ok) {
3621
+ return {
3622
+ ok: false,
3623
+ detail: `primary Runtime owner remained live after service shutdown and bounded identity-verified termination was refused/failed: ${orphanTermination.detail}`,
3624
+ };
3625
+ }
3626
+ runtimeStopped = await waitForPrimaryRuntimeState({
3627
+ config: input.config,
3628
+ expectedRunning: false,
3629
+ timeoutMs: 10_000,
3630
+ now: input.now,
3631
+ wait: input.wait,
3632
+ runtimeRunning: input.runtimeRunning,
3633
+ });
3634
+ if (!runtimeStopped) {
3635
+ return { ok: false, detail: 'primary Runtime owner remained live after verified orphan termination' };
3636
+ }
3637
+ }
3638
+
2541
3639
  let portReleased = await waitForPrimaryRuntimePortReleased({
2542
3640
  config: input.config,
2543
3641
  timeoutMs: 20_000,
@@ -2563,46 +3661,148 @@ async function stopPrimaryRuntimeForReleaseTransition(input: {
2563
3661
  staleListenerCleanup,
2564
3662
  };
2565
3663
  }
2566
- return { ok: true, detail: `primary Runtime ${input.service.platform} service, owner, and TCP listener are stopped`, staleListenerCleanup };
2567
- }
2568
3664
 
2569
- async function verifyPrimaryRuntimeAfterStart(input: {
2570
- config: RecoveryConfig;
2571
- timeoutMs: number;
2572
- now: () => number;
2573
- wait: (ms: number) => Promise<void>;
2574
- verifyLocal: (config: RecoveryConfig) => Promise<VerifyResult>;
2575
- }): Promise<VerifyResult> {
2576
- const deadline = input.now() + input.timeoutMs;
2577
- let observed = await input.verifyLocal(input.config);
2578
- while (!observed.ok && input.now() < deadline) {
2579
- await input.wait(1_000);
2580
- observed = await input.verifyLocal(input.config);
3665
+ let ownership = reconcileStoppedRuntimeOwnership(input.config.controllerHome);
3666
+ if (!ownership.ok) {
3667
+ // A Runtime can finish its listener shutdown before its owner process has
3668
+ // exited. The service and port fences above make this a safe, bounded
3669
+ // orphan-termination point; use the same process-identity proof as the
3670
+ // running-state path instead of treating the durable owner as permanent.
3671
+ const orphanTermination = await terminateVerifiedRuntimeOwner(input.config.controllerHome, ownerBeforeStop);
3672
+ if (!orphanTermination.ok) {
3673
+ return {
3674
+ ok: false,
3675
+ detail: `primary Runtime ownership did not quiesce after bounded ${input.service.platform} stop: ${orphanTermination.detail}`,
3676
+ staleListenerCleanup,
3677
+ };
3678
+ }
3679
+ ownership = reconcileStoppedRuntimeOwnership(input.config.controllerHome);
3680
+ if (!ownership.ok) {
3681
+ return {
3682
+ ok: false,
3683
+ detail: `primary Runtime ownership did not quiesce after verified orphan termination: ${ownership.detail}`,
3684
+ staleListenerCleanup,
3685
+ };
3686
+ }
2581
3687
  }
2582
- return observed;
3688
+ const supervisorSocket = workflowSupervisorSocketPath(resolveWorkflowSupervisorForgeHome(input.config.controllerHome));
3689
+ const supervisor = await reconcileStoppedWorkflowSupervisorSocket(supervisorSocket);
3690
+ if (!supervisor.ok) {
3691
+ return {
3692
+ ok: false,
3693
+ detail: `Workflow Supervisor writer did not quiesce after Runtime stop: ${supervisor.detail}`,
3694
+ staleListenerCleanup,
3695
+ };
3696
+ }
3697
+ return {
3698
+ ok: true,
3699
+ detail: `primary Runtime ${input.service.platform} service, owner, TCP listener, and Workflow Supervisor writer are quiescent`,
3700
+ staleListenerCleanup,
3701
+ };
2583
3702
  }
2584
3703
 
2585
- interface PrimaryRuntimeRebindStartResult {
2586
- ok: boolean;
2587
- detail: string;
2588
- verify: VerifyResult;
2589
- }
3704
+ const PRIMARY_RUNTIME_READINESS_POLL_INTERVAL_MS = 250;
2590
3705
 
2591
- /**
2592
- * Complete the only safe restart sequence after release authority changes:
2593
- * rebind the release-pinned launchd contract, start the sole Runtime service,
2594
- * and require whole-Runtime verification. A started-but-unverified Runtime is
2595
- * stopped again so callers never leave a restart loop behind.
2596
- */
2597
- async function rebindStartAndVerifyPrimaryRuntime(input: {
2598
- config: RecoveryConfig;
2599
- service: PrimaryRuntimeServiceOwner;
2600
- runCommand: CommandRunner;
2601
- now: () => number;
2602
- wait: (ms: number) => Promise<void>;
2603
- verifyLocal: (config: RecoveryConfig) => Promise<VerifyResult>;
2604
- ensureRuntimeLaunchContract?: (controllerHome: string) => void;
3706
+ async function observePrimaryRuntimeActivationReadiness(config: RecoveryConfig): Promise<VerifyResult> {
3707
+ const observation = observeRuntimeStatus(config.controllerHome);
3708
+ const authority = releaseAuthority(config);
3709
+ const active = releaseEvidence(config.controllerHome, authority?.active, authority);
3710
+ const previous = releaseEvidence(config.controllerHome, authority?.previous, authority);
3711
+ const runtimeHealthy = observation.running && observation.ready && !observation.stale;
3712
+ const coherent = Boolean(
3713
+ authority
3714
+ && active
3715
+ && observation.snapshot?.releaseId === active.revision
3716
+ && observation.snapshot?.artifactIdentity === active.artifactIdentity
3717
+ && authority.active.workerProtocolVersion === active.workerProtocolVersion,
3718
+ );
3719
+ const probes: VerifyResult['probes'] = {
3720
+ runtime_status: {
3721
+ ok: observation.running && !observation.stale,
3722
+ detail: observation.running
3723
+ ? observation.stale ? 'canonical Runtime status is stale' : 'canonical Runtime owner is live'
3724
+ : 'canonical Runtime is not running',
3725
+ },
3726
+ };
3727
+ const endpoint = observation.snapshot?.endpoint;
3728
+ probes.active_gateway = endpoint
3729
+ ? await probe(createRecoveryHttpTransport(config.controllerHome), runtimeHealthEndpoint(endpoint), 750)
3730
+ : { ok: false, detail: 'canonical Runtime endpoint is unavailable' };
3731
+ return {
3732
+ ok: Boolean(runtimeHealthy && coherent && probes.active_gateway.ok),
3733
+ at: new Date().toISOString(),
3734
+ runtime: {
3735
+ ok: runtimeHealthy,
3736
+ running: observation.running,
3737
+ ready: observation.ready,
3738
+ stale: observation.stale,
3739
+ reasonCodes: [...observation.reasonCodes],
3740
+ },
3741
+ releases: { active, previous, coherent },
3742
+ probes,
3743
+ };
3744
+ }
3745
+
3746
+ async function waitForPrimaryRuntimeRecoveryHealth(input: {
3747
+ config: RecoveryConfig;
3748
+ timeoutMs: number;
3749
+ now: () => number;
3750
+ wait: (ms: number) => Promise<void>;
3751
+ observeLocal: (config: RecoveryConfig) => Promise<VerifyResult>;
3752
+ }): Promise<VerifyResult> {
3753
+ const deadline = input.now() + input.timeoutMs;
3754
+ let observed = await input.observeLocal(input.config);
3755
+ while (!observed.ok && input.now() < deadline) {
3756
+ await input.wait(PRIMARY_RUNTIME_READINESS_POLL_INTERVAL_MS);
3757
+ observed = await input.observeLocal(input.config);
3758
+ }
3759
+ return observed;
3760
+ }
3761
+
3762
+ async function verifyPrimaryRuntimeAfterStart(input: {
3763
+ config: RecoveryConfig;
3764
+ timeoutMs: number;
3765
+ now: () => number;
3766
+ wait: (ms: number) => Promise<void>;
3767
+ verifyLocal: (config: RecoveryConfig) => Promise<VerifyResult>;
3768
+ observeLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
3769
+ }): Promise<VerifyResult> {
3770
+ const readiness = await waitForPrimaryRuntimeRecoveryHealth({
3771
+ config: input.config,
3772
+ timeoutMs: input.timeoutMs,
3773
+ now: input.now,
3774
+ wait: input.wait,
3775
+ observeLocal: input.observeLocal ?? input.verifyLocal,
3776
+ });
3777
+ if (!readiness.ok) return readiness;
3778
+ // Readiness convergence is intentionally cheap. The acceptance gate remains
3779
+ // one strict whole-Runtime verification after the bounded observation phase.
3780
+ return input.verifyLocal(input.config);
3781
+ }
3782
+
3783
+ interface PrimaryRuntimeRebindStartResult {
3784
+ ok: boolean;
3785
+ detail: string;
3786
+ verify: VerifyResult;
3787
+ }
3788
+
3789
+ /**
3790
+ * Complete the only safe restart sequence after release authority changes:
3791
+ * rebind the release-pinned launchd contract, start the sole Runtime service,
3792
+ * and require whole-Runtime verification. A started-but-unverified Runtime is
3793
+ * stopped again so callers never leave a restart loop behind.
3794
+ */
3795
+ async function rebindStartAndVerifyPrimaryRuntime(input: {
3796
+ config: RecoveryConfig;
3797
+ service: PrimaryRuntimeServiceOwner;
3798
+ runCommand: CommandRunner;
3799
+ now: () => number;
3800
+ wait: (ms: number) => Promise<void>;
3801
+ verifyLocal: (config: RecoveryConfig) => Promise<VerifyResult>;
3802
+ observeLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
3803
+ ensureRuntimeLaunchContract?: (controllerHome: string) => void;
2605
3804
  beforeStart?: () => void;
3805
+ afterRuntimeReady?: () => Promise<{ ok: boolean; detail: string }>;
2606
3806
  contractFailureContext?: string;
2607
3807
  timeoutMs: number;
2608
3808
  successDetail: string;
@@ -2648,13 +3848,42 @@ async function rebindStartAndVerifyPrimaryRuntime(input: {
2648
3848
  if (!started.ok) {
2649
3849
  return { ok: false, detail: started.detail, verify: await input.verifyLocal(input.config) };
2650
3850
  }
2651
- const verify = await verifyPrimaryRuntimeAfterStart({
2652
- config: input.config,
3851
+ const observeLocal = input.observeLocal ?? input.verifyLocal;
3852
+ const runtimeOnlyConfig: RecoveryConfig = input.afterRuntimeReady
3853
+ ? {
3854
+ ...input.config,
3855
+ publicMcpUrl: undefined,
3856
+ primaryConnectorService: undefined,
3857
+ primaryPublicTunnelService: undefined,
3858
+ }
3859
+ : input.config;
3860
+ const runtimeHealth = await waitForPrimaryRuntimeRecoveryHealth({
3861
+ config: runtimeOnlyConfig,
2653
3862
  timeoutMs: input.timeoutMs,
2654
3863
  now: input.now,
2655
3864
  wait: input.wait,
2656
- verifyLocal: input.verifyLocal,
3865
+ observeLocal,
2657
3866
  });
3867
+ if (!runtimeHealth.ok) {
3868
+ await stopPrimaryRuntimeServiceOwner(input.service, input.runCommand);
3869
+ return {
3870
+ ok: false,
3871
+ detail: input.afterRuntimeReady
3872
+ ? 'release transition candidate Runtime did not reach local readiness before Connector rebinding'
3873
+ : 'release transition candidate Runtime did not reach local readiness',
3874
+ verify: runtimeHealth,
3875
+ };
3876
+ }
3877
+ if (input.afterRuntimeReady) {
3878
+ const postReady = await input.afterRuntimeReady();
3879
+ if (!postReady.ok) {
3880
+ await stopPrimaryRuntimeServiceOwner(input.service, input.runCommand);
3881
+ return { ok: false, detail: postReady.detail, verify: await input.verifyLocal(input.config) };
3882
+ }
3883
+ }
3884
+ // Readiness has converged and any Connector rebind has completed. Strict
3885
+ // acceptance runs once here; there is no second readiness polling phase.
3886
+ const verify = await input.verifyLocal(input.config);
2658
3887
  if (verify.ok) return { ok: true, detail: input.successDetail, verify };
2659
3888
  await stopPrimaryRuntimeServiceOwner(input.service, input.runCommand);
2660
3889
  return {
@@ -2669,8 +3898,10 @@ export async function restartPrimaryRuntime(
2669
3898
  dependencies: PrimaryRuntimeRecoveryDependencies = {},
2670
3899
  ): Promise<PrimaryRuntimeRestartResult> {
2671
3900
  const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
3901
+ const observeLocal = dependencies.observeLocal ?? (dependencies.verifyLocal ? dependencies.verifyLocal : observePrimaryRuntimeActivationReadiness);
2672
3902
  const initial = await verifyLocal(config);
2673
- if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime is already healthy', verify: initial };
3903
+ const forceHealthyRestart = dependencies.forceHealthyRestart === true;
3904
+ if (initial.ok && !forceHealthyRestart) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime is already healthy', verify: initial };
2674
3905
  const platform = dependencies.platform ?? process.platform;
2675
3906
  const uid = await (dependencies.currentUid ?? currentUid)();
2676
3907
  const service = primaryRuntimeServiceOwner(config, platform, uid);
@@ -2681,8 +3912,52 @@ export async function restartPrimaryRuntime(
2681
3912
  const wait = dependencies.sleep ?? sleep;
2682
3913
  const locked = await withLock(config, { action: 'restart_primary_runtime' }, async () => {
2683
3914
  const before = await verifyLocal(config);
2684
- if (before.ok) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime recovered before restart', serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRestartResult;
2685
- const started = await ensurePrimaryRuntimeServiceStarted(service, dependencies.runCommand ?? command, 'restart');
3915
+ if (before.ok && !forceHealthyRestart) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime recovered before restart', serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRestartResult;
3916
+ const runCommand = dependencies.runCommand ?? command;
3917
+ const runtimeRunning = dependencies.runtimeRunning ?? ((value: RecoveryConfig) => observeRuntimeStatus(value.controllerHome).running);
3918
+ const stopped = await stopPrimaryRuntimeForReleaseTransition({
3919
+ config,
3920
+ service,
3921
+ now,
3922
+ wait,
3923
+ runCommand,
3924
+ runtimeRunning,
3925
+ });
3926
+ if (!stopped.ok) {
3927
+ audit(config, 'primary_runtime_restart_quiescence_failed', { serviceTarget: service.target, detail: stopped.detail });
3928
+ return { ok: false, attempted: true, detail: stopped.detail, serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRestartResult;
3929
+ }
3930
+ const activeAuthority = releaseAuthority(config);
3931
+ if (activeAuthority) {
3932
+ const repairConnectorBinding = dependencies.repairPrimaryConnectorBinding
3933
+ ?? ((value: RecoveryConfig) => repairPrimaryConnectorBinding(value, platform));
3934
+ const rebound = await rebindStartAndVerifyPrimaryRuntime({
3935
+ config,
3936
+ service,
3937
+ runCommand,
3938
+ now,
3939
+ wait,
3940
+ verifyLocal,
3941
+ observeLocal,
3942
+ ensureRuntimeLaunchContract: dependencies.ensureRuntimeLaunchContract,
3943
+ contractFailureContext: 'before restart',
3944
+ timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 30_000,
3945
+ successDetail: 'Canonical Forge Runtime restarted and passed whole-Runtime verification',
3946
+ afterRuntimeReady: async () => {
3947
+ const connector = await repairConnectorBinding(config);
3948
+ return connector.ok
3949
+ ? { ok: true, detail: connector.detail }
3950
+ : { ok: false, detail: `persistent Connector binding failed after Runtime readiness: ${connector.detail}` };
3951
+ },
3952
+ });
3953
+ if (rebound.ok) {
3954
+ audit(config, 'primary_runtime_restart_succeeded', { serviceTarget: service.target, release: rebound.verify.releases.active?.revision, launchContractRebound: true });
3955
+ return { ok: true, attempted: true, detail: rebound.detail, serviceTarget: service.target, verify: rebound.verify } satisfies PrimaryRuntimeRestartResult;
3956
+ }
3957
+ audit(config, 'primary_runtime_restart_unverified', { serviceTarget: service.target, reasonCodes: rebound.verify.runtime.reasonCodes, detail: rebound.detail, launchContractRebound: true });
3958
+ return { ok: false, attempted: true, detail: rebound.detail, serviceTarget: service.target, verify: rebound.verify } satisfies PrimaryRuntimeRestartResult;
3959
+ }
3960
+ const started = await ensurePrimaryRuntimeServiceStarted(service, runCommand, 'restart');
2686
3961
  if (!started.ok) {
2687
3962
  audit(config, 'primary_runtime_restart_failed', { serviceTarget: service.target, detail: started.detail });
2688
3963
  return { ok: false, attempted: true, detail: started.detail, serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRestartResult;
@@ -2693,6 +3968,7 @@ export async function restartPrimaryRuntime(
2693
3968
  now,
2694
3969
  wait,
2695
3970
  verifyLocal,
3971
+ observeLocal,
2696
3972
  });
2697
3973
  if (after.ok) {
2698
3974
  audit(config, 'primary_runtime_restart_succeeded', { serviceTarget: service.target, release: after.releases.active?.revision });
@@ -2711,8 +3987,63 @@ export async function recoverPrimaryRuntime(
2711
3987
  dependencies: PrimaryRuntimeRecoveryDependencies = {},
2712
3988
  ): Promise<PrimaryRuntimeRecoveryResult> {
2713
3989
  const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
3990
+ const observeLocal = dependencies.observeLocal ?? (dependencies.verifyLocal ? dependencies.verifyLocal : observePrimaryRuntimeActivationReadiness);
2714
3991
  const initial = await verifyLocal(config);
2715
3992
  if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime recovered before rollback', verify: initial };
3993
+
3994
+ const releaseAuthoritySnapshot = releaseAuthority(config);
3995
+ if (releaseAuthoritySnapshot) {
3996
+ const inventory = listCompleteReleaseSessions(config.controllerHome);
3997
+ if (inventory.invalidSessionFiles.length > 0) {
3998
+ return {
3999
+ ok: false,
4000
+ attempted: false,
4001
+ noOp: true,
4002
+ detail: `RELEASE_SESSION_INVENTORY_INCOMPLETE: invalid=${inventory.invalidSessionFiles.join(',') || 'none'}`,
4003
+ verify: initial,
4004
+ };
4005
+ }
4006
+ const matching = inventory.sessions.filter((session) => (
4007
+ ['cutover_attempting', 'cutover_committed', 'soaking'].includes(session.phase)
4008
+ && session.transaction
4009
+ && session.candidateRelease?.releaseId === releaseAuthoritySnapshot.active.releaseId
4010
+ && session.candidateRelease.artifactIdentity === releaseAuthoritySnapshot.active.artifactIdentity
4011
+ ));
4012
+ if (matching.length > 1) {
4013
+ return {
4014
+ ok: false,
4015
+ attempted: false,
4016
+ noOp: true,
4017
+ detail: `RELEASE_SESSION_MULTIPLE_ACTIVE_FOR_RUNTIME: ${matching.map((session) => session.sessionId).join(',')}`,
4018
+ verify: initial,
4019
+ };
4020
+ }
4021
+ const session = matching[0];
4022
+ if (session) {
4023
+ const sessionRollback = await rollbackConfiguredRuntimeReleaseSession(
4024
+ config,
4025
+ session.sessionId,
4026
+ dependencies,
4027
+ `recover-primary-runtime:${session.transaction!.operationId}`,
4028
+ );
4029
+ const after = await verifyLocal(config);
4030
+ const rollback: RollbackResult = {
4031
+ ok: sessionRollback.ok,
4032
+ ...(sessionRollback.noOp === true ? { noOp: true } : {}),
4033
+ detail: sessionRollback.detail,
4034
+ verify: after,
4035
+ };
4036
+ return {
4037
+ ok: sessionRollback.ok,
4038
+ attempted: sessionRollback.attempted,
4039
+ ...(sessionRollback.noOp === true ? { noOp: true } : {}),
4040
+ detail: sessionRollback.detail,
4041
+ rollback,
4042
+ verify: after,
4043
+ };
4044
+ }
4045
+ }
4046
+
2716
4047
  const platform = dependencies.platform ?? process.platform;
2717
4048
  const uid = await (dependencies.currentUid ?? currentUid)();
2718
4049
  const service = primaryRuntimeServiceOwner(config, platform, uid);
@@ -2757,6 +4088,7 @@ export async function recoverPrimaryRuntime(
2757
4088
  now,
2758
4089
  wait,
2759
4090
  verifyLocal,
4091
+ observeLocal,
2760
4092
  ensureRuntimeLaunchContract: dependencies.ensureRuntimeLaunchContract,
2761
4093
  contractFailureContext: 'after rollback',
2762
4094
  timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 45_000,
@@ -2829,17 +4161,17 @@ function validateRuntimeReleaseCandidate(
2829
4161
 
2830
4162
  /**
2831
4163
  * Activate an already staged and validated immutable Runtime release without
2832
- * depending on the primary Runtime execution plane. The transaction mirrors
2833
- * recoverPrimaryRuntime: stop the complete canonical service, atomically switch
2834
- * active/previous whole-release authority (with a local SQLite backup), start
2835
- * the one Runtime service, require whole-Runtime verification, and on failure
2836
- * restore the previous whole release and its SQLite backup before restarting.
4164
+ * depending on the primary Runtime execution plane. RuntimeReleaseAuthority
4165
+ * changes only physical active/previous identity plus the SQLite backup; when a
4166
+ * ReleaseSession owns the cutover, its transaction snapshot is the sole durable
4167
+ * semantic rollback authority through soak.
2837
4168
  */
2838
- export async function activateRuntimeRelease(
4169
+ async function activateRuntimeReleaseInternal(
2839
4170
  config: RecoveryConfig,
2840
4171
  candidateManifestPath: string,
2841
4172
  dependencies: PrimaryRuntimeRecoveryDependencies = {},
2842
4173
  guard: RuntimeReleaseActivationGuard = {},
4174
+ heldRecoveryLock?: RecoveryLock,
2843
4175
  ): Promise<RuntimeReleaseActivationResult> {
2844
4176
  let candidate: { manifest: RuntimeReleaseManifest; releaseRoot: string; manifestPath: string };
2845
4177
  try {
@@ -2849,6 +4181,56 @@ export async function activateRuntimeRelease(
2849
4181
  audit(config, 'runtime_release_activation_candidate_invalid', { detail, ...(guard.requestId ? { requestId: guard.requestId } : {}) });
2850
4182
  return { ok: false, attempted: false, noOp: true, detail };
2851
4183
  }
4184
+ if (candidate.manifest.deploymentScope === 'portable' && !guard.requiredPinnedReleaseRevision) {
4185
+ const releaseSessionId = guard.releaseSessionId?.trim();
4186
+ if (!releaseSessionId) {
4187
+ const detail = 'RELEASE_SESSION_CUTOVER_REQUIRED: portable Runtime candidates cannot activate outside an exact ReleaseSession';
4188
+ audit(config, 'runtime_release_activation_release_session_required', {
4189
+ candidateRevision: candidate.manifest.releaseId,
4190
+ ...(guard.requestId ? { requestId: guard.requestId } : {}),
4191
+ });
4192
+ return { ok: false, attempted: false, noOp: true, detail };
4193
+ }
4194
+ let session: ReleaseSession | undefined;
4195
+ try {
4196
+ session = readReleaseSession(config.controllerHome, releaseSessionId);
4197
+ } catch {
4198
+ session = undefined;
4199
+ }
4200
+ if (!session) {
4201
+ return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
4202
+ }
4203
+ if (session.phase !== 'cutover_attempting') {
4204
+ return {
4205
+ ok: false,
4206
+ attempted: false,
4207
+ noOp: true,
4208
+ detail: `RELEASE_SESSION_NOT_CUTOVER_ATTEMPTING: ${session.phase}`,
4209
+ };
4210
+ }
4211
+ if (resolve(session.stable.controllerHome) !== resolve(config.controllerHome)) {
4212
+ return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_STABLE_LANE_MISMATCH' };
4213
+ }
4214
+ const bound = session.candidateRelease;
4215
+ if (!bound) {
4216
+ return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED' };
4217
+ }
4218
+ const observedManifestSha256 = createHash('sha256').update(readFileSync(candidate.manifestPath)).digest('hex');
4219
+ const observedTreeSha256 = runtimeReleaseTreeSha256(candidate.releaseRoot);
4220
+ if (
4221
+ bound.releaseId !== candidate.manifest.releaseId
4222
+ || bound.artifactIdentity !== candidate.manifest.artifactIdentity
4223
+ || bound.manifestSha256 !== observedManifestSha256
4224
+ || bound.treeSha256 !== observedTreeSha256
4225
+ ) {
4226
+ return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_IDENTITY_MISMATCH' };
4227
+ }
4228
+ try {
4229
+ assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
4230
+ } catch (error) {
4231
+ return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) };
4232
+ }
4233
+ }
2852
4234
  const platform = dependencies.platform ?? process.platform;
2853
4235
  const uid = await (dependencies.currentUid ?? currentUid)();
2854
4236
  const service = primaryRuntimeServiceOwner(config, platform, uid);
@@ -2860,17 +4242,25 @@ export async function activateRuntimeRelease(
2860
4242
  const wait = dependencies.sleep ?? sleep;
2861
4243
  const runtimeRunning = dependencies.runtimeRunning ?? ((value: RecoveryConfig) => observeRuntimeStatus(value.controllerHome).running);
2862
4244
  const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
4245
+ const observeLocal = dependencies.observeLocal ?? (dependencies.verifyLocal ? dependencies.verifyLocal : observePrimaryRuntimeActivationReadiness);
2863
4246
  const repairConnectorBinding = dependencies.repairPrimaryConnectorBinding
2864
4247
  ?? ((value: RecoveryConfig) => repairPrimaryConnectorBinding(value, platform));
2865
4248
  const operationId = `recovery-activate-runtime-${Date.now()}-${randomUUID().slice(0, 8)}`;
2866
4249
  const lockRequestId = guard.requestId?.trim() || operationId;
2867
- const locked = await withLock(config, { action: 'activate_runtime_release', requestId: lockRequestId }, async () => {
4250
+ const activateUnderRecoveryAuthority = async () => {
2868
4251
  // Re-read authority only after acquiring the mutation lock. A caller may
2869
4252
  // have selected its candidate before another activation completed; stale
2870
4253
  // decisions must fail before bootout/publish rather than overwrite the newer
2871
4254
  // active release.
2872
4255
  const current = releaseAuthority(config);
2873
4256
  const previousActive = current?.active;
4257
+ if (guard.requiredPinnedReleaseRevision) {
4258
+ const pinned = runtimePin(config)?.release;
4259
+ if (!pinned || pinned.revision !== guard.requiredPinnedReleaseRevision || pinned.revision !== candidate.manifest.releaseId) {
4260
+ const detail = `RUNTIME_PIN_AUTHORITY_CHANGED: expected ${guard.requiredPinnedReleaseRevision}, observed ${pinned?.revision ?? 'none'}`;
4261
+ return { ok: false, attempted: false, noOp: true, detail, operationId } satisfies RuntimeReleaseActivationResult;
4262
+ }
4263
+ }
2874
4264
  const expectedRevision = guard.expectedAuthorityRevision;
2875
4265
  const expectedActiveReleaseId = typeof guard.expectedActiveReleaseId === 'string' ? guard.expectedActiveReleaseId.trim() : guard.expectedActiveReleaseId;
2876
4266
  if (expectedRevision !== undefined && (expectedRevision === null ? current !== undefined : current?.revision !== expectedRevision)) {
@@ -2958,7 +4348,7 @@ export async function activateRuntimeRelease(
2958
4348
  } satisfies RuntimeReleaseActivationResult;
2959
4349
  }
2960
4350
  }
2961
- if (current?.previous?.releaseId === candidate.manifest.releaseId) {
4351
+ if (!guard.allowPreviousRelease && current?.previous?.releaseId === candidate.manifest.releaseId) {
2962
4352
  const detail = 'RUNTIME_RELEASE_REVERSE_ACTIVATION_REQUIRES_ROLLBACK: activate_runtime_release cannot replace the active release with current.previous; use rollback_previous or recover_primary_runtime';
2963
4353
  audit(config, 'runtime_release_reverse_activation_rejected', {
2964
4354
  operationId,
@@ -2969,9 +4359,23 @@ export async function activateRuntimeRelease(
2969
4359
  });
2970
4360
  return { ok: false, attempted: false, noOp: true, detail, operationId } satisfies RuntimeReleaseActivationResult;
2971
4361
  }
2972
- const before = await verifyLocal(config);
4362
+ // Snapshot Stable A while it is still serving. The release store keeps a
4363
+ // database-wide SQLite data_version observer open across this snapshot and
4364
+ // the stop boundary. Any intervening commit invalidates the speculative
4365
+ // snapshot and publish falls back to a stopped-state backup.
4366
+ let preparedDatabaseBackup;
4367
+ try {
4368
+ preparedDatabaseBackup = current
4369
+ ? prepareRuntimeReleaseDatabaseBackup(config.controllerHome, operationId)
4370
+ : undefined;
4371
+ } catch (error) {
4372
+ const detail = error instanceof Error ? error.message : 'runtime release rollback database preparation failed';
4373
+ audit(config, 'runtime_release_activation_backup_prepare_failed', { serviceTarget: service.target, operationId, detail });
4374
+ return { ok: false, attempted: false, noOp: true, detail, serviceTarget: service.target, verify: await observeLocal(config) } satisfies RuntimeReleaseActivationResult;
4375
+ }
2973
4376
  const stopped = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
2974
4377
  if (!stopped.ok) {
4378
+ preparedDatabaseBackup?.discard();
2975
4379
  const action = stopped.detail.startsWith('primary Runtime bootout failed')
2976
4380
  ? 'runtime_release_activation_stop_failed'
2977
4381
  : 'runtime_release_activation_stop_unverified';
@@ -2989,7 +4393,13 @@ export async function activateRuntimeRelease(
2989
4393
  }
2990
4394
  let committed: RuntimeReleaseAuthority;
2991
4395
  try {
2992
- committed = publishRuntimeRelease(config.controllerHome, candidate.manifestPath, operationId);
4396
+ committed = publishRuntimeRelease(
4397
+ config.controllerHome,
4398
+ candidate.manifestPath,
4399
+ operationId,
4400
+ undefined,
4401
+ preparedDatabaseBackup,
4402
+ );
2993
4403
  } catch (error) {
2994
4404
  const detail = error instanceof Error ? error.message : 'runtime release authority publish failed';
2995
4405
  audit(config, 'runtime_release_activation_publish_failed', { serviceTarget: service.target, operationId, detail });
@@ -3000,65 +4410,113 @@ export async function activateRuntimeRelease(
3000
4410
  audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId });
3001
4411
  return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
3002
4412
  }
3003
- // The Connector is independently supervised, but its executable is part of
3004
- // this immutable Runtime release. Rebind it immediately after publishing
3005
- // authority and before starting the candidate: full Runtime verification
3006
- // includes the Connector and must never observe a stale previous release.
3007
- const candidateConnectorBinding = await repairConnectorBinding(config);
3008
- let storageMigration: ControllerHomeStorageMigration | undefined;
4413
+
4414
+ // Once the physical authority switch is durable, persist exact Stable A
4415
+ // rollback identity before Candidate B starts. A later start/Connector/verify
4416
+ // failure therefore never precedes ReleaseSession rollback authority.
4417
+ let releaseSessionTransactionPersisted = false;
3009
4418
  let activationFailureDetail: string | undefined;
3010
- let activated: PrimaryRuntimeRebindStartResult;
3011
- if (!candidateConnectorBinding.ok) {
3012
- const detail = `persistent Connector binding failed before Runtime activation: ${candidateConnectorBinding.detail}`;
3013
- activationFailureDetail = detail;
3014
- audit(config, 'runtime_release_activation_connector_binding_failed', {
3015
- serviceTarget: service.target,
3016
- operationId,
3017
- requestId: lockRequestId,
3018
- detail: candidateConnectorBinding.detail,
3019
- });
3020
- activated = { ok: false, detail, verify: await verifyLocal(config) };
3021
- } else {
3022
- activated = await rebindStartAndVerifyPrimaryRuntime({
3023
- config,
3024
- service,
3025
- runCommand,
3026
- now,
3027
- wait,
3028
- verifyLocal,
3029
- timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
3030
- successDetail: 'requested Runtime release started and passed whole-Runtime verification',
3031
- beforeStart: () => {
3032
- storageMigration = migrateStoppedRepoLocalControllerHomeStorage(config.controllerHome, platform);
3033
- if (storageMigration.migrated) {
3034
- audit(config, 'runtime_controller_home_noindex_migrated', {
3035
- serviceTarget: service.target,
3036
- operationId,
3037
- logicalHome: storageMigration.logicalHome,
3038
- physicalHome: storageMigration.physicalHome,
3039
- });
3040
- }
3041
- },
3042
- });
4419
+ if (!guard.preserveDatabaseOnFailure && guard.releaseSessionId?.trim()) {
4420
+ try {
4421
+ const session = readReleaseSession(config.controllerHome, guard.releaseSessionId.trim());
4422
+ if (
4423
+ committed.operationId !== operationId
4424
+ || !committed.previous?.databaseBackup
4425
+ || !session
4426
+ || session.phase !== 'cutover_attempting'
4427
+ ) throw new Error('RELEASE_SESSION_TRANSACTION_CAPTURE_PRECONDITION_FAILED');
4428
+ recordReleaseSessionTransaction({
4429
+ controllerHome: config.controllerHome,
4430
+ sessionId: session.sessionId,
4431
+ expectedRevision: session.revision,
4432
+ transaction: {
4433
+ schemaVersion: 1,
4434
+ operationId,
4435
+ candidateReleaseId: candidate.manifest.releaseId,
4436
+ cutoverAuthorityRevision: committed.revision,
4437
+ rollbackRelease: committed.previous,
4438
+ startedAt: committed.committedAt,
4439
+ },
4440
+ });
4441
+ releaseSessionTransactionPersisted = true;
4442
+ } catch (error) {
4443
+ activationFailureDetail = `ReleaseSession rollback transaction capture failed before Runtime start: ${error instanceof Error ? error.message : String(error)}`;
4444
+ audit(config, 'release_session_transaction_capture_failed', {
4445
+ serviceTarget: service.target,
4446
+ operationId,
4447
+ requestId: lockRequestId,
4448
+ releaseSessionId: guard.releaseSessionId.trim(),
4449
+ detail: activationFailureDetail,
4450
+ });
4451
+ }
3043
4452
  }
4453
+
4454
+ // The Connector is independently supervised, but its package snapshot proxies
4455
+ // the Canonical Runtime. Starting/rebinding it while the Runtime is stopped
4456
+ // creates a dependency cycle: Connector readiness waits on a Runtime that has
4457
+ // not started yet. Start the candidate first, require Runtime-only readiness,
4458
+ // then rebind the Connector and finally require whole-Runtime verification.
4459
+ let candidateConnectorBinding: { ok: boolean; attempted: boolean; noOp?: boolean; detail: string } | undefined;
4460
+ let storageMigration: ControllerHomeStorageMigration | undefined;
4461
+ const activated = activationFailureDetail
4462
+ ? { ok: false, detail: activationFailureDetail, verify: await observeLocal(config) }
4463
+ : await rebindStartAndVerifyPrimaryRuntime({
4464
+ config,
4465
+ service,
4466
+ runCommand,
4467
+ now,
4468
+ wait,
4469
+ verifyLocal,
4470
+ observeLocal,
4471
+ timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
4472
+ successDetail: 'requested Runtime release started and passed whole-Runtime verification',
4473
+ beforeStart: () => {
4474
+ storageMigration = migrateStoppedRepoLocalControllerHomeStorage(config.controllerHome, platform);
4475
+ if (storageMigration.migrated) {
4476
+ audit(config, 'runtime_controller_home_noindex_migrated', {
4477
+ serviceTarget: service.target,
4478
+ operationId,
4479
+ logicalHome: storageMigration.logicalHome,
4480
+ physicalHome: storageMigration.physicalHome,
4481
+ });
4482
+ }
4483
+ },
4484
+ afterRuntimeReady: async () => {
4485
+ candidateConnectorBinding = await repairConnectorBinding(config);
4486
+ if (candidateConnectorBinding.ok) return { ok: true, detail: candidateConnectorBinding.detail };
4487
+ const detail = `persistent Connector binding failed after Runtime readiness: ${candidateConnectorBinding.detail}`;
4488
+ activationFailureDetail = detail;
4489
+ audit(config, 'runtime_release_activation_connector_binding_failed', {
4490
+ serviceTarget: service.target,
4491
+ operationId,
4492
+ requestId: lockRequestId,
4493
+ detail: candidateConnectorBinding.detail,
4494
+ });
4495
+ return { ok: false, detail };
4496
+ },
4497
+ });
3044
4498
  let after = activated.verify;
3045
4499
  if (activated.ok && after.releases.active?.revision === candidate.manifest.releaseId) {
3046
4500
  audit(config, 'runtime_release_activation_succeeded', {
3047
- serviceTarget: service.target,
3048
- operationId,
3049
- requestId: lockRequestId,
3050
- activeRevision: after.releases.active?.revision,
3051
- expectedAuthorityRevision: guard.expectedAuthorityRevision,
3052
- expectedActiveReleaseId: guard.expectedActiveReleaseId,
3053
- controllerHomeStorageMigrated: storageMigration?.migrated === true,
3054
- connectorBindingRepaired: candidateConnectorBinding.attempted,
4501
+ serviceTarget: service.target,
4502
+ operationId,
4503
+ requestId: lockRequestId,
4504
+ activeRevision: after.releases.active?.revision,
4505
+ expectedAuthorityRevision: guard.expectedAuthorityRevision,
4506
+ expectedActiveReleaseId: guard.expectedActiveReleaseId,
4507
+ controllerHomeStorageMigrated: storageMigration?.migrated === true,
4508
+ connectorBindingRepaired: candidateConnectorBinding?.attempted === true,
4509
+ releaseSessionTransactionPersisted,
4510
+ ...(guard.releaseSessionId?.trim() ? { releaseSessionId: guard.releaseSessionId.trim() } : {}),
3055
4511
  });
3056
4512
  return {
3057
4513
  ok: true,
3058
4514
  attempted: true,
3059
- detail: storageMigration?.migrated
3060
- ? 'requested Runtime release activated, Controller Home migrated to .noindex storage, persistent Connector rebound, and whole-Runtime verification passed'
3061
- : 'requested Runtime release activated, persistent Connector rebound, and whole-Runtime verification passed',
4515
+ detail: releaseSessionTransactionPersisted
4516
+ ? 'requested Runtime release activated and verified; physical activation committed and ReleaseSession owns rollback authority through soak'
4517
+ : storageMigration?.migrated
4518
+ ? 'requested Runtime release activated, Controller Home migrated to .noindex storage, persistent Connector rebound, and whole-Runtime verification passed'
4519
+ : 'requested Runtime release activated, persistent Connector rebound, and whole-Runtime verification passed',
3062
4520
  serviceTarget: service.target,
3063
4521
  operationId,
3064
4522
  verify: after,
@@ -3079,8 +4537,9 @@ export async function activateRuntimeRelease(
3079
4537
  audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId, detail: activationFailureDetail });
3080
4538
  }
3081
4539
 
3082
- // Activation failed: stop, restore the previous whole release and its SQLite
3083
- // backup, restart the one service, and require verification again.
4540
+ // Activation failed: stop and restore the previous whole release. SQLite
4541
+ // rollback is allowed only when the live durable-mutation generation still
4542
+ // equals the captured backup; newer Controller Home state is preserved.
3084
4543
  let rollback: RollbackResult;
3085
4544
  const rollbackStop = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
3086
4545
  if (!rollbackStop.ok) {
@@ -3092,8 +4551,25 @@ export async function activateRuntimeRelease(
3092
4551
  };
3093
4552
  } else {
3094
4553
  try {
3095
- const rollbackOperationId = `recovery-activate-runtime-rollback-${Date.now()}-${randomUUID().slice(0, 8)}`;
3096
- const restored = rollbackRuntimeRelease(config.controllerHome, rollbackOperationId);
4554
+ const rollbackOperationId = guard.preserveDatabaseOnFailure
4555
+ ? `recovery-activate-runtime-rollback-${Date.now()}-${randomUUID().slice(0, 8)}`
4556
+ : operationId;
4557
+ let rollbackDatabaseDisposition: RuntimeDatabaseRollbackDisposition | 'preserved_by_guard' = 'preserved_by_guard';
4558
+ const physicallyRestored = guard.preserveDatabaseOnFailure && previousActive
4559
+ ? publishRuntimeRelease(config.controllerHome, previousActive.manifestPath, rollbackOperationId)
4560
+ : (() => {
4561
+ const rollbackResult = rollbackRuntimeReleaseWithResult(config.controllerHome, rollbackOperationId);
4562
+ rollbackDatabaseDisposition = rollbackResult.databaseDisposition;
4563
+ return rollbackResult.authority;
4564
+ })();
4565
+ const restored = current && previousActive
4566
+ ? reconcileFailedRuntimeReleaseActivationAuthority(
4567
+ config.controllerHome,
4568
+ current,
4569
+ candidate.manifest.releaseId,
4570
+ rollbackOperationId,
4571
+ )
4572
+ : physicallyRestored;
3097
4573
  if (storageMigration?.migrated) {
3098
4574
  rollbackStoppedRepoLocalControllerHomeStorage(storageMigration);
3099
4575
  audit(config, 'runtime_controller_home_noindex_migration_rolled_back', {
@@ -3112,6 +4588,7 @@ export async function activateRuntimeRelease(
3112
4588
  ) {
3113
4589
  throw new Error('RECOVERY_RUNTIME_RELEASE_ROLLBACK_AUTHORITY_MISMATCH');
3114
4590
  }
4591
+ let rollbackConnectorBinding: { ok: boolean; attempted: boolean; noOp?: boolean; detail: string } | undefined;
3115
4592
  const restarted = await rebindStartAndVerifyPrimaryRuntime({
3116
4593
  config,
3117
4594
  service,
@@ -3119,25 +4596,31 @@ export async function activateRuntimeRelease(
3119
4596
  now,
3120
4597
  wait,
3121
4598
  verifyLocal,
4599
+ observeLocal,
3122
4600
  contractFailureContext: 'after rollback',
3123
4601
  timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
3124
- successDetail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified',
4602
+ successDetail: rollbackDatabaseDisposition === 'restored_backup'
4603
+ ? 'previous whole-Runtime release and unchanged SQLite backup restored, restarted, rebound, and verified'
4604
+ : 'previous Runtime release restored while live SQLite state was preserved, restarted, rebound, and verified',
4605
+ afterRuntimeReady: async () => {
4606
+ rollbackConnectorBinding = await repairConnectorBinding(config);
4607
+ if (rollbackConnectorBinding.ok) return { ok: true, detail: rollbackConnectorBinding.detail };
4608
+ audit(config, 'runtime_release_activation_rollback_connector_binding_failed', {
4609
+ serviceTarget: service.target,
4610
+ operationId,
4611
+ rollbackOperationId,
4612
+ detail: rollbackConnectorBinding.detail,
4613
+ });
4614
+ return {
4615
+ ok: false,
4616
+ detail: `previous persistent Connector binding failed after rollback Runtime readiness: ${rollbackConnectorBinding.detail}`,
4617
+ };
4618
+ },
3125
4619
  });
3126
- const connectorBinding = restarted.ok ? await repairConnectorBinding(config) : undefined;
3127
- if (connectorBinding && !connectorBinding.ok) {
3128
- audit(config, 'runtime_release_activation_rollback_connector_binding_failed', {
3129
- serviceTarget: service.target,
3130
- operationId,
3131
- rollbackOperationId,
3132
- detail: connectorBinding.detail,
3133
- });
3134
- }
3135
4620
  rollback = {
3136
- ok: restarted.ok && (connectorBinding?.ok ?? true),
4621
+ ok: restarted.ok,
3137
4622
  operationId: rollbackOperationId,
3138
- detail: connectorBinding && !connectorBinding.ok
3139
- ? `${restarted.detail}; previous persistent Connector binding failed: ${connectorBinding.detail}`
3140
- : restarted.detail,
4623
+ detail: restarted.detail,
3141
4624
  verify: restarted.verify,
3142
4625
  };
3143
4626
  } catch (error) {
@@ -3164,80 +4647,1950 @@ export async function activateRuntimeRelease(
3164
4647
  rollback,
3165
4648
  verify: after,
3166
4649
  } satisfies RuntimeReleaseActivationResult;
3167
- });
4650
+ };
4651
+
4652
+ if (heldRecoveryLock) {
4653
+ const live = liveRecoveryMutationLock(config);
4654
+ if (
4655
+ !live
4656
+ || live.instanceId !== heldRecoveryLock.instanceId
4657
+ || live.pid !== heldRecoveryLock.pid
4658
+ || live.processStartTime !== heldRecoveryLock.processStartTime
4659
+ ) {
4660
+ return {
4661
+ ok: false,
4662
+ attempted: false,
4663
+ noOp: true,
4664
+ detail: 'RECOVERY_OPERATION_LOCK_AUTHORITY_CHANGED',
4665
+ serviceTarget: service.target,
4666
+ verify: await verifyStableRuntime(config),
4667
+ };
4668
+ }
4669
+ return activateUnderRecoveryAuthority();
4670
+ }
4671
+
4672
+ const locked = await withLock(
4673
+ config,
4674
+ { action: 'activate_runtime_release', requestId: lockRequestId },
4675
+ async () => activateUnderRecoveryAuthority(),
4676
+ );
3168
4677
  if (!locked.acquired) {
3169
4678
  return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), serviceTarget: service.target, verify: await verifyStableRuntime(config) };
3170
4679
  }
3171
4680
  return locked.value;
3172
4681
  }
3173
4682
 
3174
- export interface ConfiguredRuntimeActivationDependencies {
3175
- stage?: typeof stageRuntimeReleaseFromCandidateSource;
3176
- activate?: (config: RecoveryConfig, manifestPath: string, guard: RuntimeReleaseActivationGuard) => Promise<RuntimeReleaseActivationResult>;
4683
+ export async function activateRuntimeRelease(
4684
+ config: RecoveryConfig,
4685
+ candidateManifestPath: string,
4686
+ dependencies: PrimaryRuntimeRecoveryDependencies = {},
4687
+ guard: RuntimeReleaseActivationGuard = {},
4688
+ ): Promise<RuntimeReleaseActivationResult> {
4689
+ return activateRuntimeReleaseInternal(config, candidateManifestPath, dependencies, guard);
3177
4690
  }
3178
4691
 
3179
- export async function stageAndActivateConfiguredRuntimeRelease(
4692
+ export async function activatePinnedRuntimeRelease(
3180
4693
  config: RecoveryConfig,
3181
- dependencies: ConfiguredRuntimeActivationDependencies = {},
3182
- requestId?: string,
3183
- ): Promise<ConfiguredRuntimeActivationResult> {
3184
- const sourceRoot = config.primaryRuntimeSourceRoot?.trim();
3185
- if (!sourceRoot) {
3186
- return { ok: false, attempted: false, noOp: true, detail: 'primary Runtime source root is not configured in standalone Recovery' };
3187
- }
3188
- const sourceRepositoryId = config.primaryRuntimeSourceRepositoryId?.trim();
3189
- if (!sourceRepositoryId) {
3190
- return { ok: false, attempted: false, noOp: true, detail: 'primary Runtime source repository id is not configured in standalone Recovery' };
3191
- }
3192
- // Capture the base before staging. Staging can take long enough for another
3193
- // activation to win; the later publish must prove this base is still current.
3194
- const expectedAuthority = releaseAuthority(config);
3195
- const stagedAttempt = await withLock(config, {
3196
- action: 'stage_and_activate_runtime_release',
3197
- ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
4694
+ dependencies: PrimaryRuntimeRecoveryDependencies = {},
4695
+ guard: RuntimeReleaseActivationGuard = {},
4696
+ ): Promise<RuntimeReleaseActivationResult> {
4697
+ let pin: RuntimePinStore | undefined;
4698
+ try { pin = runtimePin(config); }
4699
+ catch (error) { return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) }; }
4700
+ if (!pin) return { ok: false, attempted: false, noOp: true, detail: 'RUNTIME_PIN_REQUIRED: no Runtime release is pinned' };
4701
+ let candidate: { manifest: RuntimeReleaseManifest; releaseRoot: string; manifestPath: string };
4702
+ try { candidate = validateRuntimeReleaseCandidate(config, pin.release.path); }
4703
+ catch (error) { return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) }; }
4704
+ if (candidate.manifest.deploymentScope === 'portable') {
4705
+ return {
4706
+ ok: false,
4707
+ attempted: false,
4708
+ noOp: true,
4709
+ detail: 'RUNTIME_PIN_PORTABLE_RELEASE_FORBIDDEN: portable source candidates require a verified Recovery ReleaseSession',
4710
+ };
4711
+ }
4712
+ let observed: ReleaseEvidence;
4713
+ try { observed = releaseEvidenceForPin(config, candidate); }
4714
+ catch (error) { return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) }; }
4715
+ if (observed.revision !== pin.release.revision || observed.artifactIdentity !== pin.release.artifactIdentity || observed.manifestSha256 !== pin.release.manifestSha256) {
4716
+ return { ok: false, attempted: false, noOp: true, detail: 'RUNTIME_PIN_IDENTITY_MISMATCH: pinned immutable release changed on disk' };
4717
+ }
4718
+ return activateRuntimeRelease(config, pin.release.path, dependencies, {
4719
+ ...guard,
4720
+ allowPreviousRelease: true,
4721
+ preserveDatabaseOnFailure: true,
4722
+ requiredPinnedReleaseRevision: pin.release.revision,
4723
+ });
4724
+ }
4725
+
4726
+
4727
+ function configuredSourceRevision(sourceRoot: string): string {
4728
+ const env = {
4729
+ ...runtimeAuthorityFreeEnvironment(process.env),
4730
+ PATH: recoveryCommandPath(),
4731
+ };
4732
+ // ReleaseSession source authority is the exact committed Git object selected
4733
+ // internally by Recovery. The configured checkout is only an object/dependency
4734
+ // provider: concurrent working-tree edits are intentionally non-authoritative
4735
+ // and are excluded by the detached immutable source snapshot used below.
4736
+ const head = spawnSync('git', ['-C', sourceRoot, 'rev-parse', '--verify', 'HEAD^{commit}'], {
4737
+ encoding: 'utf8',
4738
+ env,
4739
+ timeout: 10_000,
4740
+ });
4741
+ const revision = (head.stdout ?? '').trim();
4742
+ if (head.status !== 0 || !/^[a-f0-9]{40}$/i.test(revision)) {
4743
+ throw new Error('RELEASE_SESSION_SOURCE_REVISION_UNAVAILABLE');
4744
+ }
4745
+ return revision;
4746
+ }
4747
+
4748
+ export interface ConfiguredRuntimeReleaseSourceState {
4749
+ configured: boolean;
4750
+ sourceRevision?: string;
4751
+ activeSourceCommit?: string;
4752
+ activeReleaseId?: string;
4753
+ }
4754
+
4755
+ /**
4756
+ * Read the two immutable identities used only to decide whether automatic
4757
+ * release reconciliation is warranted. This does not create or advance a
4758
+ * ReleaseSession and therefore never becomes release intent authority.
4759
+ */
4760
+ export function configuredRuntimeReleaseSourceState(config: RecoveryConfig): ConfiguredRuntimeReleaseSourceState {
4761
+ const sourceRoot = config.primaryRuntimeSourceRoot?.trim();
4762
+ if (!sourceRoot) return { configured: false };
4763
+ const authority = readRuntimeReleaseAuthority(config.controllerHome);
4764
+ if (!authority) throw new Error('RELEASE_AUTOMATION_ACTIVE_RELEASE_AUTHORITY_UNKNOWN');
4765
+ const manifest = loadRuntimeReleaseManifest(authority.active.manifestPath, config.controllerHome);
4766
+ return {
4767
+ configured: true,
4768
+ sourceRevision: configuredSourceRevision(sourceRoot),
4769
+ activeSourceCommit: manifest.sourceCommit?.trim() || undefined,
4770
+ activeReleaseId: authority.active.releaseId,
4771
+ };
4772
+ }
4773
+
4774
+ async function allocateCandidateLoopbackPort(stablePort: number): Promise<number> {
4775
+ for (let attempt = 0; attempt < 8; attempt += 1) {
4776
+ const port = await new Promise<number>((resolvePort, rejectPort) => {
4777
+ const server = createNetServer();
4778
+ server.unref();
4779
+ server.once('error', rejectPort);
4780
+ server.listen({ host: '127.0.0.1', port: 0, exclusive: true }, () => {
4781
+ const address = server.address();
4782
+ const selected = address && typeof address === 'object' ? address.port : 0;
4783
+ server.close((error) => error ? rejectPort(error) : resolvePort(selected));
4784
+ });
4785
+ });
4786
+ if (port > 0 && port !== stablePort) return port;
4787
+ }
4788
+ throw new Error('RELEASE_SESSION_CANDIDATE_PORT_UNAVAILABLE');
4789
+ }
4790
+
4791
+ function stableReleaseSessionIdentity(config: RecoveryConfig): ReleaseSessionStableRelease {
4792
+ const authority = releaseAuthority(config);
4793
+ if (!authority) throw new Error('RELEASE_SESSION_STABLE_RELEASE_AUTHORITY_REQUIRED');
4794
+ return {
4795
+ authorityRevision: authority.revision,
4796
+ releaseId: authority.active.releaseId,
4797
+ artifactIdentity: authority.active.artifactIdentity,
4798
+ manifestSha256: authority.active.manifestSha256,
4799
+ workerProtocolVersion: authority.active.workerProtocolVersion,
4800
+ releaseFencingTokenSha256: createHash('sha256').update(authority.fencingToken).digest('hex'),
4801
+ };
4802
+ }
4803
+
4804
+ function sameStableReleaseSessionIdentity(left: ReleaseSessionStableRelease, right: ReleaseSessionStableRelease): boolean {
4805
+ return left.authorityRevision === right.authorityRevision
4806
+ && left.releaseId === right.releaseId
4807
+ && left.artifactIdentity === right.artifactIdentity
4808
+ && left.manifestSha256 === right.manifestSha256
4809
+ && left.workerProtocolVersion === right.workerProtocolVersion
4810
+ && left.releaseFencingTokenSha256 === right.releaseFencingTokenSha256;
4811
+ }
4812
+
4813
+ function assertStableReleaseSessionIdentityCurrent(config: RecoveryConfig, expected: ReleaseSessionStableRelease): void {
4814
+ const actual = stableReleaseSessionIdentity(config);
4815
+ if (
4816
+ actual.authorityRevision !== expected.authorityRevision
4817
+ || actual.releaseId !== expected.releaseId
4818
+ || actual.artifactIdentity !== expected.artifactIdentity
4819
+ || actual.manifestSha256 !== expected.manifestSha256
4820
+ || actual.workerProtocolVersion !== expected.workerProtocolVersion
4821
+ || actual.releaseFencingTokenSha256 !== expected.releaseFencingTokenSha256
4822
+ ) throw new Error('RELEASE_SESSION_STABLE_AUTHORITY_CHANGED');
4823
+ }
4824
+
4825
+ function candidateControllerHomeForSession(config: RecoveryConfig, sessionId: string): string {
4826
+ return join(dirname(resolve(config.controllerHome)), 'candidate-runtime-lanes', sessionId);
4827
+ }
4828
+
4829
+ export async function prepareConfiguredRuntimeReleaseSession(
4830
+ config: RecoveryConfig,
4831
+ dependencies: Pick<ConfiguredRuntimeActivationDependencies, 'stage'> = {},
4832
+ requestId?: string,
4833
+ ): Promise<ConfiguredRuntimeActivationResult> {
4834
+ const sourceRoot = config.primaryRuntimeSourceRoot?.trim();
4835
+ if (!sourceRoot) return { ok: false, attempted: false, noOp: true, detail: 'primary Runtime source root is not configured in standalone Recovery' };
4836
+ const sourceRepositoryId = config.primaryRuntimeSourceRepositoryId?.trim();
4837
+ if (!sourceRepositoryId) return { ok: false, attempted: false, noOp: true, detail: 'primary Runtime source repository id is not configured in standalone Recovery' };
4838
+
4839
+ const verifiedStable = await verifyStableRuntime(config);
4840
+ if (!verifiedStable.ok) {
4841
+ return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_STABLE_RUNTIME_NOT_VERIFIED' };
4842
+ }
4843
+
4844
+ const locked = await withLock(config, {
4845
+ action: 'release_session_prepare',
4846
+ ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
4847
+ }, async () => {
4848
+ const stable = readStableExecutionLane(config.controllerHome);
4849
+ const stableRelease = stableReleaseSessionIdentity(config);
4850
+ const sourceRevision = configuredSourceRevision(sourceRoot);
4851
+
4852
+ // ReleaseSession owns semantic progression. This Recovery lock serializes only
4853
+ // physical preparation. Resume the exact source_frozen session after an
4854
+ // interruption instead of inventing a second release authority.
4855
+ const inventory = listCompleteReleaseSessions(config.controllerHome);
4856
+ if (inventory.invalidSessionFiles.length > 0) {
4857
+ return {
4858
+ ok: false as const,
4859
+ attempted: false,
4860
+ noOp: true,
4861
+ detail: `RELEASE_SESSION_INVENTORY_INCOMPLETE: invalid=${inventory.invalidSessionFiles.join(',') || 'none'}`,
4862
+ };
4863
+ }
4864
+ await reconcileOrphanCandidateLanes(config, stable, inventory.sessions);
4865
+ let resumableMatchingSession: ReleaseSession | undefined;
4866
+ for (const existing of [...inventory.sessions].sort((left, right) => left.updatedAt.localeCompare(right.updatedAt))) {
4867
+ // Candidate B becomes disposable once the semantic ReleaseSession says it
4868
+ // is retired. Cleanup is deliberately retried here because process death
4869
+ // can happen after the phase CAS but before the physical lane removal. The
4870
+ // remover is session/path fenced and idempotent, so this reconciles leaked
4871
+ // bytes without creating another retention authority.
4872
+ if (releaseSessionCandidateIsRetired(existing)) cleanupRetiredCandidateLane(config, existing);
4873
+ if (existing.phase === 'failed' || existing.phase === 'rolled_back' || existing.phase === 'known_good') continue;
4874
+ const preparationReusablePhase = existing.phase === 'source_frozen'
4875
+ || existing.phase === 'built'
4876
+ || existing.phase === 'static_verified'
4877
+ || existing.phase === 'candidate_booted'
4878
+ || existing.phase === 'candidate_verified'
4879
+ || existing.phase === 'cutover_eligible';
4880
+ if (
4881
+ preparationReusablePhase
4882
+ && existing.sourceRevision === sourceRevision
4883
+ && sameStableReleaseSessionIdentity(existing.stableRelease, stableRelease)
4884
+ ) {
4885
+ if (resumableMatchingSession) {
4886
+ return {
4887
+ ok: false as const,
4888
+ attempted: false,
4889
+ noOp: true,
4890
+ detail: `RELEASE_SESSION_MULTIPLE_ACTIVE: ${resumableMatchingSession.sessionId}:${resumableMatchingSession.phase},${existing.sessionId}:${existing.phase}`,
4891
+ };
4892
+ }
4893
+ resumableMatchingSession = existing;
4894
+ continue;
4895
+ }
4896
+ if (existing.phase === 'cutover_attempting' || existing.phase === 'cutover_committed') {
4897
+ return {
4898
+ ok: false as const,
4899
+ attempted: false,
4900
+ noOp: true,
4901
+ detail: `RELEASE_SESSION_PREPARE_REQUIRES_RECONCILIATION: ${existing.sessionId}:${existing.phase}`,
4902
+ releaseSession: existing,
4903
+ };
4904
+ }
4905
+ if (existing.phase === 'soaking') {
4906
+ const candidateIsCurrentStable = releaseSessionIsSoakingPredecessorOfStable(existing, stableRelease);
4907
+ if (candidateIsCurrentStable && existing.sourceRevision === sourceRevision) {
4908
+ return {
4909
+ ok: false as const,
4910
+ attempted: false,
4911
+ noOp: true,
4912
+ detail: `RELEASE_SESSION_PREPARE_REQUIRES_SOAK_RESOLUTION: ${existing.sessionId}`,
4913
+ releaseSession: existing,
4914
+ };
4915
+ }
4916
+ if (candidateIsCurrentStable) {
4917
+ // A newer source is allowed to prepare against the exact release that
4918
+ // this predecessor already cut over. The predecessor remains soaking
4919
+ // so its rollback authority survives until the successor commits.
4920
+ continue;
4921
+ }
4922
+ if (releaseSessionIsSoakingSupersededByStableAuthority(existing, stableRelease)) {
4923
+ const superseded = advanceReleaseSession({
4924
+ controllerHome: config.controllerHome,
4925
+ sessionId: existing.sessionId,
4926
+ expectedRevision: existing.revision,
4927
+ phase: 'failed',
4928
+ receipts: [{
4929
+ id: 'soak_superseded_by_runtime_authority',
4930
+ kind: 'soak',
4931
+ summary: `Soaking release ${existing.candidateRelease?.releaseId ?? 'unknown'} was superseded by active Runtime authority ${stableRelease.releaseId}@${stableRelease.authorityRevision}`.slice(0, 500),
4932
+ }],
4933
+ });
4934
+ audit(config, 'release_session_soak_superseded_by_runtime_authority', {
4935
+ sessionId: existing.sessionId,
4936
+ priorCutoverAuthorityRevision: existing.transaction?.cutoverAuthorityRevision,
4937
+ activeAuthorityRevision: stableRelease.authorityRevision,
4938
+ activeReleaseId: stableRelease.releaseId,
4939
+ });
4940
+ cleanupRetiredCandidateLane(config, superseded);
4941
+ continue;
4942
+ }
4943
+ return {
4944
+ ok: false as const,
4945
+ attempted: false,
4946
+ noOp: true,
4947
+ detail: `RELEASE_SESSION_SOAK_RUNTIME_IDENTITY_MISMATCH: ${existing.sessionId}`,
4948
+ releaseSession: existing,
4949
+ };
4950
+ }
4951
+ const superseded = await cancelReleaseSessionUnderLock(
4952
+ config,
4953
+ existing,
4954
+ 'superseded by a newer ReleaseSession source freeze',
4955
+ );
4956
+ if (!superseded.ok) return superseded;
4957
+ }
4958
+
4959
+ if (resumableMatchingSession && resumableMatchingSession.phase !== 'source_frozen') {
4960
+ return {
4961
+ ok: true as const,
4962
+ attempted: false,
4963
+ noOp: true,
4964
+ detail: `matching ReleaseSession ${resumableMatchingSession.sessionId} already advanced to ${resumableMatchingSession.phase}; preparation is idempotent and must not supersede it`,
4965
+ releaseSession: resumableMatchingSession,
4966
+ };
4967
+ }
4968
+
4969
+ // Reconcile and reclaim only bytes that no current rollback authority needs.
4970
+ // This is one bounded lifecycle attempt, not a second GC owner, and it
4971
+ // retains fail-closed behavior whenever an authority cannot be proven.
4972
+ reconcileKnownGoodRecoveryRetention(config);
4973
+ const releaseRetention = cleanupControllerReleaseHistory(config.controllerHome, {
4974
+ graceMs: 0,
4975
+ stagingGraceMs: 0,
4976
+ maxRemovals: 16,
4977
+ });
4978
+ if (releaseRetention.errors.length > 0) {
4979
+ throw new Error(`RELEASE_SESSION_RETENTION_RECONCILIATION_FAILED: ${releaseRetention.errors.slice(0, 3).join('; ')}`);
4980
+ }
4981
+
4982
+ // Capacity is operation-specific: the candidate is a measured copy of the
4983
+ // current immutable Runtime tree plus a consistent SQLite snapshot. The
4984
+ // global warning threshold is observability, not rollback reserve; existing
4985
+ // active/previous authorities are already allocated and protected above.
4986
+ const candidateRoot = join(dirname(resolve(config.controllerHome)), 'candidate-runtime-lanes');
4987
+ const storageEstimate = estimateReleaseSessionCandidateBytes(config, releaseAuthority(config)!.active.manifestPath);
4988
+ assertStorageHeadroom(candidateRoot, {
4989
+ operation: 'release_session_prepare',
4990
+ requiredBytes: storageEstimate.requiredBytes,
4991
+ reserveBytes: storageEstimate.reserveBytes,
4992
+ });
4993
+
4994
+ let session: ReleaseSession;
4995
+ let candidateLane: ReturnType<typeof createCandidateExecutionLane>['candidate'];
4996
+ if (resumableMatchingSession) {
4997
+ session = resumableMatchingSession;
4998
+ candidateLane = session.candidate;
4999
+ } else {
5000
+ const nextSessionId = `release-${Date.now()}-${randomUUID().replaceAll('-', '').slice(0, 16)}`;
5001
+ const candidatePort = await allocateCandidateLoopbackPort(stable.port);
5002
+ candidateLane = createCandidateExecutionLane({
5003
+ stableControllerHome: stable.controllerHome,
5004
+ candidateControllerHome: candidateControllerHomeForSession(config, nextSessionId),
5005
+ candidatePort,
5006
+ sessionId: nextSessionId,
5007
+ }).candidate;
5008
+ session = createReleaseSession({
5009
+ controllerHome: config.controllerHome,
5010
+ sessionId: nextSessionId,
5011
+ stable,
5012
+ stableRelease,
5013
+ candidate: candidateLane,
5014
+ sourceRevision,
5015
+ });
5016
+ }
5017
+ const sessionId = session.sessionId;
5018
+
5019
+ try {
5020
+ assertStableReleaseSessionIdentityCurrent(config, stableRelease);
5021
+ const staged = withRuntimeReleaseSourceSnapshot({ sourceRoot, sourceRevision }, frozenSourceRoot =>
5022
+ (dependencies.stage ?? stageRuntimeReleaseFromCandidateSource)({
5023
+ controllerHome: candidateLane.controllerHome,
5024
+ sourceRoot: frozenSourceRoot,
5025
+ dependencyRoot: sourceRoot,
5026
+ sourceRepositoryId,
5027
+ }));
5028
+ assertRuntimeReleaseFiles(staged);
5029
+ if (staged.sourceCommit !== sourceRevision) throw new Error('RELEASE_SESSION_SOURCE_CHANGED_DURING_BUILD');
5030
+ const candidateRelease: ReleaseSessionCandidateRelease = {
5031
+ releaseId: staged.releaseId,
5032
+ manifestPath: staged.manifestPath,
5033
+ artifactIdentity: staged.artifactIdentity,
5034
+ manifestSha256: staged.manifestSha256,
5035
+ treeSha256: runtimeReleaseTreeSha256(staged.releasePath),
5036
+ sourceCommit: staged.sourceCommit,
5037
+ ...(staged.sourceRepositoryId ? { sourceRepositoryId: staged.sourceRepositoryId } : {}),
5038
+ };
5039
+ assertStableReleaseSessionIdentityCurrent(config, stableRelease);
5040
+ session = advanceReleaseSession({
5041
+ controllerHome: config.controllerHome,
5042
+ sessionId,
5043
+ expectedRevision: session.revision,
5044
+ phase: 'built',
5045
+ candidateRelease,
5046
+ receipts: [{
5047
+ id: 'build',
5048
+ kind: 'build',
5049
+ summary: `portable candidate ${staged.releaseId} built under isolated Candidate B; Stable A unchanged`,
5050
+ }],
5051
+ });
5052
+ audit(config, 'release_session_candidate_built', {
5053
+ sessionId,
5054
+ sourceRevision,
5055
+ stableReleaseId: stableRelease.releaseId,
5056
+ stableAuthorityRevision: stableRelease.authorityRevision,
5057
+ candidateControllerHome: candidateLane.controllerHome,
5058
+ candidatePort: candidateLane.port,
5059
+ candidateReleaseId: staged.releaseId,
5060
+ candidateTreeSha256: candidateRelease.treeSha256,
5061
+ });
5062
+ return {
5063
+ ok: true as const,
5064
+ attempted: true,
5065
+ detail: 'configured Runtime source was built into an isolated Candidate B ReleaseSession; Stable A was not stopped or activated',
5066
+ staged,
5067
+ releaseSession: session,
5068
+ };
5069
+ } catch (error) {
5070
+ const detail = error instanceof Error ? error.message : String(error);
5071
+ try {
5072
+ session = advanceReleaseSession({
5073
+ controllerHome: config.controllerHome,
5074
+ sessionId,
5075
+ expectedRevision: session.revision,
5076
+ phase: 'failed',
5077
+ receipts: [{ id: 'build_failed', kind: 'build', summary: detail.slice(0, 500) }],
5078
+ });
5079
+ } catch { /* Preserve the original build failure. */ }
5080
+ cleanupRetiredCandidateLane(config, session);
5081
+ audit(config, 'release_session_candidate_build_failed', { sessionId, sourceRevision, detail });
5082
+ return { ok: false as const, attempted: true, detail, releaseSession: session };
5083
+ }
5084
+ });
5085
+
5086
+ if (!locked.acquired) {
5087
+ return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
5088
+ }
5089
+ return locked.value;
5090
+ }
5091
+
5092
+
5093
+ const RELEASE_SESSION_STATIC_GATES = [
5094
+ { id: 'type', args: ['run', 'check:type'], timeoutMs: 10 * 60_000 },
5095
+ { id: 'runtime_architecture', args: ['run', 'check:runtime-architecture'], timeoutMs: 5 * 60_000 },
5096
+ { id: 'architecture_sync', args: ['run', 'check:architecture-sync'], timeoutMs: 5 * 60_000 },
5097
+ { id: 'bootstrap', args: ['run', 'check:bootstrap-files'], timeoutMs: 5 * 60_000 },
5098
+ ] as const;
5099
+
5100
+ export async function verifyConfiguredRuntimeReleaseSessionStaticGates(
5101
+ config: RecoveryConfig,
5102
+ sessionId: string,
5103
+ requestId?: string,
5104
+ ): Promise<ConfiguredRuntimeActivationResult> {
5105
+ const sourceRoot = config.primaryRuntimeSourceRoot?.trim();
5106
+ if (!sourceRoot) return { ok: false, attempted: false, noOp: true, detail: 'primary Runtime source root is not configured in standalone Recovery' };
5107
+ const locked = await withLock(config, {
5108
+ action: 'release_session_static_verify',
5109
+ ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
5110
+ }, async () => {
5111
+ let session = readReleaseSession(config.controllerHome, sessionId);
5112
+ if (!session) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
5113
+ if (session.phase !== 'built') {
5114
+ return { ok: false as const, attempted: false, noOp: true, detail: `RELEASE_SESSION_STATIC_VERIFY_REQUIRES_BUILT: ${session.phase}`, releaseSession: session };
5115
+ }
5116
+ const candidate = session.candidateRelease;
5117
+ if (!candidate) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED', releaseSession: session };
5118
+ const frozenSourceRevision = session.sourceRevision;
5119
+ const stableRelease = session.stableRelease;
5120
+ try {
5121
+ assertStableReleaseSessionIdentityCurrent(config, stableRelease);
5122
+ const manifest = loadRuntimeReleaseManifest(candidate.manifestPath, session.candidate.controllerHome);
5123
+ if (
5124
+ manifest.deploymentScope !== 'portable'
5125
+ || manifest.releaseId !== candidate.releaseId
5126
+ || manifest.artifactIdentity !== candidate.artifactIdentity
5127
+ || createHash('sha256').update(readFileSync(candidate.manifestPath)).digest('hex') !== candidate.manifestSha256
5128
+ || runtimeReleaseTreeSha256(dirname(candidate.manifestPath)) !== candidate.treeSha256
5129
+ ) throw new Error('RELEASE_SESSION_CANDIDATE_IDENTITY_MISMATCH');
5130
+
5131
+ const receipts = withRuntimeReleaseSourceSnapshot({
5132
+ sourceRoot,
5133
+ sourceRevision: frozenSourceRevision,
5134
+ }, frozenSourceRoot => {
5135
+ const gateReceipts: Array<{ id: string; kind: 'static_gate'; summary: string }> = [];
5136
+ for (const gate of RELEASE_SESSION_STATIC_GATES) {
5137
+ assertStableReleaseSessionIdentityCurrent(config, stableRelease);
5138
+ const startedAt = Date.now();
5139
+ const bunExecutable = resolveBunExecutable();
5140
+ const boundedRunner = join(frozenSourceRoot, 'scripts', 'run-bounded-command.mjs');
5141
+ const result = spawnSync(bunExecutable, [
5142
+ boundedRunner,
5143
+ '--timeout-ms', String(gate.timeoutMs),
5144
+ '--',
5145
+ bunExecutable,
5146
+ ...gate.args,
5147
+ ], {
5148
+ cwd: frozenSourceRoot,
5149
+ env: { ...runtimeAuthorityFreeEnvironment(process.env), PATH: recoveryCommandPath() },
5150
+ encoding: 'utf8',
5151
+ // The inner bounded runner owns the command process group and gets a
5152
+ // brief escalation window to terminate descendants before this
5153
+ // outer Recovery watchdog is allowed to intervene.
5154
+ timeout: gate.timeoutMs + 15_000,
5155
+ maxBuffer: 8 * 1024 * 1024,
5156
+ });
5157
+ const durationMs = Date.now() - startedAt;
5158
+ if (result.error || result.status !== 0) {
5159
+ const detail = result.error instanceof Error
5160
+ ? result.error.message
5161
+ : (result.stderr || result.stdout || `exit ${result.status ?? 'unknown'}`).trim().slice(-2000);
5162
+ throw new Error(`RELEASE_SESSION_STATIC_GATE_FAILED:${gate.id}: ${detail}`);
5163
+ }
5164
+ gateReceipts.push({
5165
+ id: gate.id,
5166
+ kind: 'static_gate',
5167
+ summary: `${gate.id} passed on frozen source ${frozenSourceRevision} in ${durationMs}ms`,
5168
+ });
5169
+ }
5170
+ return gateReceipts;
5171
+ });
5172
+ assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
5173
+ session = advanceReleaseSession({
5174
+ controllerHome: config.controllerHome,
5175
+ sessionId,
5176
+ expectedRevision: session.revision,
5177
+ phase: 'static_verified',
5178
+ receipts,
5179
+ });
5180
+ audit(config, 'release_session_static_verified', {
5181
+ sessionId,
5182
+ sourceRevision: session.sourceRevision,
5183
+ candidateReleaseId: candidate.releaseId,
5184
+ gates: RELEASE_SESSION_STATIC_GATES.map((gate) => gate.id),
5185
+ });
5186
+ return {
5187
+ ok: true as const,
5188
+ attempted: true,
5189
+ detail: 'ReleaseSession static gates passed on the frozen source revision; Stable A remained unchanged',
5190
+ releaseSession: session,
5191
+ };
5192
+ } catch (error) {
5193
+ const detail = error instanceof Error ? error.message : String(error);
5194
+ try {
5195
+ session = advanceReleaseSession({
5196
+ controllerHome: config.controllerHome,
5197
+ sessionId,
5198
+ expectedRevision: session.revision,
5199
+ phase: 'failed',
5200
+ receipts: [{ id: 'static_failed', kind: 'static_gate', summary: detail.slice(0, 500) }],
5201
+ });
5202
+ } catch { /* preserve the original static verification failure */ }
5203
+ cleanupRetiredCandidateLane(config, session);
5204
+ audit(config, 'release_session_static_failed', { sessionId, detail });
5205
+ return { ok: false as const, attempted: true, detail, releaseSession: session };
5206
+ }
5207
+ });
5208
+ if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
5209
+ return locked.value;
5210
+ }
5211
+
5212
+
5213
+ const RELEASE_SESSION_CANDIDATE_VERIFY_TIMEOUT_MS = 60_000;
5214
+
5215
+ function candidateRecoveryConfig(session: ReleaseSession): RecoveryConfig {
5216
+ return {
5217
+ schemaVersion: 1,
5218
+ controllerHome: session.candidate.controllerHome,
5219
+ installProfile: 'manual',
5220
+ primaryRuntimeService: {
5221
+ ...defaultPrimaryRuntimeServiceConfig(),
5222
+ postRestartVerifyTimeoutMs: RELEASE_SESSION_CANDIDATE_VERIFY_TIMEOUT_MS,
5223
+ },
5224
+ mainMcpTokenFile: session.candidate.authTokenFile,
5225
+ readOnlyTool: {
5226
+ name: STABLE_RECOVERY_READ_ONLY_TOOL.name,
5227
+ arguments: { ...STABLE_RECOVERY_READ_ONLY_TOOL.arguments },
5228
+ },
5229
+ };
5230
+ }
5231
+
5232
+ async function installReleaseSessionCandidateService(
5233
+ session: ReleaseSession,
5234
+ runCommand: CommandRunner = command,
5235
+ ): Promise<void> {
5236
+ const candidateRelease = session.candidateRelease;
5237
+ if (!candidateRelease) throw new Error('RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED');
5238
+ const home = session.candidate.controllerHome;
5239
+ const serviceConfig = readForgeRuntimeServiceConfig(forgeRuntimeServicePaths(home).configPath);
5240
+ if (
5241
+ resolve(serviceConfig.controllerHome) !== resolve(home)
5242
+ || serviceConfig.port !== session.candidate.port
5243
+ || resolve(serviceConfig.authTokenFile) !== resolve(session.candidate.authTokenFile)
5244
+ ) throw new Error('RELEASE_SESSION_CANDIDATE_SERVICE_CONTRACT_MISMATCH');
5245
+
5246
+ const operationId = `release-session-candidate-${session.sessionId}`;
5247
+ const authority = publishRuntimeRelease(home, candidateRelease.manifestPath, operationId);
5248
+ if (
5249
+ authority.active.releaseId !== candidateRelease.releaseId
5250
+ || authority.active.artifactIdentity !== candidateRelease.artifactIdentity
5251
+ || authority.active.manifestSha256 !== candidateRelease.manifestSha256
5252
+ ) throw new Error('RELEASE_SESSION_CANDIDATE_AUTHORITY_MISMATCH');
5253
+ syncForgeRuntimeActiveEntrypoint(home);
5254
+
5255
+ if (process.platform === 'darwin') {
5256
+ await installForgeRuntimeService({
5257
+ config: serviceConfig,
5258
+ // Ignored in release mode; a non-empty bounded path keeps bootstrap fallback explicit.
5259
+ runnerPath: candidateRelease.manifestPath,
5260
+ });
5261
+ return;
5262
+ }
5263
+ if (process.platform === 'linux') {
5264
+ const env = runtimeAuthorityFreeEnvironment(process.env);
5265
+ writePackageRuntimeSystemdUserService(home, env);
5266
+ for (const argv of systemdRuntimeInstallCommands(forgeRuntimeServicePaths(home).label)) {
5267
+ const [executable, ...args] = argv;
5268
+ if (!executable) continue;
5269
+ const result = await runCommand(executable, args, 20_000);
5270
+ if (!result.ok) throw new Error(`RELEASE_SESSION_CANDIDATE_SYSTEMD_INSTALL_FAILED: ${result.stderr || result.stdout || executable}`);
5271
+ }
5272
+ return;
5273
+ }
5274
+ throw new Error(`RELEASE_SESSION_CANDIDATE_SERVICE_PLATFORM_UNSUPPORTED: ${process.platform}`);
5275
+ }
5276
+
5277
+ interface ReleaseSessionCandidateRetirement {
5278
+ ok: boolean;
5279
+ detail: string;
5280
+ }
5281
+
5282
+ function cleanupRetiredCandidateLane(config: RecoveryConfig, session: ReleaseSession): ReleaseSessionCandidateRetirement {
5283
+ try {
5284
+ const removed = removeRetiredCandidateExecutionLane(session.stable, session.candidate);
5285
+ if (removed) {
5286
+ audit(config, 'release_session_candidate_lane_cleaned', {
5287
+ sessionId: session.sessionId,
5288
+ candidateControllerHome: session.candidate.controllerHome,
5289
+ phase: session.phase,
5290
+ });
5291
+ }
5292
+ return {
5293
+ ok: true,
5294
+ detail: removed
5295
+ ? 'Candidate B Controller Home was removed after terminal ReleaseSession cleanup'
5296
+ : 'Candidate B Controller Home was already absent after terminal ReleaseSession cleanup',
5297
+ };
5298
+ } catch (error) {
5299
+ const detail = error instanceof Error ? error.message : String(error);
5300
+ audit(config, 'release_session_candidate_lane_cleanup_failed', {
5301
+ sessionId: session.sessionId,
5302
+ candidateControllerHome: session.candidate.controllerHome,
5303
+ phase: session.phase,
5304
+ detail,
5305
+ });
5306
+ return { ok: false, detail };
5307
+ }
5308
+ }
5309
+
5310
+ async function stopCandidateServiceAtHome(
5311
+ home: string,
5312
+ runCommand: CommandRunner = command,
5313
+ ): Promise<ReleaseSessionCandidateRetirement> {
5314
+ try {
5315
+ if (process.platform === 'darwin') {
5316
+ try {
5317
+ await uninstallForgeRuntimeService(home);
5318
+ } catch (error) {
5319
+ const uid = await currentUid();
5320
+ if (uid === undefined) {
5321
+ return { ok: false, detail: error instanceof Error ? error.message : String(error) };
5322
+ }
5323
+ const paths = forgeRuntimeServicePaths(home);
5324
+ const loaded = await runCommand('launchctl', ['print', `gui/${uid}/${paths.label}`], 5_000);
5325
+ if (loaded.ok) {
5326
+ return { ok: false, detail: error instanceof Error ? error.message : String(error) };
5327
+ }
5328
+ // An already-absent launchd job is an idempotent retirement. Remove only
5329
+ // its stale installed declaration, then verify the Runtime itself below.
5330
+ rmSync(paths.installedPlistPath, { force: true });
5331
+ }
5332
+ } else if (process.platform === 'linux') {
5333
+ const label = forgeRuntimeServicePaths(home).label;
5334
+ const unitName = systemdUserUnitName(label);
5335
+ const stopped = await runCommand('systemctl', ['--user', 'disable', '--now', unitName], 20_000);
5336
+ const stoppedDetail = `${stopped.stderr}\n${stopped.stdout}`;
5337
+ if (!stopped.ok && !/not loaded|not found|does not exist|no such file/i.test(stoppedDetail)) {
5338
+ return { ok: false, detail: `RELEASE_SESSION_CANDIDATE_SYSTEMD_STOP_FAILED: ${stoppedDetail.trim() || stopped.status}` };
5339
+ }
5340
+ rmSync(systemdUserUnitPath(unitName), { force: true });
5341
+ const reloaded = await runCommand('systemctl', ['--user', 'daemon-reload'], 20_000);
5342
+ if (!reloaded.ok) {
5343
+ return { ok: false, detail: `RELEASE_SESSION_CANDIDATE_SYSTEMD_RELOAD_FAILED: ${reloaded.stderr || reloaded.stdout || reloaded.status}` };
5344
+ }
5345
+ } else {
5346
+ return { ok: false, detail: `RELEASE_SESSION_CANDIDATE_SERVICE_PLATFORM_UNSUPPORTED: ${process.platform}` };
5347
+ }
5348
+ } catch (error) {
5349
+ return { ok: false, detail: error instanceof Error ? error.message : String(error) };
5350
+ }
5351
+
5352
+ const observation = observeRuntimeStatus(home);
5353
+ if (observation.running) {
5354
+ return { ok: false, detail: 'RELEASE_SESSION_CANDIDATE_STILL_RUNNING_AFTER_RETIREMENT' };
5355
+ }
5356
+ return { ok: true, detail: 'Candidate B persistent service is absent and its Runtime process is not running' };
5357
+ }
5358
+
5359
+ async function stopReleaseSessionCandidateService(
5360
+ session: ReleaseSession,
5361
+ runCommand: CommandRunner = command,
5362
+ ): Promise<ReleaseSessionCandidateRetirement> {
5363
+ return stopCandidateServiceAtHome(session.candidate.controllerHome, runCommand);
5364
+ }
5365
+
5366
+ async function reconcileOrphanCandidateLanes(
5367
+ config: RecoveryConfig,
5368
+ stable: ReturnType<typeof readStableExecutionLane>,
5369
+ sessions: ReleaseSession[],
5370
+ ): Promise<void> {
5371
+ const candidateRoot = resolve(dirname(resolve(config.controllerHome)), 'candidate-runtime-lanes');
5372
+ if (!existsSync(candidateRoot)) return;
5373
+ const knownHomes = new Set(sessions.map((session) => resolve(session.candidate.controllerHome)));
5374
+ const entries = readdirSync(candidateRoot, { withFileTypes: true })
5375
+ .filter((entry) => entry.isDirectory())
5376
+ .sort((left, right) => left.name.localeCompare(right.name));
5377
+ if (entries.length > 512) throw new Error(`RELEASE_SESSION_CANDIDATE_LANE_INVENTORY_INCOMPLETE: ${entries.length}`);
5378
+
5379
+ for (const entry of entries) {
5380
+ const candidateHome = resolve(candidateRoot, entry.name);
5381
+ if (knownHomes.has(candidateHome)) continue;
5382
+ try {
5383
+ // Reuse ReleaseSession's own id parser as the identity fence. A directory
5384
+ // whose name is not a valid session id is visible debt, not delete authority.
5385
+ if (readReleaseSession(config.controllerHome, entry.name)) continue;
5386
+ } catch (error) {
5387
+ audit(config, 'release_session_orphan_candidate_identity_invalid', {
5388
+ candidateControllerHome: candidateHome,
5389
+ detail: error instanceof Error ? error.message : String(error),
5390
+ });
5391
+ continue;
5392
+ }
5393
+
5394
+ let candidateConfig;
5395
+ try {
5396
+ candidateConfig = readForgeRuntimeServiceConfig(forgeRuntimeServicePaths(candidateHome).configPath, { missingRepositoryRoot: 'omit' });
5397
+ } catch (error) {
5398
+ audit(config, 'release_session_orphan_candidate_service_contract_unproven', {
5399
+ sessionId: entry.name,
5400
+ candidateControllerHome: candidateHome,
5401
+ detail: error instanceof Error ? error.message : String(error),
5402
+ });
5403
+ continue;
5404
+ }
5405
+ if (resolve(candidateConfig.controllerHome) !== candidateHome || candidateConfig.port === stable.port) {
5406
+ audit(config, 'release_session_orphan_candidate_service_contract_mismatch', {
5407
+ sessionId: entry.name,
5408
+ candidateControllerHome: candidateHome,
5409
+ configuredControllerHome: candidateConfig.controllerHome,
5410
+ configuredPort: candidateConfig.port,
5411
+ stablePort: stable.port,
5412
+ });
5413
+ continue;
5414
+ }
5415
+
5416
+ let running = true;
5417
+ try {
5418
+ running = observeRuntimeStatus(candidateHome).running;
5419
+ } catch (error) {
5420
+ audit(config, 'release_session_orphan_candidate_owner_unknown', {
5421
+ sessionId: entry.name,
5422
+ candidateControllerHome: candidateHome,
5423
+ detail: error instanceof Error ? error.message : String(error),
5424
+ });
5425
+ continue;
5426
+ }
5427
+ if (running) {
5428
+ audit(config, 'release_session_orphan_candidate_active_owner', { sessionId: entry.name, candidateControllerHome: candidateHome });
5429
+ continue;
5430
+ }
5431
+
5432
+ const retirement = await stopCandidateServiceAtHome(candidateHome);
5433
+ if (!retirement.ok) {
5434
+ audit(config, 'release_session_orphan_candidate_service_retirement_failed', {
5435
+ sessionId: entry.name,
5436
+ candidateControllerHome: candidateHome,
5437
+ detail: retirement.detail,
5438
+ });
5439
+ continue;
5440
+ }
5441
+ try {
5442
+ const removed = removeRetiredCandidateExecutionLane(stable, { controllerHome: candidateHome, sessionId: entry.name });
5443
+ if (removed) {
5444
+ audit(config, 'release_session_orphan_candidate_lane_cleaned', { sessionId: entry.name, candidateControllerHome: candidateHome });
5445
+ }
5446
+ } catch (error) {
5447
+ audit(config, 'release_session_orphan_candidate_lane_cleanup_failed', {
5448
+ sessionId: entry.name,
5449
+ candidateControllerHome: candidateHome,
5450
+ detail: error instanceof Error ? error.message : String(error),
5451
+ });
5452
+ }
5453
+ }
5454
+ }
5455
+
5456
+ function candidateCanaryReceipts(
5457
+ session: ReleaseSession,
5458
+ verified: VerifyResult,
5459
+ ): Array<{ id: string; kind: 'candidate_canary'; summary: string }> {
5460
+ const observation = observeRuntimeStatus(session.candidate.controllerHome);
5461
+ const diagnostics = observation.snapshot?.readiness.diagnostics;
5462
+ const mcpProbeNames = ['mcp_initialize', 'mcp_initialized_notification', 'mcp_tools_list', 'mcp_read_only_call', 'mcp_session_close'];
5463
+ const failedMcp = mcpProbeNames.filter((name) => verified.probes[name]?.ok !== true);
5464
+ if (failedMcp.length > 0) throw new Error(`RELEASE_SESSION_CANDIDATE_MCP_CANARY_FAILED: ${failedMcp.join(',')}`);
5465
+ if (diagnostics?.scheduler.outcome !== 'pass') throw new Error('RELEASE_SESSION_CANDIDATE_SCHEDULER_CANARY_FAILED');
5466
+ if (diagnostics?.database.outcome !== 'pass' || diagnostics?.releaseCoherence.outcome !== 'pass' || diagnostics?.mcpEndToEnd.outcome !== 'pass') {
5467
+ throw new Error('RELEASE_SESSION_CANDIDATE_RUNTIME_DIAGNOSTICS_FAILED');
5468
+ }
5469
+ const supervisorSocket = workflowSupervisorSocketPath(resolveWorkflowSupervisorForgeHome(session.candidate.controllerHome));
5470
+ if (!existsSync(supervisorSocket)) throw new Error('RELEASE_SESSION_CANDIDATE_SUPERVISOR_CANARY_FAILED');
5471
+ return [
5472
+ { id: 'recovery', kind: 'candidate_canary', summary: 'Candidate B stopped and restarted through Recovery service lifecycle and returned whole-Runtime healthy' },
5473
+ { id: 'mcp', kind: 'candidate_canary', summary: 'Candidate B MCP initialize/tools-list/read-only-call/session-close passed using the Candidate credential' },
5474
+ { id: 'scheduler', kind: 'candidate_canary', summary: 'Candidate B scheduler readiness diagnostic passed after recovery restart' },
5475
+ { id: 'supervisor', kind: 'candidate_canary', summary: `Candidate B Workflow Supervisor socket is live at ${supervisorSocket}` },
5476
+ { id: 'controller', kind: 'candidate_canary', summary: 'Candidate B controller read-only facade completed through MCP after restart' },
5477
+ ];
5478
+ }
5479
+
5480
+ export async function bootAndVerifyConfiguredRuntimeReleaseSessionCandidate(
5481
+ config: RecoveryConfig,
5482
+ sessionId: string,
5483
+ requestId?: string,
5484
+ ): Promise<ConfiguredRuntimeActivationResult> {
5485
+ const locked = await withLock(config, {
5486
+ action: 'release_session_candidate_boot',
5487
+ ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
5488
+ }, async () => {
5489
+ let session = readReleaseSession(config.controllerHome, sessionId);
5490
+ if (!session) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
5491
+ if (session.phase === 'candidate_booted') {
5492
+ const retirement = await stopReleaseSessionCandidateService(session);
5493
+ if (!retirement.ok) {
5494
+ return {
5495
+ ok: false as const,
5496
+ attempted: true,
5497
+ detail: `Candidate B boot/restart canary was interrupted and retirement failed: ${retirement.detail}`,
5498
+ releaseSession: session,
5499
+ };
5500
+ }
5501
+ session = advanceReleaseSession({
5502
+ controllerHome: config.controllerHome,
5503
+ sessionId,
5504
+ expectedRevision: session.revision,
5505
+ phase: 'failed',
5506
+ receipts: [{
5507
+ id: 'candidate_failed',
5508
+ kind: 'candidate_canary',
5509
+ summary: 'Candidate B boot/restart canary was interrupted; Candidate B was retired before terminalizing the ReleaseSession',
5510
+ }],
5511
+ });
5512
+ cleanupRetiredCandidateLane(config, session);
5513
+ return {
5514
+ ok: false as const,
5515
+ attempted: true,
5516
+ detail: 'Candidate B boot/restart canary was interrupted; Candidate B is retired and the ReleaseSession is terminal failed',
5517
+ releaseSession: session,
5518
+ };
5519
+ }
5520
+ if (session.phase !== 'static_verified') {
5521
+ return { ok: false as const, attempted: false, noOp: true, detail: `RELEASE_SESSION_CANDIDATE_BOOT_REQUIRES_STATIC_VERIFIED: ${session.phase}`, releaseSession: session };
5522
+ }
5523
+ const candidateRelease = session.candidateRelease;
5524
+ if (!candidateRelease) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED', releaseSession: session };
5525
+ const runCommand = command;
5526
+ const candidateConfig = candidateRecoveryConfig(session);
5527
+ try {
5528
+ assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
5529
+ if (runtimeReleaseTreeSha256(dirname(candidateRelease.manifestPath)) !== candidateRelease.treeSha256) {
5530
+ throw new Error('RELEASE_SESSION_CANDIDATE_TREE_CHANGED');
5531
+ }
5532
+ await installReleaseSessionCandidateService(session, runCommand);
5533
+
5534
+ let initialVerify = await verifyPrimaryRuntimeAfterStart({
5535
+ config: candidateConfig,
5536
+ timeoutMs: RELEASE_SESSION_CANDIDATE_VERIFY_TIMEOUT_MS,
5537
+ now: Date.now,
5538
+ wait: sleep,
5539
+ verifyLocal: verifyLocalRuntime,
5540
+ observeLocal: observePrimaryRuntimeActivationReadiness,
5541
+ });
5542
+ if (!initialVerify.ok) throw new Error(`RELEASE_SESSION_CANDIDATE_BOOT_UNVERIFIED: ${initialVerify.runtime.reasonCodes.join(',')}`);
5543
+ assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
5544
+ session = advanceReleaseSession({
5545
+ controllerHome: config.controllerHome,
5546
+ sessionId,
5547
+ expectedRevision: session.revision,
5548
+ phase: 'candidate_booted',
5549
+ receipts: [{ id: 'candidate_boot', kind: 'candidate_canary', summary: 'Candidate B booted and passed initial whole-Runtime verification while Stable A remained healthy' }],
5550
+ });
5551
+
5552
+ const platform = process.platform;
5553
+ const uid = await currentUid();
5554
+ const service = primaryRuntimeServiceOwner(candidateConfig, platform, uid);
5555
+ if (!service) throw new Error('RELEASE_SESSION_CANDIDATE_SERVICE_OWNER_MISSING');
5556
+ const stopped = await stopPrimaryRuntimeForReleaseTransition({
5557
+ config: candidateConfig,
5558
+ service,
5559
+ now: Date.now,
5560
+ wait: sleep,
5561
+ runCommand,
5562
+ runtimeRunning: (candidate) => observeRuntimeStatus(candidate.controllerHome).running,
5563
+ });
5564
+ if (!stopped.ok) throw new Error(`RELEASE_SESSION_CANDIDATE_RECOVERY_STOP_FAILED: ${stopped.detail}`);
5565
+ const restarted = await restartPrimaryRuntime(candidateConfig);
5566
+ if (!restarted.ok) throw new Error(`RELEASE_SESSION_CANDIDATE_RECOVERY_RESTART_FAILED: ${restarted.detail}`);
5567
+ initialVerify = restarted.verify;
5568
+ const receipts = candidateCanaryReceipts(session, initialVerify);
5569
+ assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
5570
+ session = advanceReleaseSession({
5571
+ controllerHome: config.controllerHome,
5572
+ sessionId,
5573
+ expectedRevision: session.revision,
5574
+ phase: 'candidate_verified',
5575
+ receipts,
5576
+ });
5577
+ session = advanceReleaseSession({
5578
+ controllerHome: config.controllerHome,
5579
+ sessionId,
5580
+ expectedRevision: session.revision,
5581
+ phase: 'cutover_eligible',
5582
+ });
5583
+ audit(config, 'release_session_candidate_verified', {
5584
+ sessionId,
5585
+ candidateReleaseId: candidateRelease.releaseId,
5586
+ candidateControllerHome: session.candidate.controllerHome,
5587
+ candidatePort: session.candidate.port,
5588
+ stableReleaseId: session.stableRelease.releaseId,
5589
+ });
5590
+ return {
5591
+ ok: true as const,
5592
+ attempted: true,
5593
+ detail: 'Candidate B booted in isolation, survived a Recovery restart canary, passed MCP/Scheduler/Supervisor/Controller canaries, and is cutover-eligible; Stable A remains active',
5594
+ releaseSession: session,
5595
+ };
5596
+ } catch (error) {
5597
+ const detail = error instanceof Error ? error.message : String(error);
5598
+ const retirement = await stopReleaseSessionCandidateService(session, runCommand);
5599
+ const terminalDetail = retirement.ok
5600
+ ? detail
5601
+ : `${detail}; Candidate B retirement failed: ${retirement.detail}`;
5602
+ try {
5603
+ const latest = readReleaseSession(config.controllerHome, sessionId) ?? session;
5604
+ session = latest;
5605
+ if (retirement.ok && !['failed', 'rolled_back', 'known_good'].includes(latest.phase)) {
5606
+ session = advanceReleaseSession({
5607
+ controllerHome: config.controllerHome,
5608
+ sessionId,
5609
+ expectedRevision: latest.revision,
5610
+ phase: 'failed',
5611
+ receipts: [{ id: 'candidate_failed', kind: 'candidate_canary', summary: terminalDetail.slice(0, 500) }],
5612
+ });
5613
+ }
5614
+ if (retirement.ok && ['failed', 'rolled_back'].includes(session.phase)) {
5615
+ cleanupRetiredCandidateLane(config, session);
5616
+ }
5617
+ } catch { /* preserve original Candidate B failure */ }
5618
+ audit(config, 'release_session_candidate_failed', {
5619
+ sessionId,
5620
+ detail,
5621
+ candidateRetired: retirement.ok,
5622
+ candidateRetirementDetail: retirement.detail,
5623
+ });
5624
+ return { ok: false as const, attempted: true, detail: terminalDetail, releaseSession: session };
5625
+ }
5626
+ });
5627
+ if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
5628
+ return locked.value;
5629
+ }
5630
+
5631
+
5632
+ function terminalizeSupersededSoakingPredecessor(config: RecoveryConfig, successor: ReleaseSession): ReleaseSession | undefined {
5633
+ const inventory = listCompleteReleaseSessions(config.controllerHome);
5634
+ if (inventory.invalidSessionFiles.length > 0) {
5635
+ throw new Error(`RELEASE_SESSION_INVENTORY_INCOMPLETE: invalid=${inventory.invalidSessionFiles.join(',') || 'none'}`);
5636
+ }
5637
+ const predecessors = inventory.sessions.filter((existing) =>
5638
+ existing.sessionId !== successor.sessionId
5639
+ && existing.sourceRevision !== successor.sourceRevision
5640
+ && releaseSessionIsSoakingPredecessorOfStable(existing, successor.stableRelease));
5641
+ if (predecessors.length > 1) {
5642
+ throw new Error(`RELEASE_SESSION_MULTIPLE_SOAKING_PREDECESSORS: ${predecessors.map((entry) => entry.sessionId).join(',')}`);
5643
+ }
5644
+ const predecessor = predecessors[0];
5645
+ if (!predecessor) return undefined;
5646
+ const terminal = advanceReleaseSession({
5647
+ controllerHome: config.controllerHome,
5648
+ sessionId: predecessor.sessionId,
5649
+ expectedRevision: predecessor.revision,
5650
+ phase: 'failed',
5651
+ receipts: [{
5652
+ id: `successor_cutover:${successor.sessionId}`,
5653
+ kind: 'soak',
5654
+ summary: `superseded without known-good promotion after successor ${successor.sessionId} committed a cutover from Stable A ${successor.stableRelease.releaseId}`,
5655
+ }],
5656
+ });
5657
+ audit(config, 'release_session_soaking_predecessor_superseded', {
5658
+ predecessorSessionId: predecessor.sessionId,
5659
+ predecessorReleaseId: predecessor.candidateRelease?.releaseId,
5660
+ successorSessionId: successor.sessionId,
5661
+ successorReleaseId: successor.candidateRelease?.releaseId,
5662
+ });
5663
+ return terminal;
5664
+ }
5665
+
5666
+ export async function cutoverConfiguredRuntimeReleaseSession(
5667
+ config: RecoveryConfig,
5668
+ sessionId: string,
5669
+ requestId?: string,
5670
+ ): Promise<ConfiguredRuntimeActivationResult> {
5671
+ const locked = await withLock(config, {
5672
+ action: 'release_session_cutover',
5673
+ ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
5674
+ }, async (releaseSessionLock) => {
5675
+ const initialSession = readReleaseSession(config.controllerHome, sessionId);
5676
+ if (!initialSession) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
5677
+ let session: ReleaseSession = initialSession;
5678
+ const candidateRelease = session.candidateRelease;
5679
+ if (!candidateRelease) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED', releaseSession: session };
5680
+
5681
+ const reconcileCutoverAttempt = async (
5682
+ reason: string,
5683
+ activation?: RuntimeReleaseActivationResult,
5684
+ ): Promise<ConfiguredRuntimeActivationResult> => {
5685
+ // Activation may have durably captured the ReleaseSession rollback
5686
+ // transaction and advanced its CAS revision. Re-read the one semantic
5687
+ // authority before any post-cutover phase transition.
5688
+ session = readReleaseSession(config.controllerHome, sessionId) ?? session;
5689
+ const retirement = await stopReleaseSessionCandidateService(session);
5690
+ if (!retirement.ok) {
5691
+ audit(config, 'release_session_candidate_retirement_failed', {
5692
+ sessionId,
5693
+ phase: session.phase,
5694
+ candidateControllerHome: session.candidate.controllerHome,
5695
+ detail: retirement.detail,
5696
+ reconciliationReason: reason,
5697
+ });
5698
+ return {
5699
+ ok: false as const,
5700
+ attempted: true,
5701
+ detail: `${reason}; Candidate B retirement failed: ${retirement.detail}`,
5702
+ releaseSession: session,
5703
+ ...(activation ? { activation } : {}),
5704
+ };
5705
+ }
5706
+
5707
+ const reconciliationTimeoutMs = Math.max(
5708
+ 1_000,
5709
+ configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
5710
+ );
5711
+ const reconciliationDeadline = Date.now() + reconciliationTimeoutMs;
5712
+ let stableNow = observeRuntimeStatus(config.controllerHome);
5713
+ const stableIdentity = () => {
5714
+ const liveRuntimeIdentityVerified = stableNow.running && stableNow.ready && !stableNow.stale;
5715
+ return {
5716
+ candidateIsStable: Boolean(
5717
+ liveRuntimeIdentityVerified
5718
+ && stableNow.snapshot?.releaseId === candidateRelease.releaseId
5719
+ && stableNow.snapshot?.artifactIdentity === candidateRelease.artifactIdentity,
5720
+ ),
5721
+ originalStableRestored: Boolean(
5722
+ liveRuntimeIdentityVerified
5723
+ && stableNow.snapshot?.releaseId === session.stableRelease.releaseId
5724
+ && stableNow.snapshot?.artifactIdentity === session.stableRelease.artifactIdentity,
5725
+ ),
5726
+ };
5727
+ };
5728
+ let observedIdentity = stableIdentity();
5729
+ while (!observedIdentity.candidateIsStable && !observedIdentity.originalStableRestored && Date.now() < reconciliationDeadline) {
5730
+ await sleep(Math.min(500, Math.max(1, reconciliationDeadline - Date.now())));
5731
+ stableNow = observeRuntimeStatus(config.controllerHome);
5732
+ observedIdentity = stableIdentity();
5733
+ }
5734
+ const { candidateIsStable, originalStableRestored } = observedIdentity;
5735
+
5736
+ if (candidateIsStable) {
5737
+ if (session.phase === 'cutover_attempting') {
5738
+ session = advanceReleaseSession({
5739
+ controllerHome: config.controllerHome,
5740
+ sessionId,
5741
+ expectedRevision: session.revision,
5742
+ phase: 'cutover_committed',
5743
+ receipts: [{
5744
+ id: 'cutover',
5745
+ kind: 'cutover',
5746
+ summary: `Stable A runs verified Candidate B release ${candidateRelease.releaseId}; Candidate B retired before cutover commit`,
5747
+ }],
5748
+ });
5749
+ }
5750
+ // Stable A now owns the successor and its transaction can roll back to
5751
+ // the predecessor release. Only at this durable boundary may the older
5752
+ // soaking session surrender its independent rollback/acceptance path.
5753
+ terminalizeSupersededSoakingPredecessor(config, session);
5754
+ session = advanceReleaseSession({
5755
+ controllerHome: config.controllerHome,
5756
+ sessionId,
5757
+ expectedRevision: session.revision,
5758
+ phase: 'soaking',
5759
+ receipts: [{
5760
+ id: 'soak_started',
5761
+ kind: 'soak',
5762
+ summary: `cutover reconciled at ${new Date().toISOString()}; Candidate B is retired and known-good promotion remains gated on later stable verification`,
5763
+ }],
5764
+ });
5765
+ const cleanup = cleanupRetiredCandidateLane(config, session);
5766
+ audit(config, 'release_session_cutover_reconciled_committed', {
5767
+ sessionId,
5768
+ releaseId: candidateRelease.releaseId,
5769
+ candidateRetired: true,
5770
+ reason,
5771
+ });
5772
+ return {
5773
+ ok: true as const,
5774
+ attempted: true,
5775
+ detail: `ReleaseSession cutover reconciled to committed: Stable A runs the verified candidate artifact and Candidate B is retired${cleanup.ok ? '' : `; Candidate B cleanup failed: ${cleanup.detail}`}`,
5776
+ releaseSession: session,
5777
+ ...(activation ? { activation } : {}),
5778
+ };
5779
+ }
5780
+
5781
+ if (originalStableRestored) {
5782
+ session = advanceReleaseSession({
5783
+ controllerHome: config.controllerHome,
5784
+ sessionId,
5785
+ expectedRevision: session.revision,
5786
+ phase: 'rolled_back',
5787
+ receipts: [{
5788
+ id: 'rollback',
5789
+ kind: 'rollback',
5790
+ summary: `cutover did not commit; exact Stable A ${session.stableRelease.releaseId} is active and verified, and Candidate B is retired`,
5791
+ }],
5792
+ });
5793
+ const cleanup = cleanupRetiredCandidateLane(config, session);
5794
+ audit(config, 'release_session_cutover_reconciled_rolled_back', {
5795
+ sessionId,
5796
+ restoredStableReleaseId: session.stableRelease.releaseId,
5797
+ candidateRetired: true,
5798
+ reason,
5799
+ });
5800
+ return {
5801
+ ok: false as const,
5802
+ attempted: true,
5803
+ detail: `ReleaseSession cutover reconciled to rolled_back: exact Stable A is active and Candidate B is retired${cleanup.ok ? '' : `; Candidate B cleanup failed: ${cleanup.detail}`}`,
5804
+ releaseSession: session,
5805
+ ...(activation ? { activation } : {}),
5806
+ };
5807
+ }
5808
+
5809
+ session = advanceReleaseSession({
5810
+ controllerHome: config.controllerHome,
5811
+ sessionId,
5812
+ expectedRevision: session.revision,
5813
+ phase: 'failed',
5814
+ receipts: [{
5815
+ id: 'cutover_failed',
5816
+ kind: 'cutover',
5817
+ summary: `${reason}; Stable A outcome is not fully verified; Candidate B is retired`.slice(0, 500),
5818
+ }],
5819
+ });
5820
+ const cleanup = cleanupRetiredCandidateLane(config, session);
5821
+ audit(config, 'release_session_cutover_reconciled_failed', {
5822
+ sessionId,
5823
+ candidateReleaseId: candidateRelease.releaseId,
5824
+ candidateRetired: true,
5825
+ reason,
5826
+ observedActiveReleaseId: stableNow.snapshot?.releaseId,
5827
+ });
5828
+ return {
5829
+ ok: false as const,
5830
+ attempted: true,
5831
+ detail: `${reason}; Candidate B is retired but Stable A cutover/rollback outcome is not fully verified${cleanup.ok ? '' : `; Candidate B cleanup failed: ${cleanup.detail}`}`,
5832
+ releaseSession: session,
5833
+ ...(activation ? { activation } : {}),
5834
+ };
5835
+ };
5836
+
5837
+ if (session.phase === 'cutover_attempting' || session.phase === 'cutover_committed') {
5838
+ return reconcileCutoverAttempt(`resuming ReleaseSession from ${session.phase} without a second activation attempt`);
5839
+ }
5840
+ if (session.phase !== 'cutover_eligible') {
5841
+ return { ok: false as const, attempted: false, noOp: true, detail: `RELEASE_SESSION_CUTOVER_REQUIRES_ELIGIBLE: ${session.phase}`, releaseSession: session };
5842
+ }
5843
+
5844
+ try {
5845
+ assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
5846
+ const stableBefore = await verifyLocalRuntimeForReleaseCutover(config);
5847
+ if (!stableBefore.ok) {
5848
+ const reasonCodes = stableBefore.runtime.reasonCodes.join(',') || 'unknown';
5849
+ audit(config, 'release_session_cutover_precondition_deferred', {
5850
+ sessionId,
5851
+ detail: 'stable Runtime verification is temporarily unavailable before cutover',
5852
+ reasonCodes,
5853
+ });
5854
+ return {
5855
+ ok: false as const,
5856
+ attempted: false,
5857
+ noOp: true,
5858
+ detail: `RELEASE_SESSION_STABLE_RUNTIME_VERIFICATION_DEFERRED: ${reasonCodes}`,
5859
+ releaseSession: session,
5860
+ };
5861
+ }
5862
+ if (
5863
+ stableBefore.releases.active?.revision !== session.stableRelease.releaseId
5864
+ || stableBefore.releases.active?.artifactIdentity !== session.stableRelease.artifactIdentity
5865
+ ) throw new Error('RELEASE_SESSION_STABLE_RUNTIME_IDENTITY_CHANGED');
5866
+
5867
+ const candidateConfig = candidateRecoveryConfig(session);
5868
+ const candidateBefore = await verifyLocalRuntimeForReleaseCutover(candidateConfig);
5869
+ if (
5870
+ !candidateBefore.ok
5871
+ || candidateBefore.releases.active?.revision !== candidateRelease.releaseId
5872
+ || candidateBefore.releases.active?.artifactIdentity !== candidateRelease.artifactIdentity
5873
+ ) throw new Error('RELEASE_SESSION_CANDIDATE_NO_LONGER_VERIFIED');
5874
+ if (runtimeReleaseTreeSha256(dirname(candidateRelease.manifestPath)) !== candidateRelease.treeSha256) {
5875
+ throw new Error('RELEASE_SESSION_CANDIDATE_TREE_CHANGED');
5876
+ }
5877
+
5878
+ const promoted = promotePortableRuntimeRelease({
5879
+ sourceManifestPath: candidateRelease.manifestPath,
5880
+ targetControllerHome: config.controllerHome,
5881
+ expectedTreeSha256: candidateRelease.treeSha256,
5882
+ });
5883
+ if (
5884
+ promoted.releaseId !== candidateRelease.releaseId
5885
+ || promoted.artifactIdentity !== candidateRelease.artifactIdentity
5886
+ || promoted.manifestSha256 !== candidateRelease.manifestSha256
5887
+ || promoted.treeSha256 !== candidateRelease.treeSha256
5888
+ ) throw new Error('RELEASE_SESSION_PROMOTION_IDENTITY_MISMATCH');
5889
+
5890
+ // Promotion writes only a new immutable release tree. Stable A must still
5891
+ // be the exact frozen authority before we enter the one cutover attempt.
5892
+ assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
5893
+ const stableAfterPromotion = await verifyLocalRuntimeForReleaseCutover(config);
5894
+ if (!stableAfterPromotion.ok) {
5895
+ const reasonCodes = stableAfterPromotion.runtime.reasonCodes.join(',') || 'unknown';
5896
+ audit(config, 'release_session_cutover_post_promotion_verification_deferred', {
5897
+ sessionId,
5898
+ detail: 'stable Runtime verification is temporarily unavailable after immutable promotion and before activation',
5899
+ reasonCodes,
5900
+ });
5901
+ return {
5902
+ ok: false as const,
5903
+ attempted: false,
5904
+ noOp: true,
5905
+ detail: `RELEASE_SESSION_STABLE_POST_PROMOTION_VERIFICATION_DEFERRED: ${reasonCodes}`,
5906
+ releaseSession: session,
5907
+ };
5908
+ }
5909
+ if (
5910
+ stableAfterPromotion.releases.active?.revision !== session.stableRelease.releaseId
5911
+ || stableAfterPromotion.releases.active?.artifactIdentity !== session.stableRelease.artifactIdentity
5912
+ ) throw new Error('RELEASE_SESSION_STABLE_CHANGED_DURING_PROMOTION');
5913
+
5914
+ // Connector quietness is a cutover invariant, not an automatic-release
5915
+ // policy. Every caller, including explicit/local-recovery ReleaseSession
5916
+ // progression, must prove that no MCP request/stream is active and that
5917
+ // remaining sessions have been idle for the bounded quiet window before
5918
+ // Stable A ownership can move. Otherwise a perfectly valid release switch
5919
+ // can tear down the ChatGPT turn that requested it.
5920
+ let readiness = await probeReleaseCutoverReadiness(config);
5921
+ // The post-promotion verification above may itself create MCP activity
5922
+ // on the primary Connector. Give that activity a bounded opportunity
5923
+ // to become quiet before deferring the session; otherwise every retry
5924
+ // re-verifies the Runtime, refreshes latestActivityAgeMs, and can defer
5925
+ // forever without ever reaching the cutover attempt.
5926
+ const quietWaitDeadline = Date.now() + AUTOMATIC_RELEASE_CUTOVER_QUIET_MS + 5_000;
5927
+ while (
5928
+ !readiness.ready
5929
+ && readiness.activePosts === 0
5930
+ && readiness.activeStreams === 0
5931
+ && readiness.initializing === 0
5932
+ && Number.isFinite(readiness.latestActivityAgeMs)
5933
+ && (readiness.latestActivityAgeMs ?? AUTOMATIC_RELEASE_CUTOVER_QUIET_MS) < AUTOMATIC_RELEASE_CUTOVER_QUIET_MS
5934
+ && Date.now() < quietWaitDeadline
5935
+ ) {
5936
+ const remainingMs = Math.max(
5937
+ 50,
5938
+ AUTOMATIC_RELEASE_CUTOVER_QUIET_MS - (readiness.latestActivityAgeMs ?? 0) + 25,
5939
+ );
5940
+ await sleep(Math.min(2_000, remainingMs));
5941
+ readiness = await probeReleaseCutoverReadiness(config);
5942
+ }
5943
+ if (!readiness.ready) {
5944
+ audit(config, 'release_session_cutover_deferred_for_mcp_activity', {
5945
+ sessionId,
5946
+ detail: readiness.detail,
5947
+ activeSessions: readiness.activeSessions,
5948
+ activePosts: readiness.activePosts,
5949
+ activeStreams: readiness.activeStreams,
5950
+ initializing: readiness.initializing,
5951
+ latestActivityAgeMs: readiness.latestActivityAgeMs,
5952
+ requestKind: requestId?.startsWith('recovery-auto-release:') ? 'automatic' : 'explicit',
5953
+ });
5954
+ return {
5955
+ ok: true as const,
5956
+ attempted: false,
5957
+ noOp: true,
5958
+ detail: `RELEASE_SESSION_CUTOVER_DEFERRED_FOR_MCP_ACTIVITY: ${readiness.detail}`,
5959
+ releaseSession: session,
5960
+ };
5961
+ }
5962
+
5963
+ session = advanceReleaseSession({
5964
+ controllerHome: config.controllerHome,
5965
+ sessionId,
5966
+ expectedRevision: session.revision,
5967
+ phase: 'cutover_attempting',
5968
+ receipts: [{
5969
+ id: 'cutover_attempt',
5970
+ kind: 'cutover',
5971
+ summary: `byte-identical Candidate B ${candidateRelease.releaseId} promoted to Stable A release storage; beginning the single fenced cutover attempt`,
5972
+ }],
5973
+ });
5974
+
5975
+ const activation = await activateRuntimeReleaseInternal(config, promoted.manifestPath, {}, {
5976
+ ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
5977
+ expectedAuthorityRevision: session.stableRelease.authorityRevision,
5978
+ expectedActiveReleaseId: session.stableRelease.releaseId,
5979
+ releaseSessionId: session.sessionId,
5980
+ }, releaseSessionLock);
5981
+
5982
+ return reconcileCutoverAttempt(
5983
+ activation.ok
5984
+ ? 'cutover activation completed; reconciling canonical Runtime identity before phase progression'
5985
+ : `cutover activation returned failure: ${activation.detail}`,
5986
+ activation,
5987
+ );
5988
+ } catch (error) {
5989
+ const detail = error instanceof Error ? error.message : String(error);
5990
+ const latest = readReleaseSession(config.controllerHome, sessionId) ?? session;
5991
+ session = latest;
5992
+ if (latest.phase === 'cutover_attempting' || latest.phase === 'cutover_committed') {
5993
+ return reconcileCutoverAttempt(`cutover interrupted after entering ${latest.phase}: ${detail}`);
5994
+ }
5995
+ if (latest.phase === 'cutover_eligible') {
5996
+ const retirement = await stopReleaseSessionCandidateService(latest);
5997
+ if (!retirement.ok) {
5998
+ audit(config, 'release_session_candidate_retirement_failed', {
5999
+ sessionId,
6000
+ phase: latest.phase,
6001
+ candidateControllerHome: latest.candidate.controllerHome,
6002
+ detail: retirement.detail,
6003
+ cutoverError: detail,
6004
+ });
6005
+ return {
6006
+ ok: false as const,
6007
+ attempted: false,
6008
+ noOp: true,
6009
+ detail: `${detail}; Candidate B retirement failed: ${retirement.detail}`,
6010
+ releaseSession: latest,
6011
+ };
6012
+ }
6013
+ try {
6014
+ session = advanceReleaseSession({
6015
+ controllerHome: config.controllerHome,
6016
+ sessionId,
6017
+ expectedRevision: latest.revision,
6018
+ phase: 'failed',
6019
+ receipts: [{
6020
+ id: 'cutover_precondition_failed',
6021
+ kind: 'cutover',
6022
+ summary: `${detail}; Candidate B retired`.slice(0, 500),
6023
+ }],
6024
+ });
6025
+ cleanupRetiredCandidateLane(config, session);
6026
+ } catch { /* preserve original cutover precondition failure */ }
6027
+ audit(config, 'release_session_cutover_precondition_failed', {
6028
+ sessionId,
6029
+ detail,
6030
+ candidateRetired: true,
6031
+ });
6032
+ }
6033
+ return { ok: false as const, attempted: false, noOp: true, detail, releaseSession: session };
6034
+ }
6035
+ });
6036
+ if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
6037
+ return locked.value;
6038
+ }
6039
+
6040
+
6041
+ async function cancelReleaseSessionUnderLock(
6042
+ config: RecoveryConfig,
6043
+ initialSession: ReleaseSession,
6044
+ reason: string,
6045
+ ): Promise<ConfiguredRuntimeActivationResult> {
6046
+ let session = initialSession;
6047
+ if (session.phase === 'failed') {
6048
+ return { ok: true, attempted: false, noOp: true, detail: 'ReleaseSession is already terminal failed', releaseSession: session };
6049
+ }
6050
+ if (['cutover_attempting', 'cutover_committed', 'soaking'].includes(session.phase)) {
6051
+ return {
6052
+ ok: false,
6053
+ attempted: false,
6054
+ noOp: true,
6055
+ detail: `RELEASE_SESSION_CANCEL_AFTER_CUTOVER_FORBIDDEN: ${session.phase}; use exact ReleaseSession rollback/soak resolution`,
6056
+ releaseSession: session,
6057
+ };
6058
+ }
6059
+ if (session.phase === 'known_good' || session.phase === 'rolled_back') {
6060
+ return {
6061
+ ok: false,
6062
+ attempted: false,
6063
+ noOp: true,
6064
+ detail: `RELEASE_SESSION_CANCEL_TERMINAL: ${session.phase}`,
6065
+ releaseSession: session,
6066
+ };
6067
+ }
6068
+
6069
+ const priorPhase = session.phase;
6070
+ if (['candidate_booted', 'candidate_verified', 'cutover_eligible'].includes(session.phase)) {
6071
+ const retirement = await stopReleaseSessionCandidateService(session);
6072
+ if (!retirement.ok) {
6073
+ return {
6074
+ ok: false,
6075
+ attempted: true,
6076
+ detail: `RELEASE_SESSION_CANCEL_CANDIDATE_RETIRE_FAILED: ${retirement.detail}`,
6077
+ releaseSession: session,
6078
+ };
6079
+ }
6080
+ }
6081
+
6082
+ session = advanceReleaseSession({
6083
+ controllerHome: config.controllerHome,
6084
+ sessionId: session.sessionId,
6085
+ expectedRevision: session.revision,
6086
+ phase: 'failed',
6087
+ receipts: [{
6088
+ id: 'candidate_cancelled',
6089
+ kind: 'candidate_canary',
6090
+ summary: `Candidate B was retired before cutover: ${reason}`.slice(0, 500),
6091
+ }],
6092
+ });
6093
+ const cleanup = cleanupRetiredCandidateLane(config, session);
6094
+ audit(config, cleanup.ok ? 'release_session_cancelled' : 'release_session_cancelled_cleanup_failed', {
6095
+ sessionId: session.sessionId,
6096
+ priorPhase,
6097
+ candidateControllerHome: session.candidate.controllerHome,
6098
+ reason,
6099
+ cleanupDetail: cleanup.detail,
6100
+ });
6101
+ return {
6102
+ ok: cleanup.ok,
6103
+ attempted: true,
6104
+ detail: cleanup.ok
6105
+ ? 'ReleaseSession Candidate B was terminalized failed and its isolated lane was retired before cutover; Stable A remained unchanged'
6106
+ : `ReleaseSession was terminalized failed but Candidate B lane cleanup failed: ${cleanup.detail}`,
6107
+ releaseSession: session,
6108
+ };
6109
+ }
6110
+
6111
+ export async function cancelConfiguredRuntimeReleaseSession(
6112
+ config: RecoveryConfig,
6113
+ sessionId: string,
6114
+ requestId?: string,
6115
+ ): Promise<ConfiguredRuntimeActivationResult> {
6116
+ const locked = await withLock(config, {
6117
+ action: 'release_session_cancel',
6118
+ ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
3198
6119
  }, async () => {
6120
+ const session = readReleaseSession(config.controllerHome, sessionId);
6121
+ if (!session) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
6122
+ return cancelReleaseSessionUnderLock(config, session, 'explicit Recovery cancellation');
6123
+ });
6124
+ if (!locked.acquired) {
6125
+ return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
6126
+ }
6127
+ return locked.value;
6128
+ }
6129
+
6130
+
6131
+ function releaseSessionRollbackTransaction(
6132
+ config: RecoveryConfig,
6133
+ session: ReleaseSession,
6134
+ ): NonNullable<ReleaseSession['transaction']> {
6135
+ const authority = readRuntimeReleaseAuthority(config.controllerHome);
6136
+ const transaction = session.transaction;
6137
+ const candidateRelease = session.candidateRelease;
6138
+ const previous = authority?.previous;
6139
+ if (!authority || !transaction || !candidateRelease || !previous?.databaseBackup) {
6140
+ throw new Error('RELEASE_SESSION_ROLLBACK_TRANSACTION_REQUIRED');
6141
+ }
6142
+ if (
6143
+ transaction.candidateReleaseId !== candidateRelease.releaseId
6144
+ || authority.active.releaseId !== candidateRelease.releaseId
6145
+ || authority.active.artifactIdentity !== candidateRelease.artifactIdentity
6146
+ || previous.releaseId !== transaction.rollbackRelease.releaseId
6147
+ || previous.artifactIdentity !== transaction.rollbackRelease.artifactIdentity
6148
+ || previous.manifestSha256 !== transaction.rollbackRelease.manifestSha256
6149
+ || previous.databaseBackup.path !== transaction.rollbackRelease.databaseBackup?.path
6150
+ || transaction.rollbackRelease.releaseId !== session.stableRelease.releaseId
6151
+ || transaction.rollbackRelease.artifactIdentity !== session.stableRelease.artifactIdentity
6152
+ || transaction.rollbackRelease.manifestSha256 !== session.stableRelease.manifestSha256
6153
+ ) {
6154
+ throw new Error('RELEASE_SESSION_ROLLBACK_TRANSACTION_IDENTITY_MISMATCH');
6155
+ }
6156
+ return transaction;
6157
+ }
6158
+
6159
+ /**
6160
+ * A rollback can durably switch release authority before the process gets to
6161
+ * write the ReleaseSession terminal phase. This is sufficient proof only when
6162
+ * the authority operation, Stable A, Candidate B, and fenced transaction all
6163
+ * agree; a merely similar active/previous pair must not terminalize a session.
6164
+ */
6165
+ function releaseSessionRollbackAlreadyCommitted(
6166
+ session: ReleaseSession,
6167
+ authority: RuntimeReleaseAuthority | undefined,
6168
+ ): boolean {
6169
+ const candidateRelease = session.candidateRelease;
6170
+ const transaction = session.transaction;
6171
+ return Boolean(
6172
+ candidateRelease
6173
+ && transaction
6174
+ && authority?.operationId?.startsWith(`release-session-rollback:${session.sessionId}:`)
6175
+ && authority.active.releaseId === session.stableRelease.releaseId
6176
+ && authority.active.artifactIdentity === session.stableRelease.artifactIdentity
6177
+ && authority.active.manifestSha256 === session.stableRelease.manifestSha256
6178
+ && authority.previous?.releaseId === candidateRelease.releaseId
6179
+ && authority.previous?.artifactIdentity === candidateRelease.artifactIdentity
6180
+ && transaction.candidateReleaseId === candidateRelease.releaseId
6181
+ && transaction.rollbackRelease.releaseId === session.stableRelease.releaseId
6182
+ && transaction.rollbackRelease.artifactIdentity === session.stableRelease.artifactIdentity
6183
+ );
6184
+ }
6185
+
6186
+ export async function rollbackConfiguredRuntimeReleaseSession(
6187
+ config: RecoveryConfig,
6188
+ sessionId: string,
6189
+ dependencies: PrimaryRuntimeRecoveryDependencies = {},
6190
+ requestId?: string,
6191
+ ): Promise<ConfiguredRuntimeActivationResult> {
6192
+ const locked = await withLock(config, {
6193
+ action: 'release_session_rollback',
6194
+ ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
6195
+ }, async () => {
6196
+ let session = readReleaseSession(config.controllerHome, sessionId);
6197
+ if (!session) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
6198
+ if (session.phase === 'rolled_back') {
6199
+ return { ok: true as const, attempted: false, noOp: true, detail: 'ReleaseSession is already rolled back', releaseSession: session };
6200
+ }
6201
+ if (!['cutover_attempting', 'cutover_committed', 'soaking'].includes(session.phase)) {
6202
+ return {
6203
+ ok: false as const,
6204
+ attempted: false,
6205
+ noOp: true,
6206
+ detail: `RELEASE_SESSION_ROLLBACK_REQUIRES_CUTOVER: ${session.phase}`,
6207
+ releaseSession: session,
6208
+ };
6209
+ }
6210
+
6211
+ const candidateRelease = session.candidateRelease;
6212
+ const transactionBeforeRollback = session.transaction;
6213
+ const authorityBeforeRollback = readRuntimeReleaseAuthority(config.controllerHome);
6214
+ const rollbackAlreadyCommitted = releaseSessionRollbackAlreadyCommitted(
6215
+ session,
6216
+ authorityBeforeRollback,
6217
+ );
6218
+ if (rollbackAlreadyCommitted) {
6219
+ const live = observeRuntimeStatus(config.controllerHome);
6220
+ const stableIsLive = live.running && live.ready && !live.stale
6221
+ && live.snapshot?.releaseId === session.stableRelease.releaseId
6222
+ && live.snapshot?.artifactIdentity === session.stableRelease.artifactIdentity;
6223
+ if (!stableIsLive) {
6224
+ return {
6225
+ ok: false as const,
6226
+ attempted: false,
6227
+ noOp: true,
6228
+ detail: 'RELEASE_SESSION_ROLLBACK_COMMITTED_RUNTIME_NOT_YET_OBSERVED',
6229
+ releaseSession: session,
6230
+ };
6231
+ }
6232
+ session = advanceReleaseSession({
6233
+ controllerHome: config.controllerHome,
6234
+ sessionId,
6235
+ expectedRevision: session.revision,
6236
+ phase: 'rolled_back',
6237
+ receipts: [{
6238
+ id: 'rollback',
6239
+ kind: 'rollback',
6240
+ summary: `reconciled already-committed rollback ${authorityBeforeRollback!.operationId}; exact Stable A ${session.stableRelease.releaseId} is live and no rollback effect was replayed`,
6241
+ }],
6242
+ });
6243
+ audit(config, 'release_session_rollback_reconciled_committed', {
6244
+ sessionId,
6245
+ rollbackOperationId: authorityBeforeRollback!.operationId,
6246
+ restoredStableReleaseId: session.stableRelease.releaseId,
6247
+ candidateReleaseId: candidateRelease!.releaseId,
6248
+ });
6249
+ return {
6250
+ ok: true as const,
6251
+ attempted: false,
6252
+ noOp: true,
6253
+ detail: 'ReleaseSession reconciled the already-committed Stable A rollback from durable authority without replaying the rollback effect',
6254
+ releaseSession: session,
6255
+ };
6256
+ }
6257
+
6258
+ let transaction: NonNullable<ReleaseSession['transaction']>;
3199
6259
  try {
3200
- const staged = (dependencies.stage ?? stageRuntimeReleaseFromCandidateSource)({ controllerHome: config.controllerHome, sourceRoot, sourceRepositoryId });
3201
- assertRuntimeReleaseFiles(staged);
3202
- return { ok: true as const, staged };
6260
+ transaction = releaseSessionRollbackTransaction(config, session);
6261
+ } catch (error) {
6262
+ return {
6263
+ ok: false as const,
6264
+ attempted: false,
6265
+ noOp: true,
6266
+ detail: error instanceof Error ? error.message : String(error),
6267
+ releaseSession: session,
6268
+ };
6269
+ }
6270
+
6271
+ const platform = dependencies.platform ?? process.platform;
6272
+ const uid = await (dependencies.currentUid ?? currentUid)();
6273
+ const service = primaryRuntimeServiceOwner(config, platform, uid);
6274
+ if (!service) {
6275
+ return {
6276
+ ok: false as const,
6277
+ attempted: false,
6278
+ noOp: true,
6279
+ detail: `primary Forge Runtime ${configuredPrimaryRuntimeService(config, platform).platform} service is not installed for ${platform}`,
6280
+ releaseSession: session,
6281
+ };
6282
+ }
6283
+ const runCommand = dependencies.runCommand ?? command;
6284
+ const now = dependencies.now ?? Date.now;
6285
+ const wait = dependencies.sleep ?? sleep;
6286
+ const runtimeRunning = dependencies.runtimeRunning ?? ((value: RecoveryConfig) => observeRuntimeStatus(value.controllerHome).running);
6287
+ const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
6288
+ const observeLocal = dependencies.observeLocal ?? (dependencies.verifyLocal ? dependencies.verifyLocal : observePrimaryRuntimeActivationReadiness);
6289
+ const repairConnectorBinding = dependencies.repairPrimaryConnectorBinding
6290
+ ?? ((value: RecoveryConfig) => repairPrimaryConnectorBinding(value, platform));
6291
+
6292
+ const stopped = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
6293
+ if (!stopped.ok) {
6294
+ return { ok: false as const, attempted: true, detail: stopped.detail, releaseSession: session };
6295
+ }
6296
+
6297
+ let restored: RuntimeReleaseAuthority;
6298
+ let databaseRollbackDisposition: RuntimeDatabaseRollbackDisposition = 'preserved_unversioned_backup';
6299
+ try {
6300
+ const rollbackResult = rollbackRuntimeReleaseWithResult(config.controllerHome, `release-session-rollback:${session.sessionId}:${Date.now()}`);
6301
+ restored = rollbackResult.authority;
6302
+ databaseRollbackDisposition = rollbackResult.databaseDisposition;
6303
+ if (
6304
+ restored.active.releaseId !== session.stableRelease.releaseId
6305
+ || restored.active.artifactIdentity !== session.stableRelease.artifactIdentity
6306
+ || restored.active.manifestSha256 !== session.stableRelease.manifestSha256
6307
+ ) throw new Error('RELEASE_SESSION_ROLLBACK_STABLE_IDENTITY_MISMATCH');
3203
6308
  } catch (error) {
3204
- return { ok: false as const, detail: error instanceof Error ? error.message : 'Runtime release staging failed' };
6309
+ const restart = await rebindStartAndVerifyPrimaryRuntime({
6310
+ config,
6311
+ service,
6312
+ runCommand,
6313
+ now,
6314
+ wait,
6315
+ verifyLocal,
6316
+ observeLocal,
6317
+ ensureRuntimeLaunchContract: dependencies.ensureRuntimeLaunchContract,
6318
+ timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
6319
+ successDetail: 'ReleaseSession rollback transaction failed; current authoritative Runtime was restarted and verified',
6320
+ });
6321
+ return {
6322
+ ok: false as const,
6323
+ attempted: true,
6324
+ detail: `${error instanceof Error ? error.message : String(error)}; authoritative Runtime restart: ${restart.detail}`,
6325
+ releaseSession: session,
6326
+ };
3205
6327
  }
6328
+
6329
+ let rollbackConnectorBinding: { ok: boolean; attempted: boolean; noOp?: boolean; detail: string } | undefined;
6330
+ const restarted = await rebindStartAndVerifyPrimaryRuntime({
6331
+ config,
6332
+ service,
6333
+ runCommand,
6334
+ now,
6335
+ wait,
6336
+ verifyLocal,
6337
+ observeLocal,
6338
+ ensureRuntimeLaunchContract: dependencies.ensureRuntimeLaunchContract,
6339
+ contractFailureContext: 'after ReleaseSession rollback',
6340
+ timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
6341
+ successDetail: databaseRollbackDisposition === 'restored_backup'
6342
+ ? 'exact Stable A whole-Runtime release and unchanged SQLite backup restored, restarted, rebound, and verified'
6343
+ : 'exact Stable A Runtime release restored while newer live SQLite state was preserved, restarted, rebound, and verified',
6344
+ afterRuntimeReady: async () => {
6345
+ rollbackConnectorBinding = await repairConnectorBinding(config);
6346
+ return rollbackConnectorBinding.ok
6347
+ ? { ok: true, detail: rollbackConnectorBinding.detail }
6348
+ : { ok: false, detail: `Stable A persistent Connector binding failed after ReleaseSession rollback: ${rollbackConnectorBinding.detail}` };
6349
+ },
6350
+ });
6351
+
6352
+ const latest = readReleaseSession(config.controllerHome, sessionId) ?? session;
6353
+ session = advanceReleaseSession({
6354
+ controllerHome: config.controllerHome,
6355
+ sessionId,
6356
+ expectedRevision: latest.revision,
6357
+ phase: restarted.ok ? 'rolled_back' : 'failed',
6358
+ receipts: [{
6359
+ id: restarted.ok ? 'rollback' : 'rollback_failed',
6360
+ kind: 'rollback',
6361
+ summary: restarted.ok
6362
+ ? databaseRollbackDisposition === 'restored_backup'
6363
+ ? `exact Stable A ${latest.stableRelease.releaseId} and its unchanged SQLite backup restored from ReleaseSession activation transaction`
6364
+ : `exact Stable A ${latest.stableRelease.releaseId} restored while newer live SQLite state was preserved across rollback`
6365
+ : `Stable A authority restored but Runtime verification failed after rollback: ${restarted.detail}`.slice(0, 500),
6366
+ }],
6367
+ });
6368
+ audit(config, restarted.ok ? 'release_session_rolled_back' : 'release_session_rollback_restart_failed', {
6369
+ sessionId,
6370
+ releaseSessionTransactionOperationId: transaction.operationId,
6371
+ restoredStableReleaseId: restored.active.releaseId,
6372
+ connectorBindingRepaired: rollbackConnectorBinding?.ok === true,
6373
+ databaseRollbackDisposition,
6374
+ detail: restarted.detail,
6375
+ });
6376
+ return {
6377
+ ok: restarted.ok,
6378
+ attempted: true,
6379
+ detail: restarted.ok
6380
+ ? databaseRollbackDisposition === 'restored_backup'
6381
+ ? 'ReleaseSession rollback restored the exact frozen Stable A release, unchanged SQLite backup, service binding, and verified Runtime'
6382
+ : 'ReleaseSession rollback restored the exact frozen Stable A release while preserving newer live SQLite state, service binding, and verified Runtime'
6383
+ : `ReleaseSession restored Stable A authority but failed post-rollback Runtime verification: ${restarted.detail}`,
6384
+ releaseSession: session,
6385
+ };
3206
6386
  });
3207
- if (!stagedAttempt.acquired) {
3208
- return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(stagedAttempt.owner) };
6387
+ if (!locked.acquired) {
6388
+ return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
3209
6389
  }
3210
- if (!stagedAttempt.value.ok) {
3211
- audit(config, 'runtime_release_stage_failed', { sourceRoot, detail: stagedAttempt.value.detail });
3212
- return { ok: false, attempted: false, noOp: true, detail: stagedAttempt.value.detail };
6390
+ return locked.value;
6391
+ }
6392
+
6393
+
6394
+ export async function promoteConfiguredRuntimeReleaseSessionKnownGood(
6395
+ config: RecoveryConfig,
6396
+ sessionId: string,
6397
+ dependencies: RuntimeReleaseKnownGoodDependencies = {},
6398
+ requestId?: string,
6399
+ ): Promise<ConfiguredRuntimeActivationResult> {
6400
+ const initial = readReleaseSession(config.controllerHome, sessionId);
6401
+ if (!initial) return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
6402
+ if (initial.phase !== 'soaking') {
6403
+ return { ok: false, attempted: false, noOp: true, detail: `RELEASE_SESSION_KNOWN_GOOD_REQUIRES_SOAKING: ${initial.phase}`, releaseSession: initial };
3213
6404
  }
3214
- const staged: StagedRuntimeRelease = stagedAttempt.value.staged;
3215
- const activationGuard: RuntimeReleaseActivationGuard = {
3216
- ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
3217
- expectedAuthorityRevision: expectedAuthority?.revision ?? null,
3218
- expectedActiveReleaseId: expectedAuthority?.active.releaseId ?? null,
3219
- };
3220
- const activation = dependencies.activate
3221
- ? await dependencies.activate(config, staged.manifestPath, activationGuard)
3222
- : await activateRuntimeRelease(config, staged.manifestPath, {}, activationGuard);
3223
- audit(config, activation.ok ? 'runtime_release_stage_and_activate_succeeded' : 'runtime_release_stage_and_activate_failed', {
3224
- sourceRoot,
6405
+ // A crash or caller timeout may happen after rollback authority is committed
6406
+ // but before its ReleaseSession receipt is written. Reconcile that terminal
6407
+ // fact before candidate identity or any expensive whole-Runtime acceptance
6408
+ // work; Candidate B is no longer active, so it can never become known-good.
6409
+ if (releaseSessionRollbackAlreadyCommitted(initial, readRuntimeReleaseAuthority(config.controllerHome))) {
6410
+ return rollbackConfiguredRuntimeReleaseSession(
6411
+ config,
6412
+ sessionId,
6413
+ dependencies.rollback ?? {},
6414
+ requestId?.trim() ? `${requestId.trim()}:reconcile-committed-rollback` : undefined,
6415
+ );
6416
+ }
6417
+ const candidateRelease = initial.candidateRelease;
6418
+ if (!candidateRelease) return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED', releaseSession: initial };
6419
+
6420
+ let attested: ReleaseEvidence;
6421
+ let continuationProof: WorkflowSupervisorContinuationProof;
6422
+ try {
6423
+ const active = activeAuthorityRelease(config);
6424
+ if (
6425
+ !active
6426
+ || active.revision !== candidateRelease.releaseId
6427
+ || active.artifactIdentity !== candidateRelease.artifactIdentity
6428
+ || active.manifestSha256 !== candidateRelease.manifestSha256
6429
+ ) throw new Error('RELEASE_SESSION_SOAK_RUNTIME_IDENTITY_MISMATCH');
6430
+
6431
+ const activeAuthority = readRuntimeReleaseAuthority(config.controllerHome);
6432
+ if (
6433
+ !activeAuthority
6434
+ || activeAuthority.active.releaseId !== candidateRelease.releaseId
6435
+ || activeAuthority.active.artifactIdentity !== candidateRelease.artifactIdentity
6436
+ || activeAuthority.active.manifestSha256 !== candidateRelease.manifestSha256
6437
+ ) throw new Error('RELEASE_SESSION_SOAK_RUNTIME_IDENTITY_MISMATCH');
6438
+ try {
6439
+ const proof = await (dependencies.continuationProof ?? getWorkflowSupervisorContinuationProof)(
6440
+ resolveWorkflowSupervisorForgeHome(config.controllerHome),
6441
+ {
6442
+ ...(candidateRelease.sourceRepositoryId ? { repoId: candidateRelease.sourceRepositoryId } : {}),
6443
+ activeReleaseId: candidateRelease.releaseId,
6444
+ notBefore: activeAuthority.committedAt,
6445
+ },
6446
+ );
6447
+ if (!proof) throw new Error('RELEASE_SESSION_AUTONOMOUS_CONTINUATION_PROOF_MISSING');
6448
+ continuationProof = proof;
6449
+ } catch (error) {
6450
+ if (error instanceof Error && error.message === 'RELEASE_SESSION_AUTONOMOUS_CONTINUATION_PROOF_MISSING') throw error;
6451
+ throw new Error(`RELEASE_SESSION_AUTONOMOUS_CONTINUATION_PROOF_UNAVAILABLE: ${error instanceof Error ? error.message : String(error)}`);
6452
+ }
6453
+
6454
+ const existingAttestation = matchingKnownGood(config, active);
6455
+ if (existingAttestation) {
6456
+ // A known-good attestation is durable release authority, not a transient
6457
+ // observation. Crash/retry reconciliation must reuse it instead of
6458
+ // replaying performance observation and public MCP probes. The live
6459
+ // Runtime still has to own the exact same release and remain locally
6460
+ // live/ready/non-stale before the ReleaseSession can terminalize.
6461
+ if (!liveRuntimeOwnsRelease(config, existingAttestation)) {
6462
+ throw new Error('RELEASE_SESSION_ATTESTED_RUNTIME_NOT_CURRENT');
6463
+ }
6464
+ attested = existingAttestation;
6465
+ } else {
6466
+ const before = await verifyStableRuntime(config);
6467
+ if (!before.ok) throw new Error('RELEASE_SESSION_SOAK_RUNTIME_VERIFY_FAILED');
6468
+ if (!sameReleaseEvidenceIdentity(before.releases.active, active)) {
6469
+ throw new Error('RELEASE_SESSION_SOAK_RUNTIME_IDENTITY_MISMATCH');
6470
+ }
6471
+ attested = await attestKnownGood(config, dependencies);
6472
+ }
6473
+ if (
6474
+ attested.revision !== candidateRelease.releaseId
6475
+ || attested.artifactIdentity !== candidateRelease.artifactIdentity
6476
+ || attested.manifestSha256 !== candidateRelease.manifestSha256
6477
+ || !attested.recoveryBundle
6478
+ ) throw new Error('RELEASE_SESSION_KNOWN_GOOD_ATTESTATION_IDENTITY_MISMATCH');
6479
+ inspectKnownGoodRecoveryBundle(config.controllerHome, attested);
6480
+ } catch (error) {
6481
+ const detail = error instanceof Error ? error.message : String(error);
6482
+ audit(config, 'release_session_known_good_failed', { sessionId, detail });
6483
+
6484
+ // A hard runaway-CPU rejection is terminal acceptance evidence, not a
6485
+ // transient observation failure. Keeping Candidate B live in `soaking`
6486
+ // caused the automatic release driver to launch another full 60-second
6487
+ // sampler on every reconciliation cycle. Restore the exact frozen Stable A
6488
+ // through the existing ReleaseSession rollback transaction instead. The
6489
+ // coordinator will then treat this source revision as already attempted and
6490
+ // will not autonomously replay it until source authority changes.
6491
+ if (/^RECOVERY_PERFORMANCE_REJECTED:/.test(detail)) {
6492
+ const rollback = await rollbackConfiguredRuntimeReleaseSession(
6493
+ config,
6494
+ sessionId,
6495
+ dependencies.rollback ?? {},
6496
+ requestId?.trim() ? `${requestId.trim()}:performance-rejected` : undefined,
6497
+ );
6498
+ audit(config, 'release_session_known_good_performance_rejected', {
6499
+ sessionId,
6500
+ detail,
6501
+ rollbackOk: rollback.ok,
6502
+ rollbackDetail: rollback.detail,
6503
+ });
6504
+ return {
6505
+ ok: false,
6506
+ attempted: true,
6507
+ detail: rollback.ok
6508
+ ? `${detail}; Candidate B rejected by the Recovery runaway-CPU gate and Stable A restored`
6509
+ : `${detail}; automatic Stable A rollback failed: ${rollback.detail}`,
6510
+ releaseSession: rollback.releaseSession ?? readReleaseSession(config.controllerHome, sessionId) ?? initial,
6511
+ };
6512
+ }
6513
+
6514
+ return { ok: false, attempted: true, detail, releaseSession: readReleaseSession(config.controllerHome, sessionId) ?? initial };
6515
+ }
6516
+
6517
+ const locked = await withLock(config, {
6518
+ action: 'release_session_known_good',
3225
6519
  ...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
3226
- expectedAuthorityRevision: activationGuard.expectedAuthorityRevision,
3227
- expectedActiveReleaseId: activationGuard.expectedActiveReleaseId,
3228
- releaseId: staged.releaseId,
3229
- sourceCommit: staged.sourceCommit,
3230
- activationAttempted: activation.attempted,
3231
- activationDetail: activation.detail,
6520
+ }, async () => {
6521
+ let session = readReleaseSession(config.controllerHome, sessionId);
6522
+ if (!session) return { ok: false as const, attempted: true, detail: 'RELEASE_SESSION_MISSING' };
6523
+ if (session.phase !== 'soaking') {
6524
+ return { ok: false as const, attempted: true, detail: `RELEASE_SESSION_CHANGED_DURING_KNOWN_GOOD_ATTESTATION: ${session.phase}`, releaseSession: session };
6525
+ }
6526
+ // The expensive independent verification and performance observation are
6527
+ // already embodied by `attested`. Re-running the full network probe suite
6528
+ // here created a split-brain transaction: the durable recovery bundle could
6529
+ // be published successfully, then one transient Gateway/MCP timeout left the
6530
+ // ReleaseSession permanently soaking. Under the mutation lock, only fence
6531
+ // the identity that could invalidate that attestation.
6532
+ if (!liveRuntimeOwnsRelease(config, attested)) {
6533
+ return { ok: false as const, attempted: true, detail: 'RELEASE_SESSION_RUNTIME_CHANGED_AFTER_KNOWN_GOOD_ATTESTATION', releaseSession: session };
6534
+ }
6535
+ try {
6536
+ releaseSessionRollbackTransaction(config, session);
6537
+ } catch (error) {
6538
+ return {
6539
+ ok: false as const,
6540
+ attempted: true,
6541
+ detail: error instanceof Error ? error.message : String(error),
6542
+ releaseSession: session,
6543
+ };
6544
+ }
6545
+ session = advanceReleaseSession({
6546
+ controllerHome: config.controllerHome,
6547
+ sessionId,
6548
+ expectedRevision: session.revision,
6549
+ phase: 'known_good',
6550
+ receipts: [
6551
+ {
6552
+ id: 'supervisor_continuation_proof',
6553
+ kind: 'soak',
6554
+ summary: `Supervisor task ${continuationProof.taskId} proved ${continuationProof.actions.join(' -> ')} on active release ${continuationProof.activeReleaseId} across Runtime instances ${continuationProof.runtimeInstanceIds.join(',')}`,
6555
+ },
6556
+ {
6557
+ id: 'known_good',
6558
+ kind: 'known_good',
6559
+ summary: `release ${attested.revision} passed soak/performance observation and owns recoverable bundle ${attested.recoveryBundle!.attestationId}`,
6560
+ },
6561
+ ],
6562
+ });
6563
+ audit(config, 'release_session_known_good', {
6564
+ sessionId,
6565
+ releaseId: attested.revision,
6566
+ attestationId: attested.recoveryBundle!.attestationId,
6567
+ releaseAuthorityRevision: attested.releaseAuthorityRevision,
6568
+ releaseSessionTransactionOperationId: session.transaction?.operationId,
6569
+ });
6570
+ return {
6571
+ ok: true as const,
6572
+ attempted: true,
6573
+ detail: 'ReleaseSession became known-good only after full Runtime verification, performance observation, and recoverable release+SQLite+service bundle attestation',
6574
+ releaseSession: session,
6575
+ };
3232
6576
  });
3233
- return {
3234
- ok: activation.ok,
3235
- attempted: activation.attempted,
3236
- noOp: activation.noOp,
3237
- detail: activation.ok ? 'configured Runtime source was staged as one immutable release and activated through standalone Recovery' : activation.detail,
3238
- staged,
3239
- activation,
3240
- };
6577
+ if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), releaseSession: initial };
6578
+ return locked.value;
6579
+ }
6580
+
6581
+ export interface ConfiguredRuntimeActivationDependencies {
6582
+ stage?: typeof stageRuntimeReleaseFromCandidateSource;
6583
+ activate?: (config: RecoveryConfig, manifestPath: string, guard: RuntimeReleaseActivationGuard) => Promise<RuntimeReleaseActivationResult>;
6584
+ }
6585
+
6586
+ export async function stageAndActivateConfiguredRuntimeRelease(
6587
+ config: RecoveryConfig,
6588
+ dependencies: ConfiguredRuntimeActivationDependencies = {},
6589
+ requestId?: string,
6590
+ ): Promise<ConfiguredRuntimeActivationResult> {
6591
+ // Frozen-client compatibility alias. It deliberately stops at Candidate B
6592
+ // build/session preparation. Activation is a separate ReleaseSession phase.
6593
+ return prepareConfiguredRuntimeReleaseSession(config, dependencies, requestId);
3241
6594
  }
3242
6595
 
3243
6596
  function tunnelLaunchdService(configured: RecoveryTunnelServiceConfig | undefined, uid: number): LaunchdService | undefined {
@@ -3276,9 +6629,26 @@ function tunnelRepairAllowed(config: RecoveryConfig, now: number): boolean {
3276
6629
  return typeof prior?.lastAttemptAt !== 'number' || now - prior.lastAttemptAt >= cooldownMs;
3277
6630
  }
3278
6631
 
6632
+ async function verifyRecoveryTunnelRepairSurface(config: RecoveryConfig): Promise<VerifyResult> {
6633
+ // Recovery tunnel repair is a bootstrap control-plane operation. It must not
6634
+ // depend on the primary public MCP/Connector transport that Recovery exists
6635
+ // to repair around, and it must stay bounded: the repair runs inside the same
6636
+ // process that serves the Recovery gateway, so strict whole-Runtime or
6637
+ // known-good bundle verification here starves that gateway, delays the tunnel's
6638
+ // own OAuth discovery, and turns a repair into a multi-minute Recovery outage.
6639
+ // Observe canonical Runtime authority/status plus the local Recovery gateway
6640
+ // and the dedicated Recovery transport only.
6641
+ return observeBoundedRuntimeHealth(config, createRecoveryHttpTransport(config.controllerHome), {
6642
+ includePrimaryConnectorLocal: false,
6643
+ });
6644
+ }
6645
+
3279
6646
  export async function repairPublicTunnel(config: RecoveryConfig, dependencies: PublicTunnelRepairDependencies = {}): Promise<PublicTunnelRepairResult> {
3280
- const verify = dependencies.verify ?? verifyStableRuntime;
3281
- const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
6647
+ const verify = dependencies.verify ?? verifyRecoveryTunnelRepairSurface;
6648
+ // The repair runs in-process with the Recovery gateway it protects, so the
6649
+ // local precondition uses the same bounded surface instead of strict
6650
+ // whole-Runtime verification.
6651
+ const verifyLocal = dependencies.verifyLocal ?? verifyRecoveryTunnelRepairSurface;
3282
6652
  const now = dependencies.now ?? Date.now;
3283
6653
  const wait = dependencies.sleep ?? sleep;
3284
6654
  const runCommand = dependencies.runCommand ?? command;
@@ -3387,7 +6757,9 @@ export async function repairPublicTunnel(config: RecoveryConfig, dependencies: P
3387
6757
  audit(config, 'public_tunnel_restart_failed', { serviceLabel, serviceTarget, detail });
3388
6758
  return { ok: false, attempted: false, noOp: true, detail, serviceLabel, serviceTarget, verify: before, localVerify: localBefore };
3389
6759
  }
3390
- const connected = await runCommand('tunnel-client', args, Math.max(30_000, configured.postRestartVerifyTimeoutMs ?? 20_000));
6760
+ const connected = await runCommand('tunnel-client', args, Math.max(30_000, configured.postRestartVerifyTimeoutMs ?? 20_000), {
6761
+ cwd: tunnelClientWorkingDirectory(config.controllerHome),
6762
+ });
3391
6763
  if (!connected.ok) {
3392
6764
  const detail = `OpenAI Secure MCP Tunnel runtime connect failed: ${connected.stderr || connected.stdout || connected.status}`;
3393
6765
  audit(config, 'public_tunnel_restart_failed', { serviceLabel, serviceTarget, detail });
@@ -3453,8 +6825,8 @@ export interface RecoveryWatchdogRestartDependencies extends RecoveryRoleRestart
3453
6825
  probeWatchdog?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
3454
6826
  }
3455
6827
 
3456
- function recoveryRoleLaunchdService(config: RecoveryConfig, uid: number, role: 'gateway' | 'watchdog'): LaunchdService {
3457
- const label = role === 'gateway' ? RECOVERY_GATEWAY_LABEL : RECOVERY_WATCHDOG_LABEL;
6828
+ function recoveryRoleLaunchdService(config: RecoveryConfig, uid: number, _role: 'gateway'): LaunchdService {
6829
+ const label = RECOVERY_DAEMON_LABEL;
3458
6830
  const generated = join(recoveryRoot(config), 'launchd', `${label}.plist`);
3459
6831
  const installed = join(homedir(), 'Library', 'LaunchAgents', `${label}.plist`);
3460
6832
  return {
@@ -3471,19 +6843,14 @@ async function probeRecoveryGateway(config: RecoveryConfig): Promise<{ ok: boole
3471
6843
  return probe(createRecoveryHttpTransport(config.controllerHome), `http://${config.gateway.host}:${config.gateway.port}/health`);
3472
6844
  }
3473
6845
 
3474
- async function probeRecoveryWatchdog(config: RecoveryConfig): Promise<{ ok: boolean; detail: string }> {
3475
- const health = observeRecoveryWatchdogHealth(config.controllerHome);
3476
- return { ok: health.ok, detail: health.detail };
3477
- }
3478
-
3479
6846
  async function restartRecoveryRole(input: {
3480
6847
  config: RecoveryConfig;
3481
- role: 'gateway' | 'watchdog';
3482
- action: 'restart_recovery_gateway' | 'restart_recovery_watchdog';
6848
+ role: 'gateway';
6849
+ action: 'restart_recovery_gateway';
3483
6850
  check: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
3484
6851
  dependencies: RecoveryRoleRestartDependencies;
3485
6852
  }): Promise<RecoveryGatewayRestartResult> {
3486
- const display = input.role === 'gateway' ? 'Recovery Gateway' : 'Recovery Watchdog';
6853
+ const display = 'Recovery daemon';
3487
6854
  if ((input.dependencies.platform ?? process.platform) !== 'darwin') return { ok: false, attempted: false, noOp: true, detail: `${display} launchd restart is only supported on macOS` };
3488
6855
  const initial = await input.check(input.config);
3489
6856
  if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: `${display} is already healthy` };
@@ -3491,7 +6858,7 @@ async function restartRecoveryRole(input: {
3491
6858
  if (uid === undefined) return { ok: false, attempted: false, noOp: true, detail: `${display} launchd UID is unavailable` };
3492
6859
  const service = recoveryRoleLaunchdService(input.config, uid, input.role);
3493
6860
  if (!existsSync(service.plistPath)) return { ok: false, attempted: false, noOp: true, detail: `${display} launchd plist is missing: ${service.plistPath}`, serviceTarget: service.target };
3494
- const eventPrefix = input.role === 'gateway' ? 'recovery_gateway' : 'recovery_watchdog';
6861
+ const eventPrefix = 'recovery_daemon';
3495
6862
  const locked = await withLock(input.config, { action: input.action }, async () => {
3496
6863
  const before = await input.check(input.config);
3497
6864
  if (before.ok) return { ok: true, attempted: false, noOp: true, detail: `${display} recovered before restart`, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
@@ -3533,19 +6900,6 @@ export async function restartRecoveryGateway(
3533
6900
  });
3534
6901
  }
3535
6902
 
3536
- export async function restartRecoveryWatchdog(
3537
- config: RecoveryConfig,
3538
- dependencies: RecoveryWatchdogRestartDependencies = {},
3539
- ): Promise<RecoveryWatchdogRestartResult> {
3540
- return restartRecoveryRole({
3541
- config,
3542
- role: 'watchdog',
3543
- action: 'restart_recovery_watchdog',
3544
- check: dependencies.probeWatchdog ?? probeRecoveryWatchdog,
3545
- dependencies,
3546
- });
3547
- }
3548
-
3549
6903
  function pidAlive(pid: number | undefined): boolean {
3550
6904
  if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 0) return false;
3551
6905
  try { process.kill(pid, 0); return true; } catch { return false; }
@@ -3565,7 +6919,7 @@ export function gatewayToken(config: RecoveryConfig): string | undefined {
3565
6919
  export function initializeStandaloneRecovery(
3566
6920
  controllerHome: string,
3567
6921
  port = 8787,
3568
- extensions: Partial<Pick<RecoveryConfig, 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryPublicTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryRuntimeSourceRepositoryId' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
6922
+ extensions: Partial<Pick<RecoveryConfig, 'installProfile' | 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryPublicTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryRuntimeSourceRepositoryId' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
3569
6923
  ): RecoveryConfig {
3570
6924
  const root = resolve(controllerHome);
3571
6925
  const tokenPath = join(root, 'recovery', 'config', 'gateway-token.json');
@@ -3604,27 +6958,17 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
3604
6958
  now,
3605
6959
  watchdogRuntimeStartupGraceMs(config),
3606
6960
  );
3607
- // Five-second watchdog ticks must stay cheap while the system is healthy.
3608
- // Full MCP protocol verification serializes the entire tool surface, so run
3609
- // it periodically and immediately escalate to it when a local health probe
3610
- // fails. A live Runtime that was just started gets a bounded grace window so
3611
- // startup/reconciliation cannot itself trigger a Recovery restart storm.
3612
- // Explicit verify/attest/release operations remain strict by default.
3613
- const fullVerifyDue = !runtimeStartupGrace
3614
- && (scopedPrior.lastFullVerifyAt === undefined || now - scopedPrior.lastFullVerifyAt >= 60_000);
3615
- let fullVerificationPerformed = fullVerifyDue;
3616
- let verified = await verifyStableRuntime(
3617
- config,
3618
- createRecoveryHttpTransport(config.controllerHome),
3619
- { probeMcpProtocol: fullVerifyDue },
3620
- );
3621
- let localVerify = await verifyLocalRuntime(config, { probeMcpProtocol: fullVerifyDue });
3622
- if (!localVerify.ok && !fullVerifyDue && !runtimeStartupGrace) {
3623
- verified = await verifyStableRuntime(config);
3624
- localVerify = await verifyLocalRuntime(config);
3625
- fullVerificationPerformed = true;
3626
- }
3627
- const lastFullVerifyAt = fullVerificationPerformed ? now : scopedPrior.lastFullVerifyAt;
6961
+ // Watchdog owns bounded health observation only. Never run strict Runtime or
6962
+ // known-good bundle verification from startup or the five-second tick, even
6963
+ // when the bounded tier degrades: those checks can hash/inspect multi-GB
6964
+ // recovery evidence and would make liveness depend on release-acceptance cost.
6965
+ // Explicit verify/attest/release/rollback operations remain the sole strict
6966
+ // recoverability boundaries. Keep the legacy timestamp as non-authoritative
6967
+ // compatibility state so existing persisted WatchdogState remains readable.
6968
+ const health = await observeWatchdogHealthTier(config);
6969
+ const verified = health;
6970
+ const localVerify = health;
6971
+ const lastFullVerifyAt = scopedPrior.lastFullVerifyAt ?? now;
3628
6972
  const activeMutation = liveRecoveryMutationLock(config);
3629
6973
  if (activeMutation) {
3630
6974
  const attributable = recoveryLockOwnerAttributable(activeMutation);
@@ -3654,7 +6998,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
3654
6998
  };
3655
6999
  return { state, decision: { action: 'degraded', reason }, verify: verified };
3656
7000
  }
3657
- if (runtimeStartupGrace && !localVerify.ok) {
7001
+ if (runtimeStartupGrace && !localVerify.runtime.ok) {
3658
7002
  const state: WatchdogState = {
3659
7003
  ...scopedPrior,
3660
7004
  failures: 0,
@@ -3670,8 +7014,24 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
3670
7014
  verify: verified,
3671
7015
  };
3672
7016
  }
7017
+ // Recovery is only usable when the transport the Recovery connector actually
7018
+ // reaches is itself verified. Treat the configured Recovery transport as part
7019
+ // of the recovery-health gate so a dead tunnel degrades the watchdog and
7020
+ // reaches the bounded `repair_public_tunnel` path instead of reporting healthy.
7021
+ const recoveryTransportHealthy = configuredRecoveryTunnel(config)?.platform === 'openai-secure-tunnel'
7022
+ ? verified.probes.recovery_tunnel_runtime?.ok === true
7023
+ : verified.probes.recovery_external_http?.ok !== false;
3673
7024
  const recoveryHealthy = verified.probes.recovery_gateway?.ok !== false
3674
- && verified.probes.recovery_external_http?.ok !== false;
7025
+ && recoveryTransportHealthy;
7026
+ const recoveryTunnelObservation = openAiTunnelObservationFromProbe(verified.probes.recovery_tunnel_runtime);
7027
+ if (isUpstreamOpenAiControlPlaneIncident(recoveryTunnelObservation) && scopedPrior.failures === 0) {
7028
+ await persistOpenAiTransportIncidentEvidence({
7029
+ config,
7030
+ verify: verified,
7031
+ runtimeObservation,
7032
+ recoveryTunnel: recoveryTunnelObservation!,
7033
+ });
7034
+ }
3675
7035
  const primaryRuntimeHealthy = canonicalRuntimeSafeForTargetedConnectorRecovery(localVerify);
3676
7036
  const primaryConnectorConfigured = Boolean(config.primaryConnectorService);
3677
7037
  const primaryConnectorLocalFailed = verified.probes.primary_connector_local?.ok === false;