@moretea-labs/forge 1.8.0 → 1.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +70 -7
- package/CHANGELOG.md +14 -0
- package/CLAUDE.md +17 -2
- package/CONTRIBUTING.md +3 -2
- package/THIRD_PARTY_NOTICES.md +1 -1
- package/adapters/chatgpt/browser-delivery-host.ts +7 -1
- package/adapters/chatgpt/browser-delivery-runtime.ts +529 -56
- package/adapters/chatgpt/controller-binding-store.ts +101 -22
- package/adapters/chatgpt/controller-round-host.ts +15 -25
- package/adapters/chatgpt/controller-round-settlement-store.ts +58 -4
- package/adapters/chatgpt/provider-delivery.ts +228 -1
- package/adapters/chatgpt/work-conversation-binding-store.ts +79 -53
- package/adapters/computer/chatgpt-extension-target.ts +463 -0
- package/adapters/computer/chatgpt-target-router.ts +103 -0
- package/adapters/computer/chrome-extension/background.js +385 -0
- package/adapters/computer/chrome-extension/content.js +378 -0
- package/adapters/computer/chrome-extension/core.js +24 -0
- package/adapters/computer/chrome-extension/manifest.json +30 -0
- package/adapters/computer/desktop-operator-discovery.ts +11 -31
- package/adapters/computer/interaction-target-authority.ts +142 -18
- package/adapters/mcp/auth.ts +0 -1
- package/adapters/mcp/instructions.ts +5 -4
- package/adapters/mcp/multi-repository.ts +3 -19
- package/adapters/mcp/oauth.ts +73 -0
- package/adapters/mcp/paths.ts +4 -1
- package/adapters/mcp/runtime-gateway/automation-receipt-adapter.ts +87 -0
- package/adapters/mcp/runtime-gateway/bounded-result-runtime.ts +62 -0
- package/adapters/mcp/runtime-gateway/cognition-settlement.ts +201 -0
- package/adapters/mcp/runtime-gateway/context-adapter.ts +411 -73
- package/adapters/mcp/runtime-gateway/context-pack-sidecar.ts +91 -0
- package/adapters/mcp/runtime-gateway/controller-authority-adapter.ts +20 -5
- package/adapters/mcp/runtime-gateway/core-capability-adapter.ts +258 -0
- package/adapters/mcp/runtime-gateway/engineering-preconditions.ts +10 -27
- package/adapters/mcp/runtime-gateway/engineering-tool-contract.ts +15 -0
- package/adapters/mcp/runtime-gateway/execution-tools.ts +8 -42
- package/adapters/mcp/runtime-gateway/gateway-execution-adapter.ts +2 -1
- package/adapters/mcp/runtime-gateway/plugin-adapter.ts +117 -45
- package/adapters/mcp/runtime-gateway/process-tools.ts +190 -22
- package/adapters/mcp/runtime-gateway/protected-computer-adapter.ts +16 -4
- package/adapters/mcp/runtime-gateway/recovery-adapter.ts +69 -52
- package/adapters/mcp/runtime-gateway/repository-compat-adapter.ts +4 -35
- package/adapters/mcp/runtime-gateway/result-adapter.ts +10 -4
- package/adapters/mcp/runtime-gateway/router.ts +18 -504
- package/adapters/mcp/runtime-gateway/routing-policy.ts +499 -0
- package/adapters/mcp/runtime-gateway/runtime-observation-adapter.ts +14 -65
- package/adapters/mcp/runtime-gateway/runtime-readiness-observation.ts +218 -0
- package/adapters/mcp/runtime-gateway/runtime-tool-definitions.ts +113 -173
- package/adapters/mcp/runtime-gateway/runtime-tools.ts +15 -5
- package/adapters/mcp/runtime-gateway/scheduler-adapter.ts +1 -0
- package/adapters/mcp/runtime-gateway/shared-adapter.ts +20 -0
- package/adapters/mcp/runtime-gateway/status-inbox-adapter.ts +358 -266
- package/adapters/mcp/runtime-gateway/typescript-navigation-sidecar.ts +136 -0
- package/adapters/mcp/runtime-gateway/work-adapter.ts +261 -3122
- package/adapters/mcp/runtime-gateway/work-controller-operations.ts +257 -0
- package/adapters/mcp/runtime-gateway/work-detail-projection.ts +60 -0
- package/adapters/mcp/runtime-gateway/work-input-wire-migration.ts +246 -0
- package/adapters/mcp/runtime-gateway/work-learning-operations.ts +145 -0
- package/adapters/mcp/runtime-gateway/work-list-projection.ts +38 -0
- package/adapters/mcp/runtime-gateway/work-plan-operations.ts +382 -0
- package/adapters/mcp/runtime-gateway/work-repair-adapter.ts +308 -0
- package/adapters/mcp/runtime-gateway/work-requirement-operations.ts +148 -0
- package/adapters/mcp/runtime-gateway/work-semantic-operations.ts +263 -0
- package/adapters/mcp/runtime-gateway/work-workflow-operations.ts +113 -0
- package/adapters/mcp/runtime-gateway/workflow-supervisor-adapter.ts +156 -0
- package/adapters/mcp/server.ts +67 -23
- package/adapters/mcp/setup.ts +1 -3
- package/adapters/mcp/tool-mapping/legacy-tool-service.ts +44 -170
- package/adapters/mcp/tool-mapping/post-finalize-work-attribution.ts +4 -9
- package/adapters/mcp/tool-mapping/repository-tools.ts +133 -142
- package/adapters/mcp/toolset-names.ts +5 -0
- package/adapters/mcp/toolset.ts +6 -5
- package/adapters/mcp/transports/http-observation.ts +365 -0
- package/adapters/mcp/transports/http.ts +154 -677
- package/adapters/mcp/transports/oauth-http.ts +322 -0
- package/adapters/mcp/transports/session-registry.ts +6 -0
- package/adapters/mcp/tunnels/openai-secure-tunnel.ts +89 -5
- package/assets/plugin-registry.v1.json +6 -4
- package/assets/reference-configs/agentic-development-flow.md +25 -0
- package/assets/reference-configs/global-working-rules.md +19 -0
- package/assets/skill-version.json +2 -2
- package/docs/architecture/CURRENT.md +111 -41
- package/docs/architecture/decisions/20260803-release-identity-binding-and-exit-policy.md +14 -5
- package/docs/architecture/decisions/20260823-immediate-controller-relay.md +6 -3
- package/docs/architecture/decisions/20260907-assistant-learning-loop.md +2 -0
- package/docs/architecture/decisions/20260907-controller-round-transition-policy.md +19 -4
- package/docs/architecture/decisions/20260916-workflow-supervisor-authority.md +249 -0
- package/docs/architecture/decisions/20260917-cognitive-memory-foundation.md +82 -0
- package/docs/architecture/decisions/20260918-stable-candidate-release-lifecycle.md +221 -0
- package/docs/architecture/decisions/20260919-autonomous-work-liveness.md +62 -0
- package/docs/architecture/decisions/20260924-thin-semantic-working-context.md +72 -0
- package/docs/architecture/decisions/20261001-supervisor-turn-authority.md +237 -0
- package/docs/architecture/decisions/20261005-forge-v3-desktop-client-boundary.md +77 -0
- package/docs/architecture/domains/verification.md +1 -0
- package/docs/architecture/domains/workflow-engine.md +6 -0
- package/docs/architecture/modules/controller-runtime/transport-lifecycle.md +1 -1
- package/docs/forge-plugin-management.md +2 -2
- package/docs/operations/controller-performance-and-502.md +17 -0
- package/docs/operations/controller-reliability-runbook.md +2 -2
- package/docs/operations/features.md +1 -1
- package/docs/operations/features.zh-CN.md +1 -1
- package/docs/operations/homebrew.md +2 -2
- package/docs/operations/mcp-tool-exposure.md +2 -0
- package/docs/operations/releasing.md +2 -0
- package/docs/operations/standalone-disaster-recovery.md +14 -6
- package/docs/wiki/Implementation.md +1 -1
- package/package.json +25 -14
- package/packages/kernel/cognition/api/index.ts +6 -0
- package/packages/kernel/cognition/application/activation-service.ts +451 -0
- package/packages/kernel/cognition/application/consolidation-service.ts +185 -0
- package/packages/kernel/cognition/application/legacy-experience-adapter.ts +38 -0
- package/packages/kernel/cognition/application/memory-service.ts +64 -0
- package/packages/kernel/cognition/domain/learning.ts +106 -0
- package/packages/kernel/cognition/domain/memory.ts +254 -0
- package/packages/kernel/controller/api/index.ts +1 -0
- package/packages/kernel/controller/application/continuation-service.ts +281 -0
- package/packages/kernel/controller/application/controller-round-service.ts +12 -0
- package/packages/kernel/controller/domain/controller-round-transition-policy.ts +242 -21
- package/packages/kernel/controller/domain/controller-round.ts +4 -1
- package/packages/kernel/controller/domain/execution-quality.ts +3 -1
- package/packages/kernel/controller/infrastructure/controller-round-store.ts +689 -103
- package/packages/kernel/controller/infrastructure/controller-session-store.ts +297 -71
- package/packages/kernel/controller/ports/controller-host.ts +2 -0
- package/packages/kernel/identity/api/index.ts +10 -0
- package/packages/kernel/identity/domain/activity-projection.ts +43 -0
- package/packages/kernel/identity/domain/capability-broker.ts +60 -0
- package/packages/kernel/identity/domain/grant.ts +72 -0
- package/packages/kernel/identity/domain/owned-resource.ts +116 -0
- package/packages/kernel/identity/domain/user-request.ts +93 -0
- package/packages/kernel/identity/infrastructure/activity-projection-store.ts +77 -0
- package/packages/kernel/identity/infrastructure/capability-broker.ts +81 -0
- package/packages/kernel/identity/infrastructure/grant-store.ts +284 -0
- package/packages/kernel/identity/infrastructure/owned-resource-store.ts +259 -0
- package/packages/kernel/identity/infrastructure/user-request-store.ts +182 -0
- package/packages/kernel/scheduler/api/index.ts +0 -2
- package/packages/kernel/scheduler/application/schedule-service.ts +3 -0
- package/packages/kernel/scheduler/application/settlement.ts +0 -15
- package/packages/kernel/scheduler/domain/schedule.ts +87 -0
- package/packages/kernel/scheduler/infrastructure/schedule-store.ts +153 -133
- package/packages/kernel/work/api/index.ts +3 -2
- package/packages/kernel/work/application/work-service.ts +17 -42
- package/packages/kernel/work/domain/current-task-lineage.ts +104 -0
- package/packages/kernel/work/domain/engineering-design.ts +10 -5
- package/packages/kernel/work/domain/engineering-profile.ts +8 -12
- package/packages/kernel/work/domain/evaluation-promotion-receipt.ts +186 -0
- package/packages/kernel/work/domain/execution-concurrency.ts +19 -26
- package/packages/kernel/work/domain/execution-snapshot.ts +0 -31
- package/packages/kernel/work/domain/implementation-review.ts +14 -331
- package/packages/kernel/work/domain/objective-graph.ts +119 -0
- package/packages/kernel/work/domain/project-engineering-contract.ts +23 -1
- package/packages/kernel/work/domain/repository-completion-receipt.ts +1 -1
- package/packages/kernel/work/domain/state-machine.ts +25 -63
- package/packages/kernel/work/domain/types.ts +57 -51
- package/packages/kernel/work/infrastructure/work-contract-store.ts +819 -728
- package/packages/kernel/work/ports/work-contract-store.ts +5 -2
- package/packages/plugin-runtime/browser/session-authority.ts +2 -1
- package/packages/plugin-runtime/computer/chatgpt-identity.ts +24 -0
- package/packages/plugin-runtime/computer/conversation-target.ts +143 -0
- package/packages/plugin-runtime/computer/index.ts +2 -0
- package/packages/plugin-runtime/computer/target-authority.ts +26 -1
- package/packages/plugin-runtime/external/unix-jsonl-transport.ts +142 -18
- package/packages/protocols/computer/contract.ts +27 -1
- package/scripts/activate-source-baseline.ts +7 -9
- package/scripts/canary-controller-semantic-wait.ts +12 -24
- package/scripts/check-mcp-compatibility.ts +48 -136
- package/scripts/check-runtime-architecture.mjs +622 -217
- package/scripts/forge-local-recovery-helper.mjs +93 -15
- package/scripts/lib/project-init-lib.sh +3 -0
- package/scripts/open-source-audit-allowlist.txt +1 -2
- package/scripts/run-bounded-command.mjs +5 -0
- package/scripts/run-governed-gate.ts +0 -1
- package/scripts/run-mobile-monitor-bridge.ts +0 -1
- package/scripts/smoke-control-plane-failure-domain.ts +94 -0
- package/scripts/smoke-mcp-http-runtime.ts +5 -0
- package/scripts/smoke-runtime-recovery.ts +12 -43
- package/scripts/smoke-schedule-engine.ts +23 -15
- package/scripts/smoke-unattended-transport-recovery.ts +223 -0
- package/scripts/smoke-user-request-continuation-restart.ts +232 -0
- package/scripts/smoke-workflow-supervisor-chrome-adapter.ts +92 -0
- package/scripts/smoke-workflow-supervisor-process-restart.ts +276 -0
- package/scripts/smoke-workflow-supervisor-restart-recovery.ts +150 -0
- package/scripts/smoke-workflow-supervisor.ts +103 -0
- package/scripts/verify-forge-runtime.sh +2 -1
- package/src/cli/agent-jobs/executor-health.ts +11 -34
- package/src/cli/agent-jobs/integration.ts +18 -49
- package/src/cli/agent-jobs/job-manager.ts +7 -3
- package/src/cli/commands/brain-assistant.ts +27 -13
- package/src/cli/commands/brain.ts +13 -15
- package/src/cli/commands/chatgpt.ts +1 -163
- package/src/cli/commands/recovery.ts +135 -121
- package/src/cli/commands/runtime.ts +3 -3
- package/src/cli/commands/setup-profile.ts +8 -0
- package/src/cli/controller/check-runner.ts +13 -2
- package/src/cli/controller/completion-backlog.ts +3 -3
- package/src/cli/controller/completion-orchestrator.ts +4 -43
- package/src/cli/controller/completion-target.ts +3 -72
- package/src/cli/controller/context/source-materializer.ts +81 -100
- package/src/cli/controller/context-pack.ts +10 -8
- package/src/cli/controller/execution-completion.ts +4 -7
- package/src/cli/controller/execution-policy.ts +36 -153
- package/src/cli/controller/governance.ts +2 -3
- package/src/cli/controller/issue-store.ts +4 -99
- package/src/cli/controller/launch-agents.ts +59 -18
- package/src/cli/controller/operational-plan.ts +24 -51
- package/src/cli/controller/runtime-config.ts +5 -1
- package/src/cli/controller/stuck-state-migration.ts +2 -3
- package/src/cli/controller/systemd-user.ts +39 -1
- package/src/cli/editing/edit-session.ts +1 -7
- package/src/cli/local-bridge/console-automations.ts +18 -13
- package/src/cli/local-bridge/console-view-models.ts +3 -12
- package/src/cli/local-bridge/facade-api.ts +64 -191
- package/src/cli/local-bridge/server.ts +24 -147
- package/src/cli/local-bridge/types.ts +0 -1
- package/src/cli/mcp/oauth-client-retirement.ts +74 -0
- package/src/cli/repositories/command-executor.ts +29 -23
- package/src/cli/repositories/command-scope.ts +35 -16
- package/src/cli/repositories/controller-home.ts +48 -0
- package/src/cli/repositories/ephemeral-workspace.ts +20 -30
- package/src/cli/repositories/git-command-executor.ts +2 -2
- package/src/cli/repositories/registry.ts +77 -11
- package/src/cli/repositories/structured-git.ts +62 -2
- package/src/cli/repository/session-cache.ts +92 -6
- package/src/runtime/context/assistant-context.ts +20 -7
- package/src/runtime/context/assistant-work-context.ts +89 -27
- package/src/runtime/context/automatic-learning.ts +565 -0
- package/src/runtime/context/cognitive-attention.ts +173 -0
- package/src/runtime/context/cognitive-skill-canary.ts +158 -0
- package/src/runtime/context/context-closure.ts +46 -9
- package/src/runtime/context/context-pack-process.ts +103 -0
- package/src/runtime/context/context-plane.ts +1 -9
- package/src/runtime/context/direct-controller-learning.ts +298 -0
- package/src/runtime/context/semantic-navigation-contract.ts +9 -0
- package/src/runtime/context/semantic-navigation.ts +6 -34
- package/src/runtime/context/typescript-navigation-process.ts +145 -0
- package/src/runtime/context/typescript-navigation.ts +89 -3
- package/src/runtime/control-plane/cleanup-artifact-retention.ts +212 -9
- package/src/runtime/control-plane/concurrency/work-execution-concurrency.ts +9 -47
- package/src/runtime/control-plane/execution/controller-authority-recovery.ts +34 -4
- package/src/runtime/control-plane/execution/edit-validation-coordinator.ts +29 -20
- package/src/runtime/control-plane/execution/execution-identity.ts +23 -3
- package/src/runtime/control-plane/execution/repository-work-attribution.ts +7 -30
- package/src/runtime/control-plane/execution/retained-work-resume.ts +206 -199
- package/src/runtime/control-plane/execution/session-store.ts +31 -1
- package/src/runtime/control-plane/execution/validation.ts +6 -8
- package/src/runtime/control-plane/execution/work-authority-reconciler.ts +13 -42
- package/src/runtime/control-plane/execution/work-check-evidence.ts +137 -0
- package/src/runtime/control-plane/execution/work-completion-authority.ts +30 -85
- package/src/runtime/control-plane/execution/work-delivery-service.ts +497 -0
- package/src/runtime/control-plane/execution/work-evidence-policy.ts +12 -50
- package/src/runtime/control-plane/execution/work-execution-support.ts +50 -149
- package/src/runtime/control-plane/execution/work-handle-authority.ts +188 -109
- package/src/runtime/control-plane/execution/work-handle-store.ts +19 -43
- package/src/runtime/control-plane/execution/work-head-settlement.ts +53 -20
- package/src/runtime/control-plane/execution/work-lifecycle-audit.ts +52 -29
- package/src/runtime/control-plane/execution/work-operation-service.ts +43 -27
- package/src/runtime/control-plane/execution/work-preparation-service.ts +126 -266
- package/src/runtime/control-plane/execution/work-remote-delivery.ts +2 -3
- package/src/runtime/control-plane/execution/work-revision-diff.ts +39 -0
- package/src/runtime/control-plane/execution/work-terminal-cleanup.ts +173 -186
- package/src/runtime/control-plane/execution/work-validation-reconciler.ts +28 -84
- package/src/runtime/control-plane/execution/work-verification-service.ts +406 -50
- package/src/runtime/control-plane/execution/work-verification-snapshot.ts +5 -1
- package/src/runtime/control-plane/facade/capability-registry.ts +362 -37
- package/src/runtime/control-plane/facade/check-normalization.ts +3 -9
- package/src/runtime/control-plane/facade/codex-delegation.ts +10 -11
- package/src/runtime/control-plane/facade/controller-connection.ts +135 -0
- package/src/runtime/control-plane/facade/goal-workloop-access.ts +1 -20
- package/src/runtime/control-plane/facade/goal-workloop.ts +281 -1753
- package/src/runtime/control-plane/facade/handoff-inbox-application.ts +12 -25
- package/src/runtime/control-plane/facade/handoff-inbox-store.ts +256 -52
- package/src/runtime/control-plane/facade/index.ts +0 -1
- package/src/runtime/control-plane/facade/operation-digest.ts +12 -15
- package/src/runtime/control-plane/facade/plan-contract-store.ts +451 -991
- package/src/runtime/control-plane/facade/policy-gate.ts +0 -23
- package/src/runtime/control-plane/facade/repository-work-admission.ts +12 -22
- package/src/runtime/control-plane/facade/requirement-authority.ts +102 -199
- package/src/runtime/control-plane/facade/requirement-board.ts +3 -3
- package/src/runtime/control-plane/facade/rh-work-operation-contract.ts +51 -14
- package/src/runtime/control-plane/facade/self-healing-loop.ts +18 -53
- package/src/runtime/control-plane/facade/suggested-actions.ts +15 -18
- package/src/runtime/control-plane/facade/types.ts +49 -108
- package/src/runtime/control-plane/facade/work-continuation.ts +8 -8
- package/src/runtime/control-plane/generated-cache-retention.ts +97 -1
- package/src/runtime/control-plane/global-scheduler/autonomous-continuation.ts +248 -0
- package/src/runtime/control-plane/global-scheduler/durable-admission.ts +1 -1
- package/src/runtime/control-plane/global-scheduler/maintenance.ts +239 -112
- package/src/runtime/control-plane/global-scheduler/reconciliation.ts +2 -1
- package/src/runtime/control-plane/global-scheduler/scheduler.ts +66 -40
- package/src/runtime/control-plane/global-scheduler/wake-signal.ts +42 -2
- package/src/runtime/control-plane/governance/access-policy.ts +19 -2
- package/src/runtime/control-plane/governance/authorization.ts +20 -3
- package/src/runtime/control-plane/launcher/chatgpt-round-continuation.ts +63 -300
- package/src/runtime/control-plane/launcher/chatgpt-work-continuation.ts +28 -300
- package/src/runtime/control-plane/launcher/local-conversation-controller.ts +238 -0
- package/src/runtime/control-plane/launcher/provider-mcp-bootstrap.ts +5 -2
- package/src/runtime/control-plane/launcher/thin-launcher.ts +96 -240
- package/src/runtime/control-plane/persistence/cognition-store.ts +820 -0
- package/src/runtime/control-plane/persistence/experience-store.ts +162 -29
- package/src/runtime/control-plane/persistence/requirement-store.ts +216 -74
- package/src/runtime/control-plane/persistence/sqlite-store.ts +113 -6
- package/src/runtime/control-plane/release-retention.ts +135 -47
- package/src/runtime/control-plane/repository-namespace-retention.ts +2 -2
- package/src/runtime/control-plane/runtime-cleanup.ts +290 -21
- package/src/runtime/control-plane/workspace/project-onboarding.ts +80 -0
- package/src/runtime/control-plane/workspace/semantic-sync-service.ts +19 -5
- package/src/runtime/control-plane/workspace/workspace-store.ts +169 -0
- package/src/runtime/diagnostics/incident-repair.ts +313 -146
- package/src/runtime/diagnostics/mcp-timing.ts +114 -0
- package/src/runtime/evidence/artifact-store.ts +158 -40
- package/src/runtime/evidence/evidence-store.ts +112 -8
- package/src/runtime/evidence/output-handle-index.ts +225 -0
- package/src/runtime/evidence/result-store.ts +158 -44
- package/src/runtime/execution/jobs/store.ts +3 -4
- package/src/runtime/execution/managed-workspace.ts +57 -0
- package/src/runtime/execution/process-runtime/check-facade.ts +1 -1
- package/src/runtime/execution/process-runtime/check-result-retention.ts +2 -2
- package/src/runtime/execution/process-runtime/check-result.ts +105 -1
- package/src/runtime/execution/process-runtime/check-runner-sidecar.ts +18 -8
- package/src/runtime/execution/process-runtime/command-facade.ts +79 -27
- package/src/runtime/execution/process-runtime/gc.ts +15 -2
- package/src/runtime/execution/process-runtime/handle-index.ts +176 -0
- package/src/runtime/execution/process-runtime/host-command-lane.ts +139 -0
- package/src/runtime/execution/process-runtime/index.ts +3 -0
- package/src/runtime/execution/process-runtime/lightweight-managed.ts +251 -72
- package/src/runtime/execution/process-runtime/persisted-check.ts +3 -3
- package/src/runtime/execution/process-runtime/process-runner-entry.ts +47 -5
- package/src/runtime/execution/process-runtime/process-scope.ts +160 -0
- package/src/runtime/execution/process-runtime/resource-claims.ts +65 -61
- package/src/runtime/execution/process-runtime/runtime.ts +35 -2
- package/src/runtime/execution/process-runtime/store.ts +30 -2
- package/src/runtime/execution/process-runtime/types.ts +6 -1
- package/src/runtime/execution/workers/executor.ts +2 -6
- package/src/runtime/platform/service-inventory.ts +130 -0
- package/src/runtime/platform/service-manager.ts +8 -0
- package/src/runtime/plugins/action-application.ts +85 -28
- package/src/runtime/plugins/app-store-connect-adapter.ts +38 -14
- package/src/runtime/plugins/browser-adapter.ts +757 -34
- package/src/runtime/plugins/browser-automation-service.ts +23 -3
- package/src/runtime/plugins/browser-macos-bridge.ts +321 -29
- package/src/runtime/plugins/browser-manifest-surface.ts +45 -11
- package/src/runtime/plugins/browser-native-session-service.ts +63 -13
- package/src/runtime/plugins/browser-registration.ts +16 -0
- package/src/runtime/plugins/browser-runtime.ts +14 -1
- package/src/runtime/plugins/browser-session-authority.ts +2 -2
- package/src/runtime/plugins/browser-session-store.ts +46 -76
- package/src/runtime/plugins/capability-authorization-grants.ts +258 -62
- package/src/runtime/plugins/computer-chatgpt-browser-adapter-installer.ts +219 -0
- package/src/runtime/plugins/computer-chatgpt-macos-target.ts +891 -0
- package/src/runtime/plugins/computer-registration.ts +222 -11
- package/src/runtime/plugins/desktop-operator-external-policy.ts +36 -4
- package/src/runtime/plugins/desktop-operator-registration.ts +12 -4
- package/src/runtime/plugins/external-adapter.ts +42 -6
- package/src/runtime/plugins/external-registration.ts +21 -1
- package/src/runtime/plugins/gmail-adapter.ts +23 -2
- package/src/runtime/plugins/google-calendar-adapter.ts +2 -2
- package/src/runtime/plugins/google-shared.ts +14 -7
- package/src/runtime/plugins/google-tasks-adapter.ts +2 -2
- package/src/runtime/plugins/ios-adapter.ts +1 -1
- package/src/runtime/plugins/lightweight-action.ts +208 -2
- package/src/runtime/plugins/local-system-adapter.ts +12 -2
- package/src/runtime/plugins/managed-process-adapter.ts +12 -5
- package/src/runtime/plugins/plugin-action-sidecar.ts +17 -13
- package/src/runtime/plugins/plugin-management-adapter.ts +1 -1
- package/src/runtime/plugins/store.ts +602 -428
- package/src/runtime/plugins/types.ts +20 -0
- package/src/runtime/plugins/xiaohongshu-publish.ts +16 -0
- package/src/runtime/projections/invalidation.ts +7 -3
- package/src/runtime/projections/materialized-view.ts +80 -36
- package/src/runtime/recovery/actions.ts +3 -3
- package/src/runtime/recovery/capability-action-application.ts +46 -45
- package/src/runtime/recovery/lifecycle-inventory.ts +29 -29
- package/src/runtime/recovery/maintenance-executor.ts +374 -123
- package/src/runtime/recovery/store.ts +29 -11
- package/src/runtime/release/release-coordinator.ts +296 -0
- package/src/runtime/release/release-session.ts +716 -0
- package/src/runtime/release/release-state-migration.ts +25 -0
- package/src/runtime/resources/leases/store.ts +56 -25
- package/src/runtime/root/computer-composition.ts +191 -6
- package/src/runtime/root/controller-progression-composition.ts +192 -0
- package/src/runtime/root/controller-round-composition.ts +20 -1
- package/src/runtime/root/deployment-topology.ts +106 -0
- package/src/runtime/root/entry.ts +4 -3
- package/src/runtime/root/known-good-recovery.ts +184 -0
- package/src/runtime/root/local-bridge.ts +0 -1
- package/src/runtime/root/mcp-transport.ts +111 -18
- package/src/runtime/root/ownership.ts +261 -9
- package/src/runtime/root/package-connector-service.ts +105 -19
- package/src/runtime/root/package-runtime-release.ts +117 -18
- package/src/runtime/root/package-runtime-service.ts +17 -11
- package/src/runtime/root/release-execution-canary.ts +44 -12
- package/src/runtime/root/release-loader.ts +90 -8
- package/src/runtime/root/release-manifest.ts +80 -6
- package/src/runtime/root/release-materialize.ts +372 -28
- package/src/runtime/root/release-store.ts +395 -36
- package/src/runtime/root/runtime-lane.ts +191 -0
- package/src/runtime/root/runtime.ts +159 -27
- package/src/runtime/root/scheduled-controller-composition.ts +28 -3
- package/src/runtime/root/service-runner.ts +2 -2
- package/src/runtime/root/service.ts +36 -33
- package/src/runtime/root/status.ts +48 -0
- package/src/runtime/root/types.ts +32 -3
- package/src/runtime/root/workflow-supervisor-composition.ts +493 -0
- package/src/runtime/root/workflow-supervisor-runtime.ts +209 -0
- package/src/runtime/root/write-fence.ts +50 -2
- package/src/runtime/safe-tooling/google-credential-store.ts +76 -21
- package/src/runtime/safe-tooling/google-oauth-broker.ts +25 -7
- package/src/runtime/safe-tooling/workspace-auth.ts +1 -1
- package/src/runtime/shared/bounded-child-supervisor.ts +9 -1
- package/src/runtime/shared/process-environment.ts +20 -9
- package/src/runtime/shared/response-budget.ts +1 -3
- package/src/runtime/standalone-recovery/controller-home-migration.ts +16 -24
- package/src/runtime/standalone-recovery/core.ts +3702 -342
- package/src/runtime/standalone-recovery/entry.ts +469 -39
- package/src/runtime/standalone-recovery/installer.ts +262 -148
- package/src/runtime/standalone-recovery/mcp-server.ts +29 -193
- package/src/runtime/standalone-recovery/operation-lock.ts +1 -1
- package/src/runtime/standalone-recovery/performance.ts +54 -16
- package/src/runtime/standalone-recovery/release.ts +53 -5
- package/src/runtime/standalone-recovery/service-labels.ts +5 -2
- package/src/runtime/standalone-recovery/watchdog-heartbeat.ts +5 -2
- package/src/runtime/workflow/schedules/browser-probe.ts +17 -6
- package/src/runtime/workflow/schedules/chatgpt-automation-migration.ts +10 -3
- package/src/runtime/workflow/schedules/engine.ts +138 -172
- package/src/runtime/workflow/schedules/work-continuation.ts +44 -5
- package/src/runtime/workflows/runtime.ts +15 -19
- package/src/testing/test-governance.ts +5 -1
- package/supervisor/chatgpt-conversation.ts +23 -0
- package/supervisor/client.ts +201 -0
- package/supervisor/control-plane.ts +868 -0
- package/supervisor/entry.ts +15 -0
- package/supervisor/forge-validators.ts +80 -0
- package/supervisor/index.ts +5 -0
- package/supervisor/native-browser-adapter.ts +786 -0
- package/supervisor/native-messaging/host.ts +39 -0
- package/supervisor/native-messaging/manifest.ts +26 -0
- package/supervisor/native-messaging/protocol.ts +27 -0
- package/supervisor/paths.ts +18 -0
- package/supervisor/protocol.ts +202 -0
- package/supervisor/server.ts +404 -0
- package/supervisor/store.ts +1451 -0
- package/supervisor/tsconfig.json +8 -0
- package/supervisor/types.ts +255 -0
- package/adapters/chatgpt/controller-host.ts +0 -119
- package/adapters/mcp/controller-round-compatibility.ts +0 -184
- package/adapters/mcp/frozen-client-semantic-compatibility.ts +0 -292
- package/adapters/mcp/runtime-gateway/work-compat-adapter.ts +0 -502
- package/packages/kernel/progression/api/index.ts +0 -2
- package/packages/kernel/progression/application/projector.ts +0 -211
- package/packages/kernel/progression/domain/types.ts +0 -116
- package/packages/kernel/progression/index.ts +0 -1
- package/packages/kernel/scheduler/application/continuation-service.ts +0 -212
- package/packages/kernel/scheduler/domain/continuation.ts +0 -38
- package/packages/kernel/scheduler/infrastructure/continuation-dispatch-store.ts +0 -38
- package/packages/kernel/work/domain/admission-policy.ts +0 -34
- package/scripts/benchmark-background-check-overlap.ts +0 -228
- package/scripts/benchmark-computer-hotpath.ts +0 -255
- package/scripts/benchmark-controller-hotpath.ts +0 -212
- package/scripts/benchmark-controller-round2.ts +0 -191
- package/scripts/benchmark-ios-agent-device.ts +0 -511
- package/scripts/benchmark-mcp-host.sh +0 -117
- package/scripts/benchmark-mcp-transport.ts +0 -169
- package/scripts/benchmark-operational-memory-activation.ts +0 -235
- package/scripts/benchmark-operational-shadow.ts +0 -28
- package/scripts/benchmark-quality-harness.ts +0 -146
- package/scripts/benchmark-route-session-concurrency.ts +0 -915
- package/scripts/benchmark-thin-harness-gateway-ab.ts +0 -632
- package/scripts/benchmark-thin-harness-mcp-e2e.ts +0 -498
- package/scripts/benchmark-thin-harness.ts +0 -251
- package/scripts/benchmark-typescript-navigation.ts +0 -186
- package/scripts/benchmark-work-efficiency.ts +0 -334
- package/scripts/build-controller-ui.ts +0 -28
- package/scripts/check-npm-release.sh +0 -32
- package/scripts/check-public-release-surface.sh +0 -19
- package/scripts/check-release-published.sh +0 -57
- package/scripts/check-release-readiness.sh +0 -37
- package/scripts/check-release-version.mjs +0 -74
- package/scripts/check-tarball-install-smoke.sh +0 -150
- package/scripts/package-source-archive.sh +0 -147
- package/scripts/public-release-files.txt +0 -39
- package/scripts/public-release.ts +0 -174
- package/scripts/publish-release-tarball.sh +0 -27
- package/scripts/route-nl-vs-ts-eval.ts +0 -679
- package/scripts/stage-runtime-release.ts +0 -44
- package/src/cli/controller/progress.ts +0 -656
- package/src/cli/controller/work-mode.ts +0 -181
- package/src/cli/local-bridge/console-assets.ts +0 -17
- package/src/cli/local-bridge/dashboard.ts +0 -17
- package/src/cli/local-bridge/frontend/src/App.tsx +0 -10
- package/src/cli/local-bridge/frontend/src/components/Button.tsx +0 -1
- package/src/cli/local-bridge/frontend/src/components/DetailPane.tsx +0 -3
- package/src/cli/local-bridge/frontend/src/components/SearchField.tsx +0 -1
- package/src/cli/local-bridge/frontend/src/components/SectionHeader.tsx +0 -1
- package/src/cli/local-bridge/frontend/src/components/Segmented.tsx +0 -1
- package/src/cli/local-bridge/frontend/src/components/StatusText.tsx +0 -3
- package/src/cli/local-bridge/frontend/src/design/icons.tsx +0 -11
- package/src/cli/local-bridge/frontend/src/design/tokens.css +0 -8
- package/src/cli/local-bridge/frontend/src/features/automations/AutomationsView.tsx +0 -153
- package/src/cli/local-bridge/frontend/src/features/capabilities/CapabilitiesView.tsx +0 -37
- package/src/cli/local-bridge/frontend/src/features/overview/OverviewView.tsx +0 -183
- package/src/cli/local-bridge/frontend/src/features/repositories/RepositoriesView.tsx +0 -2
- package/src/cli/local-bridge/frontend/src/features/system/SystemView.tsx +0 -2
- package/src/cli/local-bridge/frontend/src/features/work/WorkView.tsx +0 -4
- package/src/cli/local-bridge/frontend/src/lib/api.ts +0 -30
- package/src/cli/local-bridge/frontend/src/lib/format.ts +0 -4
- package/src/cli/local-bridge/frontend/src/lib/routes.ts +0 -7
- package/src/cli/local-bridge/frontend/src/main.tsx +0 -1
- package/src/cli/local-bridge/frontend/src/shell/AppShell.tsx +0 -1
- package/src/cli/local-bridge/frontend/src/shell/CommandBar.tsx +0 -2
- package/src/cli/local-bridge/frontend/src/shell/Sidebar.tsx +0 -4
- package/src/cli/local-bridge/frontend/src/styles.css +0 -28
- package/src/cli/local-bridge/frontend/src/types.ts +0 -15
- package/src/cli/local-bridge/frontend/src/vite-env.d.ts +0 -1
- package/src/cli/local-bridge/frontend/vite.config.ts +0 -27
- package/src/cli/local-bridge/ui-dist/app.css +0 -1
- package/src/cli/local-bridge/ui-dist/app.js +0 -11228
- package/src/runtime/control-plane/execution/content-equivalent-commit-authority.ts +0 -169
- package/src/runtime/control-plane/execution/direct-canonical-work-reconciliation.ts +0 -85
- package/src/runtime/control-plane/execution/direct-edit-work-completion.ts +0 -655
- package/src/runtime/control-plane/execution/work-finalization-service.ts +0 -2850
- package/src/runtime/control-plane/execution/work-path-scope.ts +0 -37
- package/src/runtime/control-plane/execution/work-task-receipt.ts +0 -468
- package/src/runtime/control-plane/facade/controller-round-relay.ts +0 -3
- package/src/runtime/control-plane/facade/work-admission-policy.ts +0 -142
- package/src/runtime/control-plane/facade/work-implementation-review.ts +0 -2
- package/src/runtime/control-plane/facade/work-state-machine.ts +0 -2
- package/src/runtime/control-plane/facade/work-task-projection.ts +0 -66
- package/src/runtime/control-plane/routing/index.ts +0 -1
- package/src/runtime/control-plane/routing/route-policy.ts +0 -401
- package/src/runtime/control-plane/routing/workspace-admission.ts +0 -70
|
@@ -1,20 +1,37 @@
|
|
|
1
1
|
import { assertRuntimePerformanceEvidence, measureRuntimePerformance, samePerformanceIdentity, type RuntimePerformanceDependencies, type RuntimePerformanceEvidence, type RuntimePerformanceIdentity } from './performance';
|
|
2
|
+
import { runBoundedChild } from '../shared/bounded-child-supervisor';
|
|
2
3
|
import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'crypto';
|
|
3
4
|
import { spawn, spawnSync } from 'child_process';
|
|
4
|
-
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from 'fs';
|
|
5
|
+
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, realpathSync, renameSync, rmSync, statSync, writeFileSync } from 'fs';
|
|
5
6
|
import { homedir, hostname } from 'os';
|
|
7
|
+
import { createServer as createNetServer } from 'net';
|
|
6
8
|
import { basename, delimiter, dirname, isAbsolute, join, resolve } from 'path';
|
|
7
9
|
import { assertStorageHeadroom } from '../shared/storage-capacity';
|
|
8
|
-
import { resolveBunExecutable } from '../shared/process-environment';
|
|
9
|
-
import {
|
|
10
|
+
import { resolveBunExecutable, runtimeAuthorityFreeEnvironment } from '../shared/process-environment';
|
|
11
|
+
import { backupControlPlaneDatabase } from '../control-plane/persistence/sqlite-store';
|
|
12
|
+
import { measureReclaimablePath } from '../control-plane/lifecycle-retention-metrics';
|
|
13
|
+
import { cleanupControllerReleaseHistory } from '../control-plane/release-retention';
|
|
14
|
+
import { observeRuntimeStatus, readRuntimeStartupFailureEvidence } from '../root/status';
|
|
15
|
+
import { reconcileStoppedRuntimeOwnership, terminateVerifiedRuntimeOwner, readRuntimeOwner } from '../root/ownership';
|
|
16
|
+
import type { RuntimeOwnerRecord } from '../root/ownership';
|
|
10
17
|
import {
|
|
11
18
|
activeRuntimeEntrypoint,
|
|
12
19
|
activeRuntimeLaunchSpec,
|
|
13
20
|
ensureForgeRuntimeLaunchAgentContract,
|
|
14
21
|
forgeRuntimeServicePaths,
|
|
15
22
|
inspectForgeRuntimeLaunchAgentContract,
|
|
23
|
+
installForgeRuntimeService,
|
|
16
24
|
readForgeRuntimeServiceConfig,
|
|
25
|
+
syncForgeRuntimeActiveEntrypoint,
|
|
26
|
+
uninstallForgeRuntimeService,
|
|
17
27
|
} from '../root/service';
|
|
28
|
+
import { removeRetiredCandidateExecutionLane } from '../root/runtime-lane';
|
|
29
|
+
import {
|
|
30
|
+
inspectKnownGoodRecoveryBundle,
|
|
31
|
+
knownGoodRecoveryBundleRoot,
|
|
32
|
+
sha256FileBounded,
|
|
33
|
+
type KnownGoodRecoveryBundle,
|
|
34
|
+
} from '../root/known-good-recovery';
|
|
18
35
|
import { systemdUserUnitName, systemdUserUnitPath } from '../../cli/controller/systemd-user';
|
|
19
36
|
import {
|
|
20
37
|
openAiSecureTunnelConnectArgs,
|
|
@@ -24,14 +41,18 @@ import {
|
|
|
24
41
|
} from '../../../adapters/mcp/tunnels/openai-secure-tunnel';
|
|
25
42
|
import {
|
|
26
43
|
renderPackageRuntimeSystemdUserService,
|
|
44
|
+
systemdRuntimeInstallCommands,
|
|
27
45
|
writePackageRuntimeSystemdUserService,
|
|
28
46
|
} from '../root/package-runtime-service';
|
|
29
47
|
import { loadRuntimeReleaseManifest } from '../root/release-manifest';
|
|
30
|
-
import { assertRuntimeReleaseExecutionCanaries, assertRuntimeReleaseFiles, stageRuntimeReleaseFromCandidateSource, type RuntimeReleaseExecutionCanaryDependencies, type StagedRuntimeRelease } from '../root/release-materialize';
|
|
48
|
+
import { assertRuntimeReleaseExecutionCanaries, assertRuntimeReleaseFiles, promotePortableRuntimeRelease, runtimeReleaseTreeSha256, stageRuntimeReleaseFromCandidateSource, withRuntimeReleaseSourceSnapshot, type RuntimeReleaseExecutionCanaryDependencies, type StagedRuntimeRelease } from '../root/release-materialize';
|
|
31
49
|
import {
|
|
50
|
+
prepareRuntimeReleaseDatabaseBackup,
|
|
32
51
|
publishRuntimeRelease,
|
|
33
52
|
readRuntimeReleaseAuthority,
|
|
34
|
-
|
|
53
|
+
reconcileFailedRuntimeReleaseActivationAuthority,
|
|
54
|
+
rollbackRuntimeReleaseWithResult,
|
|
55
|
+
type RuntimeDatabaseRollbackDisposition,
|
|
35
56
|
type RuntimePublishedRelease,
|
|
36
57
|
type RuntimeReleaseAuthority,
|
|
37
58
|
} from '../root/release-store';
|
|
@@ -39,16 +60,34 @@ import type { RuntimeReleaseManifest } from '../root/types';
|
|
|
39
60
|
import { ensurePackageConnectorService, packageConnectorAuthMode, packageConnectorServicePaths, type PackageConnectorReleaseBinding } from '../root/package-connector-service';
|
|
40
61
|
import { createRecoveryHttpTransport, type RecoveryHttpTransport } from './http-transport';
|
|
41
62
|
import { observeRecoveryWatchdogHealth } from './watchdog-heartbeat';
|
|
42
|
-
import {
|
|
63
|
+
import { RECOVERY_DAEMON_LABEL } from './service-labels';
|
|
64
|
+
import { resolveWorkflowSupervisorForgeHome, workflowSupervisorSocketPath } from '../../../supervisor/paths';
|
|
65
|
+
import { getWorkflowSupervisorContinuationProof } from '../../../supervisor/client';
|
|
66
|
+
import type { WorkflowSupervisorContinuationProof } from '../../../supervisor/types';
|
|
67
|
+
import { reconcileStoppedWorkflowSupervisorSocket } from '../../../supervisor/server';
|
|
43
68
|
import {
|
|
44
69
|
migrateStoppedRepoLocalControllerHomeStorage,
|
|
45
70
|
repoLocalControllerHomeStorageNeedsMigration,
|
|
46
71
|
rollbackStoppedRepoLocalControllerHomeStorage,
|
|
47
72
|
type ControllerHomeStorageMigration,
|
|
48
73
|
} from '../../cli/repositories/controller-home';
|
|
49
|
-
import { readCurrentRecoveryRelease } from './release';
|
|
74
|
+
import { readCurrentRecoveryRelease, type RecoveryRuntimeRole } from './release';
|
|
50
75
|
import { recoveryOperationLockPath } from './operation-lock';
|
|
51
76
|
import { RECOVERY_MUTATION_IDENTITY_FIELDS, type RecoveryMutationIdentityArguments } from './mutation-identity-contract';
|
|
77
|
+
import { createCandidateExecutionLane, readStableExecutionLane } from '../root/runtime-lane';
|
|
78
|
+
import {
|
|
79
|
+
advanceReleaseSession,
|
|
80
|
+
createReleaseSession,
|
|
81
|
+
listCompleteReleaseSessions,
|
|
82
|
+
readReleaseSession,
|
|
83
|
+
recordReleaseSessionTransaction,
|
|
84
|
+
releaseSessionCandidateIsRetired,
|
|
85
|
+
releaseSessionIsSoakingPredecessorOfStable,
|
|
86
|
+
releaseSessionIsSoakingSupersededByStableAuthority,
|
|
87
|
+
type ReleaseSession,
|
|
88
|
+
type ReleaseSessionCandidateRelease,
|
|
89
|
+
type ReleaseSessionStableRelease,
|
|
90
|
+
} from '../release/release-session';
|
|
52
91
|
|
|
53
92
|
/** Standalone recovery reads only canonical Runtime observation and whole-release authority. */
|
|
54
93
|
interface PublicTunnelRecoveryPolicy {
|
|
@@ -125,9 +164,24 @@ export interface SystemdPrimaryConnectorServiceConfig extends PrimaryConnectorRe
|
|
|
125
164
|
|
|
126
165
|
export type PrimaryConnectorServiceConfig = LaunchdPrimaryConnectorServiceConfig | SystemdPrimaryConnectorServiceConfig;
|
|
127
166
|
|
|
167
|
+
export type RecoveryInstallProfile = 'manual' | 'gateway' | 'self-healing';
|
|
168
|
+
|
|
169
|
+
export function normalizeRecoveryInstallProfile(value: unknown, fallback: RecoveryInstallProfile = 'self-healing'): RecoveryInstallProfile {
|
|
170
|
+
if (value === undefined || value === null || value === '') return fallback;
|
|
171
|
+
if (value === 'manual' || value === 'gateway' || value === 'self-healing') return value;
|
|
172
|
+
throw new Error(`RECOVERY_INSTALL_PROFILE_INVALID: ${String(value)}`);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
export function recoveryInstallProfileRoles(profile: RecoveryInstallProfile): RecoveryRuntimeRole[] {
|
|
176
|
+
if (profile === 'manual') return [];
|
|
177
|
+
return ['daemon'];
|
|
178
|
+
}
|
|
179
|
+
|
|
128
180
|
export interface RecoveryConfig {
|
|
129
181
|
schemaVersion: 1;
|
|
130
182
|
controllerHome: string;
|
|
183
|
+
/** Persistent Recovery role selection. Legacy configs default to self-healing. */
|
|
184
|
+
installProfile: RecoveryInstallProfile;
|
|
131
185
|
publicMcpUrl?: string;
|
|
132
186
|
recoveryPublicUrl?: string;
|
|
133
187
|
recoveryTunnelService?: RecoveryTunnelServiceConfig;
|
|
@@ -220,28 +274,46 @@ interface ReleaseEvidence {
|
|
|
220
274
|
manifestSha256: string;
|
|
221
275
|
workerProtocolVersion: number;
|
|
222
276
|
controllerHome?: string;
|
|
277
|
+
sourceRepositoryId?: string;
|
|
223
278
|
releaseAuthorityRevision?: number;
|
|
224
279
|
releaseFencingTokenSha256?: string;
|
|
225
280
|
attestedAt?: string;
|
|
281
|
+
pinnedAt?: string;
|
|
226
282
|
performance?: RuntimePerformanceEvidence;
|
|
283
|
+
recoveryBundle?: KnownGoodRecoveryBundle;
|
|
227
284
|
}
|
|
228
285
|
|
|
229
286
|
interface KnownGoodStore {
|
|
230
|
-
schemaVersion: 1;
|
|
287
|
+
schemaVersion: 1 | 2;
|
|
231
288
|
releases: ReleaseEvidence[];
|
|
232
289
|
updatedAt: string;
|
|
233
290
|
}
|
|
234
291
|
|
|
292
|
+
interface RuntimePinStore {
|
|
293
|
+
schemaVersion: 1;
|
|
294
|
+
release: ReleaseEvidence;
|
|
295
|
+
updatedAt: string;
|
|
296
|
+
}
|
|
297
|
+
|
|
235
298
|
export type RecoveryMutationAction =
|
|
236
299
|
| 'attest_known_good'
|
|
237
300
|
| 'rollback_previous'
|
|
238
301
|
| 'restart_primary_runtime'
|
|
239
302
|
| 'recover_primary_runtime'
|
|
240
303
|
| 'activate_runtime_release'
|
|
304
|
+
| 'pin_runtime_release'
|
|
305
|
+
| 'unpin_runtime_release'
|
|
306
|
+
| 'activate_pinned_runtime_release'
|
|
241
307
|
| 'stage_and_activate_runtime_release'
|
|
308
|
+
| 'release_session_prepare'
|
|
309
|
+
| 'release_session_static_verify'
|
|
310
|
+
| 'release_session_candidate_boot'
|
|
311
|
+
| 'release_session_cutover'
|
|
312
|
+
| 'release_session_cancel'
|
|
313
|
+
| 'release_session_rollback'
|
|
314
|
+
| 'release_session_known_good'
|
|
242
315
|
| 'restart_primary_connector'
|
|
243
316
|
| 'restart_recovery_gateway'
|
|
244
|
-
| 'restart_recovery_watchdog'
|
|
245
317
|
| 'repair_public_tunnel';
|
|
246
318
|
|
|
247
319
|
interface RecoveryLock {
|
|
@@ -275,6 +347,26 @@ export interface VerifyStableRuntimeOptions extends RuntimeReleaseExecutionCanar
|
|
|
275
347
|
probeMcpProtocol?: boolean;
|
|
276
348
|
}
|
|
277
349
|
|
|
350
|
+
/**
|
|
351
|
+
* Remote tunnel control-plane lookup is weaker evidence than the live data plane.
|
|
352
|
+
* Once the configured tunnel is locally healthy/ready with matching identities and
|
|
353
|
+
* the public MCP protocol succeeds end-to-end, a degraded/unknown control-plane
|
|
354
|
+
* lookup remains diagnostic instead of vetoing whole-Runtime verification.
|
|
355
|
+
* Any local health, readiness, identity, endpoint, or MCP failure still gates.
|
|
356
|
+
*/
|
|
357
|
+
export function isPrimaryTunnelControlPlaneOnlyDegradation(
|
|
358
|
+
probe: VerifyResult['probes'][string] | undefined,
|
|
359
|
+
mcpProtocolHealthy: boolean,
|
|
360
|
+
): boolean {
|
|
361
|
+
if (!mcpProtocolHealthy || !probe || probe.ok) return false;
|
|
362
|
+
const observed = probe.value as Partial<OpenAiSecureTunnelRuntimeObservation> | undefined;
|
|
363
|
+
return observed?.healthy === true
|
|
364
|
+
&& observed.ready === true
|
|
365
|
+
&& observed.tunnelMatches === true
|
|
366
|
+
&& observed.endpointMatches === true
|
|
367
|
+
&& (observed.controlPlaneState === 'degraded' || observed.controlPlaneState === 'unknown');
|
|
368
|
+
}
|
|
369
|
+
|
|
278
370
|
/** Bounded, release-scoped explanation for a failed watchdog verification. */
|
|
279
371
|
export interface RecoveryWatchdogDiagnosticEvidence {
|
|
280
372
|
fingerprint: string;
|
|
@@ -370,6 +462,8 @@ export interface ConfiguredRuntimeActivationResult {
|
|
|
370
462
|
noOp?: boolean;
|
|
371
463
|
detail: string;
|
|
372
464
|
staged?: StagedRuntimeRelease;
|
|
465
|
+
releaseSession?: ReleaseSession;
|
|
466
|
+
/** Legacy/home-bound recovery activation only. Fresh portable source candidates never set this during preparation. */
|
|
373
467
|
activation?: RuntimeReleaseActivationResult;
|
|
374
468
|
}
|
|
375
469
|
|
|
@@ -415,6 +509,7 @@ export function defaultPrimaryRuntimeServiceConfig(platform: NodeJS.Platform = p
|
|
|
415
509
|
|
|
416
510
|
const DEFAULT_CONFIG: Omit<RecoveryConfig, 'controllerHome'> = {
|
|
417
511
|
schemaVersion: 1,
|
|
512
|
+
installProfile: 'self-healing',
|
|
418
513
|
readOnlyTool: { name: STABLE_RECOVERY_READ_ONLY_TOOL.name, arguments: { ...STABLE_RECOVERY_READ_ONLY_TOOL.arguments } },
|
|
419
514
|
primaryRuntimeService: defaultPrimaryRuntimeServiceConfig(),
|
|
420
515
|
};
|
|
@@ -439,6 +534,7 @@ function writeJson(path: string, value: unknown): void {
|
|
|
439
534
|
|
|
440
535
|
function recoveryRoot(config: RecoveryConfig): string { return join(resolve(config.controllerHome), 'recovery'); }
|
|
441
536
|
function statePath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'state', 'known-good.json'); }
|
|
537
|
+
function runtimePinPath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'state', 'runtime-pin.json'); }
|
|
442
538
|
function lockPath(config: RecoveryConfig): string { return recoveryOperationLockPath(config.controllerHome); }
|
|
443
539
|
function auditPath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'audit', 'recovery.jsonl'); }
|
|
444
540
|
function watchdogDiagnosticPath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'state', 'watchdog-diagnostics.json'); }
|
|
@@ -488,6 +584,7 @@ export function loadRecoveryConfig(controllerHome: string, explicit?: string): R
|
|
|
488
584
|
...DEFAULT_CONFIG,
|
|
489
585
|
schemaVersion: 1,
|
|
490
586
|
controllerHome: resolve(typeof loaded.controllerHome === 'string' ? loaded.controllerHome : controllerHome),
|
|
587
|
+
installProfile: normalizeRecoveryInstallProfile(loaded.installProfile, 'self-healing'),
|
|
491
588
|
...(typeof loaded.publicMcpUrl === 'string' ? { publicMcpUrl: loaded.publicMcpUrl } : {}),
|
|
492
589
|
...(typeof loaded.recoveryPublicUrl === 'string' ? { recoveryPublicUrl: loaded.recoveryPublicUrl } : {}),
|
|
493
590
|
...(loaded.recoveryTunnelService ? { recoveryTunnelService: loaded.recoveryTunnelService } : {}),
|
|
@@ -550,33 +647,193 @@ function previousAuthorityRelease(config: RecoveryConfig): ReleaseEvidence | und
|
|
|
550
647
|
return releaseEvidence(config.controllerHome, authority?.previous, authority);
|
|
551
648
|
}
|
|
552
649
|
|
|
553
|
-
function
|
|
650
|
+
function sameReleaseIdentity(left: ReleaseEvidence, right: ReleaseEvidence): boolean {
|
|
651
|
+
return left.path === right.path
|
|
652
|
+
&& left.revision === right.revision
|
|
653
|
+
&& left.artifactIdentity === right.artifactIdentity
|
|
654
|
+
&& left.manifestSha256 === right.manifestSha256
|
|
655
|
+
&& left.workerProtocolVersion === right.workerProtocolVersion;
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
function knownGoodAttestationEvidence(config: RecoveryConfig, entry: ReleaseEvidence | undefined): ReleaseEvidence | undefined {
|
|
554
659
|
if (!entry || !entry.attestedAt || !entry.releaseFencingTokenSha256 || typeof entry.controllerHome !== 'string') return undefined;
|
|
660
|
+
// Legacy metadata-only attestations are audit history, not recoverable authority.
|
|
661
|
+
// The cheap tier requires durable bundle identity but never reads bundle contents.
|
|
662
|
+
if (!entry.recoveryBundle || entry.recoveryBundle.schemaVersion !== 1) return undefined;
|
|
555
663
|
if (resolve(entry.controllerHome) !== resolve(config.controllerHome)) return undefined;
|
|
556
664
|
const authority = releaseAuthority(config);
|
|
557
665
|
const candidates = [
|
|
558
666
|
releaseEvidence(config.controllerHome, authority?.active, authority),
|
|
559
667
|
releaseEvidence(config.controllerHome, authority?.previous, authority),
|
|
560
668
|
].filter((item): item is ReleaseEvidence => Boolean(item));
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
);
|
|
568
|
-
|
|
669
|
+
return candidates.some((release) => sameReleaseIdentity(release, entry)) ? entry : undefined;
|
|
670
|
+
}
|
|
671
|
+
|
|
672
|
+
function knownGoodEvidence(config: RecoveryConfig, entry: ReleaseEvidence | undefined): ReleaseEvidence | undefined {
|
|
673
|
+
const attested = knownGoodAttestationEvidence(config, entry);
|
|
674
|
+
if (!attested) return undefined;
|
|
675
|
+
try { inspectKnownGoodRecoveryBundle(config.controllerHome, attested); }
|
|
676
|
+
catch { return undefined; }
|
|
677
|
+
return attested;
|
|
569
678
|
}
|
|
570
679
|
|
|
571
680
|
function knownGood(config: RecoveryConfig): KnownGoodStore {
|
|
572
681
|
return json<KnownGoodStore>(statePath(config)) ?? { schemaVersion: 1, releases: [], updatedAt: new Date(0).toISOString() };
|
|
573
682
|
}
|
|
574
683
|
|
|
684
|
+
function inspectKnownGoodRecoverability(config: RecoveryConfig): {
|
|
685
|
+
available: ReleaseEvidence[];
|
|
686
|
+
unavailable: Array<{ revision: string; path: string; reason: string }>;
|
|
687
|
+
} {
|
|
688
|
+
const available: ReleaseEvidence[] = [];
|
|
689
|
+
const unavailable: Array<{ revision: string; path: string; reason: string }> = [];
|
|
690
|
+
const store = knownGood(config);
|
|
691
|
+
// Pre-bundle records remain audit history only. They are deliberately
|
|
692
|
+
// neither usable Recovery authority nor a readiness failure for an
|
|
693
|
+
// otherwise healthy Runtime, because no action can make them restorable.
|
|
694
|
+
if (store.schemaVersion !== 2) return { available, unavailable };
|
|
695
|
+
const releasesRoot = resolve(config.controllerHome, 'runtime', 'releases');
|
|
696
|
+
const authority = releaseAuthority(config);
|
|
697
|
+
const rollbackReleasePaths = new Set<string>();
|
|
698
|
+
if (authority) {
|
|
699
|
+
rollbackReleasePaths.add(resolve(dirname(authority.active.manifestPath)));
|
|
700
|
+
if (authority.previous) rollbackReleasePaths.add(resolve(dirname(authority.previous.manifestPath)));
|
|
701
|
+
}
|
|
702
|
+
for (const entry of store.releases) {
|
|
703
|
+
if (!rollbackReleasePaths.has(resolve(dirname(entry.path)))) continue;
|
|
704
|
+
try {
|
|
705
|
+
const inspected = inspectKnownGoodRecoveryBundle(config.controllerHome, entry);
|
|
706
|
+
if (dirname(inspected.releaseRoot) !== releasesRoot) throw new Error('known-good path is outside Runtime release authority');
|
|
707
|
+
available.push(entry);
|
|
708
|
+
} catch (error) {
|
|
709
|
+
unavailable.push({
|
|
710
|
+
revision: entry.revision,
|
|
711
|
+
path: entry.path,
|
|
712
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
713
|
+
});
|
|
714
|
+
}
|
|
715
|
+
}
|
|
716
|
+
return { available, unavailable };
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
function matchingKnownGoodAttestation(config: RecoveryConfig, release: ReleaseEvidence | undefined): ReleaseEvidence | undefined {
|
|
720
|
+
if (!release) return undefined;
|
|
721
|
+
return knownGoodAttestationEvidence(config, knownGood(config).releases.find((entry) => sameReleaseIdentity(entry, release)));
|
|
722
|
+
}
|
|
723
|
+
|
|
575
724
|
function matchingKnownGood(config: RecoveryConfig, release: ReleaseEvidence | undefined): ReleaseEvidence | undefined {
|
|
576
725
|
if (!release) return undefined;
|
|
577
|
-
return knownGoodEvidence(config, knownGood(config).releases.find((entry) =>
|
|
578
|
-
|
|
579
|
-
|
|
726
|
+
return knownGoodEvidence(config, knownGood(config).releases.find((entry) => sameReleaseIdentity(entry, release)));
|
|
727
|
+
}
|
|
728
|
+
|
|
729
|
+
function sameReleaseEvidenceIdentity(left: ReleaseEvidence | undefined, right: ReleaseEvidence | undefined): boolean {
|
|
730
|
+
return Boolean(
|
|
731
|
+
left
|
|
732
|
+
&& right
|
|
733
|
+
&& left.path === right.path
|
|
734
|
+
&& left.revision === right.revision
|
|
735
|
+
&& left.artifactIdentity === right.artifactIdentity
|
|
736
|
+
&& left.manifestSha256 === right.manifestSha256
|
|
737
|
+
&& left.workerProtocolVersion === right.workerProtocolVersion,
|
|
738
|
+
);
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
function liveRuntimeOwnsRelease(config: RecoveryConfig, release: ReleaseEvidence): boolean {
|
|
742
|
+
const active = activeAuthorityRelease(config);
|
|
743
|
+
const observed = observeRuntimeStatus(config.controllerHome);
|
|
744
|
+
const snapshot = observed.snapshot;
|
|
745
|
+
return Boolean(
|
|
746
|
+
sameReleaseEvidenceIdentity(active, release)
|
|
747
|
+
&& observed.running
|
|
748
|
+
&& observed.ready
|
|
749
|
+
&& !observed.stale
|
|
750
|
+
&& snapshot
|
|
751
|
+
&& snapshot.releaseId === release.revision
|
|
752
|
+
&& snapshot.artifactIdentity === release.artifactIdentity,
|
|
753
|
+
);
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
function runtimePin(config: RecoveryConfig): RuntimePinStore | undefined {
|
|
757
|
+
const path = runtimePinPath(config);
|
|
758
|
+
if (!existsSync(path)) return undefined;
|
|
759
|
+
let parsed: RuntimePinStore;
|
|
760
|
+
try { parsed = JSON.parse(readFileSync(path, 'utf8')) as RuntimePinStore; }
|
|
761
|
+
catch (error) { throw new Error(`RUNTIME_PIN_AUTHORITY_INVALID: ${error instanceof Error ? error.message : String(error)}`); }
|
|
762
|
+
const release = parsed?.release;
|
|
763
|
+
if (parsed?.schemaVersion !== 1 || !release || !release.revision?.trim() || !release.path?.trim() || !release.artifactIdentity?.trim() || !release.manifestSha256?.trim() || !Number.isInteger(release.workerProtocolVersion)) {
|
|
764
|
+
throw new Error('RUNTIME_PIN_AUTHORITY_INVALID');
|
|
765
|
+
}
|
|
766
|
+
const releasesRoot = resolve(config.controllerHome, 'runtime', 'releases');
|
|
767
|
+
const manifestPath = resolve(release.path);
|
|
768
|
+
const releaseRoot = dirname(manifestPath);
|
|
769
|
+
if (basename(manifestPath) !== 'manifest.json' || basename(releaseRoot) !== release.revision || dirname(releaseRoot) !== releasesRoot) {
|
|
770
|
+
throw new Error('RUNTIME_PIN_AUTHORITY_PATH_INVALID');
|
|
771
|
+
}
|
|
772
|
+
return parsed;
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
function releaseEvidenceForPin(
|
|
776
|
+
config: RecoveryConfig,
|
|
777
|
+
candidate: { manifest: RuntimeReleaseManifest; manifestPath: string },
|
|
778
|
+
): ReleaseEvidence {
|
|
779
|
+
const releasesRoot = resolve(config.controllerHome, 'runtime', 'releases');
|
|
780
|
+
const releaseRoot = dirname(resolve(candidate.manifestPath));
|
|
781
|
+
if (!existsSync(releasesRoot) || dirname(realpathSync(releaseRoot)) !== realpathSync(releasesRoot)) {
|
|
782
|
+
throw new Error('RUNTIME_PIN_RELEASE_OUTSIDE_CONTROLLER_RUNTIME_ROOT');
|
|
783
|
+
}
|
|
784
|
+
const expectedRepositoryId = config.primaryRuntimeSourceRepositoryId?.trim();
|
|
785
|
+
if (expectedRepositoryId && candidate.manifest.sourceRepositoryId !== expectedRepositoryId) {
|
|
786
|
+
throw new Error(`RUNTIME_PIN_SOURCE_REPOSITORY_MISMATCH: expected ${expectedRepositoryId}, got ${candidate.manifest.sourceRepositoryId ?? 'missing'}`);
|
|
787
|
+
}
|
|
788
|
+
return {
|
|
789
|
+
path: candidate.manifestPath,
|
|
790
|
+
revision: candidate.manifest.releaseId,
|
|
791
|
+
artifactIdentity: candidate.manifest.artifactIdentity,
|
|
792
|
+
manifestSha256: createHash('sha256').update(readFileSync(candidate.manifestPath)).digest('hex'),
|
|
793
|
+
workerProtocolVersion: candidate.manifest.workerProtocolVersion,
|
|
794
|
+
controllerHome: resolve(config.controllerHome),
|
|
795
|
+
...(candidate.manifest.sourceRepositoryId ? { sourceRepositoryId: candidate.manifest.sourceRepositoryId } : {}),
|
|
796
|
+
pinnedAt: new Date().toISOString(),
|
|
797
|
+
};
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
export async function pinRuntimeRelease(config: RecoveryConfig, candidateManifestPath: string, requestId?: string): Promise<Record<string, unknown>> {
|
|
801
|
+
let candidate: { manifest: RuntimeReleaseManifest; releaseRoot: string; manifestPath: string };
|
|
802
|
+
try { candidate = validateRuntimeReleaseCandidate(config, candidateManifestPath); }
|
|
803
|
+
catch (error) { return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) }; }
|
|
804
|
+
if (candidate.manifest.deploymentScope === 'portable') {
|
|
805
|
+
return {
|
|
806
|
+
ok: false,
|
|
807
|
+
attempted: false,
|
|
808
|
+
noOp: true,
|
|
809
|
+
detail: 'RUNTIME_PIN_PORTABLE_RELEASE_FORBIDDEN: portable source candidates require a verified Recovery ReleaseSession',
|
|
810
|
+
};
|
|
811
|
+
}
|
|
812
|
+
let evidence: ReleaseEvidence;
|
|
813
|
+
try { evidence = releaseEvidenceForPin(config, candidate); }
|
|
814
|
+
catch (error) { return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) }; }
|
|
815
|
+
const locked = await withLock(config, { action: 'pin_runtime_release', requestId }, async () => {
|
|
816
|
+
const current = runtimePin(config);
|
|
817
|
+
if (current?.release.revision === evidence.revision && current.release.manifestSha256 === evidence.manifestSha256) {
|
|
818
|
+
return { ok: true, attempted: false, noOp: true, detail: 'requested Runtime release is already pinned', pinned: current.release };
|
|
819
|
+
}
|
|
820
|
+
const store: RuntimePinStore = { schemaVersion: 1, release: evidence, updatedAt: new Date().toISOString() };
|
|
821
|
+
writeJson(runtimePinPath(config), store);
|
|
822
|
+
audit(config, 'runtime_release_pinned', { requestId, revision: evidence.revision, artifactIdentity: evidence.artifactIdentity });
|
|
823
|
+
return { ok: true, attempted: true, detail: 'Runtime release pinned for retention and explicit Runtime-only activation', pinned: evidence };
|
|
824
|
+
});
|
|
825
|
+
return locked.acquired ? locked.value : { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
export async function unpinRuntimeRelease(config: RecoveryConfig, requestId?: string): Promise<Record<string, unknown>> {
|
|
829
|
+
const locked = await withLock(config, { action: 'unpin_runtime_release', requestId }, async () => {
|
|
830
|
+
const current = runtimePin(config);
|
|
831
|
+
if (!current) return { ok: true, attempted: false, noOp: true, detail: 'no Runtime release is pinned' };
|
|
832
|
+
rmSync(runtimePinPath(config), { force: true });
|
|
833
|
+
audit(config, 'runtime_release_unpinned', { requestId, revision: current.release.revision, artifactIdentity: current.release.artifactIdentity });
|
|
834
|
+
return { ok: true, attempted: true, detail: 'Runtime release pin removed', unpinned: current.release };
|
|
835
|
+
});
|
|
836
|
+
return locked.acquired ? locked.value : { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
|
|
580
837
|
}
|
|
581
838
|
|
|
582
839
|
|
|
@@ -586,6 +843,75 @@ function audit(config: RecoveryConfig, event: string, detail: Record<string, unk
|
|
|
586
843
|
writeFileSync(auditPath(config), `${line}\n`, { encoding: 'utf8', mode: 0o600, flag: 'a' });
|
|
587
844
|
}
|
|
588
845
|
|
|
846
|
+
export interface RecoveryHistoryEvent {
|
|
847
|
+
at: string;
|
|
848
|
+
event: string;
|
|
849
|
+
ok?: boolean;
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
const RECOVERY_HISTORY_MAX_EVENTS = 50;
|
|
853
|
+
const RECOVERY_HISTORY_MAX_BYTES = 256 * 1024;
|
|
854
|
+
|
|
855
|
+
/**
|
|
856
|
+
* Bounded read-only projection of the canonical Recovery audit. The projection
|
|
857
|
+
* intentionally omits arbitrary detail payloads, filesystem paths, request ids,
|
|
858
|
+
* service targets, and other mechanics that the desktop does not need.
|
|
859
|
+
*/
|
|
860
|
+
export function listRecoveryHistory(
|
|
861
|
+
config: RecoveryConfig,
|
|
862
|
+
requestedLimit = 20,
|
|
863
|
+
): { events: RecoveryHistoryEvent[]; bounded: true; truncated: boolean } {
|
|
864
|
+
const limit = Number.isFinite(requestedLimit)
|
|
865
|
+
? Math.max(1, Math.min(Math.trunc(requestedLimit), RECOVERY_HISTORY_MAX_EVENTS))
|
|
866
|
+
: 20;
|
|
867
|
+
const path = auditPath(config);
|
|
868
|
+
if (!existsSync(path)) return { events: [], bounded: true, truncated: false };
|
|
869
|
+
|
|
870
|
+
const size = statSync(path).size;
|
|
871
|
+
if (size <= 0) return { events: [], bounded: true, truncated: false };
|
|
872
|
+
const start = Math.max(0, size - RECOVERY_HISTORY_MAX_BYTES);
|
|
873
|
+
const length = size - start;
|
|
874
|
+
const buffer = Buffer.alloc(length);
|
|
875
|
+
const fd = openSync(path, 'r');
|
|
876
|
+
let read = 0;
|
|
877
|
+
try {
|
|
878
|
+
while (read < length) {
|
|
879
|
+
const count = readSync(fd, buffer, read, length - read, start + read);
|
|
880
|
+
if (count <= 0) break;
|
|
881
|
+
read += count;
|
|
882
|
+
}
|
|
883
|
+
} finally {
|
|
884
|
+
closeSync(fd);
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
let text = buffer.subarray(0, read).toString('utf8');
|
|
888
|
+
if (start > 0) {
|
|
889
|
+
const firstNewline = text.indexOf('\n');
|
|
890
|
+
text = firstNewline >= 0 ? text.slice(firstNewline + 1) : '';
|
|
891
|
+
}
|
|
892
|
+
const parsed: RecoveryHistoryEvent[] = [];
|
|
893
|
+
for (const line of text.split('\n')) {
|
|
894
|
+
const trimmed = line.trim();
|
|
895
|
+
if (!trimmed) continue;
|
|
896
|
+
try {
|
|
897
|
+
const value = JSON.parse(trimmed) as { at?: unknown; event?: unknown; detail?: unknown };
|
|
898
|
+
if (typeof value.at !== 'string' || typeof value.event !== 'string') continue;
|
|
899
|
+
const detail = value.detail && typeof value.detail === 'object' && !Array.isArray(value.detail)
|
|
900
|
+
? value.detail as Record<string, unknown>
|
|
901
|
+
: undefined;
|
|
902
|
+
parsed.push({
|
|
903
|
+
at: value.at,
|
|
904
|
+
event: value.event,
|
|
905
|
+
...(typeof detail?.ok === 'boolean' ? { ok: detail.ok } : {}),
|
|
906
|
+
});
|
|
907
|
+
} catch {
|
|
908
|
+
// A partially written final line is not a durable audit event yet.
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
const truncated = start > 0 || parsed.length > limit;
|
|
912
|
+
return { events: parsed.slice(-limit).reverse(), bounded: true, truncated };
|
|
913
|
+
}
|
|
914
|
+
|
|
589
915
|
const WATCHDOG_DIAGNOSTIC_LIMIT = 32;
|
|
590
916
|
|
|
591
917
|
function watchdogProbeComponent(name: string): RecoveryWatchdogDiagnosticEvidence['components'][number] {
|
|
@@ -649,7 +975,7 @@ function recoveryLockOwnerAlive(lock: RecoveryLock): boolean {
|
|
|
649
975
|
if (!pidAlive(lock.pid)) return false;
|
|
650
976
|
if (!lock.processStartTime) return true;
|
|
651
977
|
const observed = processStartTime(lock.pid);
|
|
652
|
-
return observed ===
|
|
978
|
+
return observed === lock.processStartTime;
|
|
653
979
|
}
|
|
654
980
|
|
|
655
981
|
function recoveryLockOwnerAttributable(owner: RecoveryLock): boolean {
|
|
@@ -907,7 +1233,16 @@ function canonicalRuntimeSafeForTargetedConnectorRecovery(verified: VerifyResult
|
|
|
907
1233
|
}
|
|
908
1234
|
|
|
909
1235
|
function mainToken(config: RecoveryConfig): string | undefined {
|
|
910
|
-
const
|
|
1236
|
+
const explicit = config.mainMcpTokenFile?.trim();
|
|
1237
|
+
const candidate = explicit || join(config.controllerHome, 'mcp', 'mcp.tokens.json');
|
|
1238
|
+
if (explicit) {
|
|
1239
|
+
try {
|
|
1240
|
+
const raw = readFileSync(candidate, 'utf8').trim();
|
|
1241
|
+
if (raw.length >= 24 && !raw.startsWith('{')) return raw;
|
|
1242
|
+
} catch {
|
|
1243
|
+
return undefined;
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
911
1246
|
const parsed = json<{ bearerToken?: unknown }>(candidate);
|
|
912
1247
|
return typeof parsed?.bearerToken === 'string' && parsed.bearerToken.length >= 24 ? parsed.bearerToken : undefined;
|
|
913
1248
|
}
|
|
@@ -1003,6 +1338,20 @@ function runtimeHealthEndpoint(endpoint: string): string {
|
|
|
1003
1338
|
return url.toString();
|
|
1004
1339
|
}
|
|
1005
1340
|
|
|
1341
|
+
/**
|
|
1342
|
+
* tunnel-client materialises its own state directory relative to the process
|
|
1343
|
+
* working directory. Persistent Forge services run under launchd/systemd with a
|
|
1344
|
+
* read-only working directory, so a Forge-owned tunnel reconnect must run from a
|
|
1345
|
+
* writable Forge-managed directory. Otherwise the reconnect fails with
|
|
1346
|
+
* `mkdir .tunnel-client: read-only file system`, the dedicated Recovery
|
|
1347
|
+
* transport can never be repaired, and the Recovery connector stays dark.
|
|
1348
|
+
*/
|
|
1349
|
+
function tunnelClientWorkingDirectory(controllerHome: string): string {
|
|
1350
|
+
const directory = join(resolve(controllerHome), 'tmp', 'tunnel-client');
|
|
1351
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
1352
|
+
return directory;
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1006
1355
|
async function observeOpenAiTunnelRuntime(
|
|
1007
1356
|
configured: OpenAiSecureTunnelServiceConfig,
|
|
1008
1357
|
runCommand: CommandRunner = command,
|
|
@@ -1013,6 +1362,8 @@ async function observeOpenAiTunnelRuntime(
|
|
|
1013
1362
|
} catch (error) {
|
|
1014
1363
|
return {
|
|
1015
1364
|
ok: false, running: false, healthy: false, ready: false, tunnelMatches: false, endpointMatches: false,
|
|
1365
|
+
controlPlaneState: 'unknown',
|
|
1366
|
+
controlPlaneDetail: 'tunnel runtime configuration could not be observed',
|
|
1016
1367
|
alias: configured.alias, tunnelId: configured.tunnelId,
|
|
1017
1368
|
detail: error instanceof Error ? error.message : 'OpenAI tunnel runtime configuration is invalid',
|
|
1018
1369
|
};
|
|
@@ -1023,6 +1374,8 @@ async function observeOpenAiTunnelRuntime(
|
|
|
1023
1374
|
if (local) return local;
|
|
1024
1375
|
return {
|
|
1025
1376
|
ok: false, running: false, healthy: false, ready: false, tunnelMatches: false, endpointMatches: false,
|
|
1377
|
+
controlPlaneState: 'unknown',
|
|
1378
|
+
controlPlaneDetail: 'tunnel-client status produced no remote control-plane evidence',
|
|
1026
1379
|
alias: configured.alias, tunnelId: configured.tunnelId,
|
|
1027
1380
|
detail: status.stderr.trim() || 'OpenAI tunnel runtime status is unavailable',
|
|
1028
1381
|
};
|
|
@@ -1069,7 +1422,11 @@ export async function observeOpenAiTunnelLocalHealthFallback(
|
|
|
1069
1422
|
try { return (await request(url.toString(), { signal: controller.signal })).ok; } catch { return false; } finally { clearTimeout(timer); }
|
|
1070
1423
|
};
|
|
1071
1424
|
const [healthy, ready] = await Promise.all([probe('/healthz'), probe('/readyz')]);
|
|
1072
|
-
const
|
|
1425
|
+
const localReady = healthy && ready && identityMatches && endpointMatches;
|
|
1426
|
+
// Loopback health proves only that the local tunnel process and MCP binding are
|
|
1427
|
+
// alive. It must never stand in for the remote OpenAI control plane when the
|
|
1428
|
+
// status command itself could not establish that path.
|
|
1429
|
+
const ok = false;
|
|
1073
1430
|
const failures: string[] = [];
|
|
1074
1431
|
if (!identityMatches) failures.push('profile tunnel id does not match');
|
|
1075
1432
|
if (!endpointMatches) failures.push('profile MCP endpoint does not match');
|
|
@@ -1082,11 +1439,13 @@ export async function observeOpenAiTunnelLocalHealthFallback(
|
|
|
1082
1439
|
ready,
|
|
1083
1440
|
tunnelMatches: identityMatches,
|
|
1084
1441
|
endpointMatches,
|
|
1442
|
+
controlPlaneState: 'unknown',
|
|
1443
|
+
controlPlaneDetail: 'tunnel-client status was unavailable; only loopback health was observed',
|
|
1085
1444
|
alias: configured.alias,
|
|
1086
1445
|
tunnelId: configured.tunnelId,
|
|
1087
1446
|
...(identityMatches ? { observedTunnelId: configured.tunnelId } : {}),
|
|
1088
|
-
detail:
|
|
1089
|
-
? `managed runtime ${configured.alias} is locally healthy and ready for ${configured.tunnelId}
|
|
1447
|
+
detail: localReady
|
|
1448
|
+
? `managed runtime ${configured.alias} is locally healthy and ready for ${configured.tunnelId}, but the OpenAI control plane is unverified because status is unavailable`
|
|
1090
1449
|
: failures.join('; ') || 'OpenAI tunnel local health fallback failed',
|
|
1091
1450
|
profilePath,
|
|
1092
1451
|
};
|
|
@@ -1104,11 +1463,22 @@ async function observeOpenAiRecoveryTunnel(
|
|
|
1104
1463
|
async function ensureOpenAiTunnelRuntimeStarted(
|
|
1105
1464
|
configured: OpenAiSecureTunnelServiceConfig,
|
|
1106
1465
|
runCommand: CommandRunner = command,
|
|
1466
|
+
workingDirectory?: string,
|
|
1107
1467
|
): Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string; serviceLabel: string; serviceTarget: string }> {
|
|
1108
1468
|
const serviceLabel = configured.alias;
|
|
1109
1469
|
const serviceTarget = `tunnel-client:${configured.alias}`;
|
|
1110
1470
|
const observed = await observeOpenAiTunnelRuntime(configured, runCommand);
|
|
1111
1471
|
if (observed.ok) return { ok: true, attempted: false, noOp: true, detail: observed.detail, serviceLabel, serviceTarget };
|
|
1472
|
+
if (observed.healthy && observed.ready && observed.tunnelMatches && observed.endpointMatches && observed.controlPlaneState !== 'ready') {
|
|
1473
|
+
return {
|
|
1474
|
+
ok: false,
|
|
1475
|
+
attempted: false,
|
|
1476
|
+
noOp: true,
|
|
1477
|
+
detail: `OpenAI Secure MCP Tunnel runtime ${configured.alias} is locally ready but its remote control plane is ${observed.controlPlaneState}; refusing a local reconnect for an upstream transport failure: ${observed.controlPlaneDetail}`,
|
|
1478
|
+
serviceLabel,
|
|
1479
|
+
serviceTarget,
|
|
1480
|
+
};
|
|
1481
|
+
}
|
|
1112
1482
|
if (observed.observedTunnelId && !observed.tunnelMatches) {
|
|
1113
1483
|
return { ok: false, attempted: false, noOp: true, detail: `OpenAI tunnel alias ${configured.alias} is already bound to a different tunnel id`, serviceLabel, serviceTarget };
|
|
1114
1484
|
}
|
|
@@ -1129,7 +1499,7 @@ async function ensureOpenAiTunnelRuntimeStarted(
|
|
|
1129
1499
|
} catch (error) {
|
|
1130
1500
|
return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : 'OpenAI tunnel runtime configuration is invalid', serviceLabel, serviceTarget };
|
|
1131
1501
|
}
|
|
1132
|
-
const connected = await runCommand('tunnel-client', args, Math.max(30_000, configured.postRestartVerifyTimeoutMs ?? 20_000));
|
|
1502
|
+
const connected = await runCommand('tunnel-client', args, Math.max(30_000, configured.postRestartVerifyTimeoutMs ?? 20_000), workingDirectory ? { cwd: workingDirectory } : {});
|
|
1133
1503
|
return connected.ok
|
|
1134
1504
|
? { ok: true, attempted: true, detail: `OpenAI Secure MCP Tunnel runtime ${configured.alias} connect dispatched`, serviceLabel, serviceTarget }
|
|
1135
1505
|
: { ok: false, attempted: true, detail: `OpenAI Secure MCP Tunnel runtime connect failed: ${connected.stderr || connected.stdout || connected.status}`, serviceLabel, serviceTarget };
|
|
@@ -1143,6 +1513,140 @@ async function probeOpenAiRecoveryTunnel(
|
|
|
1143
1513
|
return observed ? { ok: observed.ok, detail: observed.detail, value: observed } : undefined;
|
|
1144
1514
|
}
|
|
1145
1515
|
|
|
1516
|
+
function openAiTunnelObservationFromProbe(
|
|
1517
|
+
probe: { value?: unknown } | undefined,
|
|
1518
|
+
): OpenAiSecureTunnelRuntimeObservation | undefined {
|
|
1519
|
+
const value = probe?.value;
|
|
1520
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
|
|
1521
|
+
const observed = value as Partial<OpenAiSecureTunnelRuntimeObservation>;
|
|
1522
|
+
if (
|
|
1523
|
+
typeof observed.alias !== 'string'
|
|
1524
|
+
|| typeof observed.tunnelId !== 'string'
|
|
1525
|
+
|| typeof observed.healthy !== 'boolean'
|
|
1526
|
+
|| typeof observed.ready !== 'boolean'
|
|
1527
|
+
|| typeof observed.tunnelMatches !== 'boolean'
|
|
1528
|
+
|| typeof observed.endpointMatches !== 'boolean'
|
|
1529
|
+
|| !['ready', 'degraded', 'unknown'].includes(String(observed.controlPlaneState))
|
|
1530
|
+
) return undefined;
|
|
1531
|
+
return observed as OpenAiSecureTunnelRuntimeObservation;
|
|
1532
|
+
}
|
|
1533
|
+
|
|
1534
|
+
function isUpstreamOpenAiControlPlaneIncident(observed: OpenAiSecureTunnelRuntimeObservation | undefined): boolean {
|
|
1535
|
+
return Boolean(
|
|
1536
|
+
observed
|
|
1537
|
+
&& observed.healthy
|
|
1538
|
+
&& observed.ready
|
|
1539
|
+
&& observed.tunnelMatches
|
|
1540
|
+
&& observed.endpointMatches
|
|
1541
|
+
&& observed.controlPlaneState !== 'ready'
|
|
1542
|
+
);
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
async function probeOpenAiApiReachability(): Promise<{ ok: boolean; detail: string; status?: number }> {
|
|
1546
|
+
const controller = new AbortController();
|
|
1547
|
+
const timer = setTimeout(() => controller.abort('RECOVERY_OPENAI_API_REACHABILITY_TIMEOUT'), 4_000);
|
|
1548
|
+
try {
|
|
1549
|
+
const response = await fetch('https://api.openai.com/v1/models', {
|
|
1550
|
+
method: 'GET',
|
|
1551
|
+
headers: { accept: 'application/json' },
|
|
1552
|
+
signal: controller.signal,
|
|
1553
|
+
});
|
|
1554
|
+
return {
|
|
1555
|
+
ok: true,
|
|
1556
|
+
status: response.status,
|
|
1557
|
+
detail: `api.openai.com returned HTTP ${response.status}; DNS/TCP/TLS/HTTP path is reachable`,
|
|
1558
|
+
};
|
|
1559
|
+
} catch (error) {
|
|
1560
|
+
const cause = error instanceof Error && error.cause instanceof Error ? `; cause=${error.cause.message}` : '';
|
|
1561
|
+
return { ok: false, detail: `${error instanceof Error ? error.message : String(error)}${cause}`.slice(0, 500) };
|
|
1562
|
+
} finally {
|
|
1563
|
+
clearTimeout(timer);
|
|
1564
|
+
}
|
|
1565
|
+
}
|
|
1566
|
+
|
|
1567
|
+
async function observeRecentMacNetworkExtensionChanges(
|
|
1568
|
+
runCommand: CommandRunner = command,
|
|
1569
|
+
): Promise<{ ok: boolean; detail: string } | undefined> {
|
|
1570
|
+
if (process.platform !== 'darwin') return undefined;
|
|
1571
|
+
const result = await runCommand('/usr/bin/log', [
|
|
1572
|
+
'show', '--last', '90s', '--style', 'compact', '--predicate',
|
|
1573
|
+
'process == "nesessionmanager" AND (eventMessage CONTAINS[c] "configuration changed" OR eventMessage CONTAINS[c] "neconfigurationchanged")',
|
|
1574
|
+
], 3_000, { maxOutputBytes: 12_000 });
|
|
1575
|
+
const output = (result.stdout || result.stderr).trim();
|
|
1576
|
+
return {
|
|
1577
|
+
ok: result.ok,
|
|
1578
|
+
detail: output ? output.slice(-4_000) : (result.ok ? 'no recent Network Extension configuration changes observed' : 'Network Extension event query failed'),
|
|
1579
|
+
};
|
|
1580
|
+
}
|
|
1581
|
+
|
|
1582
|
+
async function persistOpenAiTransportIncidentEvidence(input: {
|
|
1583
|
+
config: RecoveryConfig;
|
|
1584
|
+
verify: VerifyResult;
|
|
1585
|
+
runtimeObservation: ReturnType<typeof observeRuntimeStatus>;
|
|
1586
|
+
recoveryTunnel: OpenAiSecureTunnelRuntimeObservation;
|
|
1587
|
+
}): Promise<void> {
|
|
1588
|
+
const primaryTunnel = configuredPrimaryPublicTunnel(input.config);
|
|
1589
|
+
const [openAiApi, connectorActivity, macNetworkExtension, primaryTunnelObservation] = await Promise.all([
|
|
1590
|
+
probeOpenAiApiReachability(),
|
|
1591
|
+
probeReleaseCutoverReadiness(input.config, { timeoutMs: 1_000 }),
|
|
1592
|
+
observeRecentMacNetworkExtensionChanges(),
|
|
1593
|
+
primaryTunnel?.platform === 'openai-secure-tunnel'
|
|
1594
|
+
? observeOpenAiTunnelRuntime(primaryTunnel)
|
|
1595
|
+
: Promise.resolve(undefined),
|
|
1596
|
+
]);
|
|
1597
|
+
const activeReleaseSessions = listCompleteReleaseSessions(input.config.controllerHome).sessions
|
|
1598
|
+
.filter((session) => !['known_good', 'rolled_back', 'failed'].includes(session.phase))
|
|
1599
|
+
.sort((left, right) => Date.parse(right.updatedAt) - Date.parse(left.updatedAt))
|
|
1600
|
+
.slice(0, 3)
|
|
1601
|
+
.map((session) => ({ sessionId: session.sessionId, phase: session.phase, revision: session.revision, updatedAt: session.updatedAt }));
|
|
1602
|
+
const incidentDetail = [
|
|
1603
|
+
`alias=${input.recoveryTunnel.alias}`,
|
|
1604
|
+
`tunnel=${input.recoveryTunnel.tunnelId}`,
|
|
1605
|
+
`client=${input.recoveryTunnel.clientInstanceId ?? 'unknown'}`,
|
|
1606
|
+
`controlPlane=${input.recoveryTunnel.controlPlaneState}`,
|
|
1607
|
+
input.recoveryTunnel.controlPlaneDetail,
|
|
1608
|
+
].join('; ');
|
|
1609
|
+
const diagnostic = persistWatchdogDiagnosticEvidence(input.config, {
|
|
1610
|
+
...input.verify,
|
|
1611
|
+
ok: false,
|
|
1612
|
+
probes: {
|
|
1613
|
+
recovery_tunnel_runtime: { ok: false, detail: incidentDetail },
|
|
1614
|
+
},
|
|
1615
|
+
});
|
|
1616
|
+
audit(input.config, 'network_transport_incident_observed', {
|
|
1617
|
+
diagnosticFingerprint: diagnostic?.fingerprint,
|
|
1618
|
+
recoveryTunnel: {
|
|
1619
|
+
alias: input.recoveryTunnel.alias,
|
|
1620
|
+
tunnelId: input.recoveryTunnel.tunnelId,
|
|
1621
|
+
clientInstanceId: input.recoveryTunnel.clientInstanceId,
|
|
1622
|
+
controlPlaneState: input.recoveryTunnel.controlPlaneState,
|
|
1623
|
+
controlPlaneDetail: input.recoveryTunnel.controlPlaneDetail,
|
|
1624
|
+
localHealthy: input.recoveryTunnel.healthy,
|
|
1625
|
+
localReady: input.recoveryTunnel.ready,
|
|
1626
|
+
},
|
|
1627
|
+
primaryTunnel: primaryTunnelObservation ? {
|
|
1628
|
+
alias: primaryTunnelObservation.alias,
|
|
1629
|
+
tunnelId: primaryTunnelObservation.tunnelId,
|
|
1630
|
+
clientInstanceId: primaryTunnelObservation.clientInstanceId,
|
|
1631
|
+
controlPlaneState: primaryTunnelObservation.controlPlaneState,
|
|
1632
|
+
controlPlaneDetail: primaryTunnelObservation.controlPlaneDetail,
|
|
1633
|
+
localHealthy: primaryTunnelObservation.healthy,
|
|
1634
|
+
localReady: primaryTunnelObservation.ready,
|
|
1635
|
+
} : undefined,
|
|
1636
|
+
openAiApi,
|
|
1637
|
+
connectorActivity,
|
|
1638
|
+
runtime: {
|
|
1639
|
+
releaseId: input.runtimeObservation.snapshot?.releaseId,
|
|
1640
|
+
runtimeInstanceId: input.runtimeObservation.snapshot?.runtimeInstanceId,
|
|
1641
|
+
pid: input.runtimeObservation.snapshot?.pid,
|
|
1642
|
+
ready: input.runtimeObservation.ready,
|
|
1643
|
+
stale: input.runtimeObservation.stale,
|
|
1644
|
+
},
|
|
1645
|
+
releaseSessions: activeReleaseSessions,
|
|
1646
|
+
macNetworkExtension,
|
|
1647
|
+
});
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1146
1650
|
async function probeMcp(config: RecoveryConfig, transport: RecoveryHttpTransport): Promise<Record<string, { ok: boolean; detail: string; value?: unknown }>> {
|
|
1147
1651
|
const url = config.publicMcpUrl ?? runtimeEndpoint(config);
|
|
1148
1652
|
if (!url) return { mcp_initialize: { ok: false, detail: 'canonical Runtime MCP endpoint is unavailable' } };
|
|
@@ -1230,6 +1734,24 @@ export async function verifyStableRuntime(
|
|
|
1230
1734
|
: 'canonical Runtime is not running',
|
|
1231
1735
|
},
|
|
1232
1736
|
};
|
|
1737
|
+
const knownGoodRecoverability = inspectKnownGoodRecoverability(config);
|
|
1738
|
+
probes.recovery_known_good_recoverability = {
|
|
1739
|
+
ok: knownGoodRecoverability.unavailable.length === 0,
|
|
1740
|
+
detail: knownGoodRecoverability.unavailable.length === 0
|
|
1741
|
+
? `${knownGoodRecoverability.available.length} bounded known-good Runtime release(s) are physically recoverable`
|
|
1742
|
+
: `${knownGoodRecoverability.unavailable.length} known-good attestation(s) no longer have a matching immutable Runtime release`,
|
|
1743
|
+
value: knownGoodRecoverability,
|
|
1744
|
+
};
|
|
1745
|
+
if (!observation.running || !observation.ready) {
|
|
1746
|
+
const startupFailure = readRuntimeStartupFailureEvidence(config.controllerHome);
|
|
1747
|
+
if (startupFailure && (!active || !startupFailure.releaseId || startupFailure.releaseId === active.revision)) {
|
|
1748
|
+
probes.runtime_startup_failure = {
|
|
1749
|
+
ok: false,
|
|
1750
|
+
detail: `latest Runtime startup failed during ${startupFailure.stage}: ${startupFailure.reasonCode}${startupFailure.message ? ` — ${startupFailure.message}` : ''}`,
|
|
1751
|
+
value: startupFailure,
|
|
1752
|
+
};
|
|
1753
|
+
}
|
|
1754
|
+
}
|
|
1233
1755
|
const endpoint = observation.snapshot?.endpoint;
|
|
1234
1756
|
probes.active_gateway = endpoint
|
|
1235
1757
|
? await probe(transport, runtimeHealthEndpoint(endpoint))
|
|
@@ -1250,7 +1772,10 @@ export async function verifyStableRuntime(
|
|
|
1250
1772
|
const primaryConnectorLocal = await probePrimaryConnectorLocal(config, transport);
|
|
1251
1773
|
if (primaryConnectorLocal) probes.primary_connector_local = primaryConnectorLocal;
|
|
1252
1774
|
if (config.gateway) probes.recovery_gateway = await probe(transport, `http://${config.gateway.host}:${config.gateway.port}/health`);
|
|
1253
|
-
const watchdogHealth = observeRecoveryWatchdogHealth(
|
|
1775
|
+
const watchdogHealth = observeRecoveryWatchdogHealth(
|
|
1776
|
+
config.controllerHome,
|
|
1777
|
+
config.installProfile === 'self-healing',
|
|
1778
|
+
);
|
|
1254
1779
|
probes.recovery_watchdog = {
|
|
1255
1780
|
ok: watchdogHealth.ok,
|
|
1256
1781
|
detail: watchdogHealth.detail,
|
|
@@ -1267,8 +1792,24 @@ export async function verifyStableRuntime(
|
|
|
1267
1792
|
const recoveryTunnelRuntime = await probeOpenAiRecoveryTunnel(config);
|
|
1268
1793
|
if (recoveryTunnelRuntime) probes.recovery_tunnel_runtime = recoveryTunnelRuntime;
|
|
1269
1794
|
if (options.probeMcpProtocol !== false) Object.assign(probes, await probeMcp(config, transport));
|
|
1795
|
+
const mcpProtocolProbeNames = ['mcp_initialize', 'mcp_initialized_notification', 'mcp_tools_list', 'mcp_read_only_call', 'mcp_session_close'] as const;
|
|
1796
|
+
const mcpProtocolHealthy = options.probeMcpProtocol !== false
|
|
1797
|
+
&& mcpProtocolProbeNames.every((name) => probes[name]?.ok === true);
|
|
1270
1798
|
const coreChecks = Object.entries(probes)
|
|
1271
1799
|
.filter(([name]) => !name.startsWith('recovery_'))
|
|
1800
|
+
// The unauthenticated public initialize probe remains diagnostic and is the
|
|
1801
|
+
// authority for dedicated external/tunnel verification. Once the stronger
|
|
1802
|
+
// authenticated MCP session completes end-to-end, that raw reachability
|
|
1803
|
+
// probe cannot independently veto full stable Runtime verification.
|
|
1804
|
+
.filter(([name]) => !(name === 'external_mcp_http' && mcpProtocolHealthy))
|
|
1805
|
+
// The OpenAI tunnel control-plane lookup is likewise diagnostic when the
|
|
1806
|
+
// local tunnel identity/readiness is exact and the stronger public MCP data
|
|
1807
|
+
// plane has already completed end-to-end. Real tunnel/data-plane failures
|
|
1808
|
+
// still remain in coreChecks and fail closed.
|
|
1809
|
+
.filter(([name, entry]) => !(
|
|
1810
|
+
name === 'primary_tunnel_runtime'
|
|
1811
|
+
&& isPrimaryTunnelControlPlaneOnlyDegradation(entry, mcpProtocolHealthy)
|
|
1812
|
+
))
|
|
1272
1813
|
.every(([, entry]) => entry.ok);
|
|
1273
1814
|
const runtimeHealthy = observation.running && observation.ready && !observation.stale;
|
|
1274
1815
|
const coherent = Boolean(
|
|
@@ -1309,6 +1850,139 @@ export async function verifyStableRuntime(
|
|
|
1309
1850
|
});
|
|
1310
1851
|
return result;
|
|
1311
1852
|
}
|
|
1853
|
+
function recoveryBundleId(): string {
|
|
1854
|
+
return `attestation-${Date.now()}-${randomUUID().replaceAll('-', '').slice(0, 16)}`;
|
|
1855
|
+
}
|
|
1856
|
+
|
|
1857
|
+
/**
|
|
1858
|
+
* Build a self-contained, Recovery-owned restore point before publishing its
|
|
1859
|
+
* attestation. The copy is intentionally made while the full Runtime is
|
|
1860
|
+
* still verified and fenced; a later crash cannot turn a metadata-only record
|
|
1861
|
+
* into a false known-good authority.
|
|
1862
|
+
*/
|
|
1863
|
+
function createKnownGoodRecoveryBundle(config: RecoveryConfig): KnownGoodRecoveryBundle {
|
|
1864
|
+
const attestationId = recoveryBundleId();
|
|
1865
|
+
const root = join(knownGoodRecoveryBundleRoot(config.controllerHome), attestationId);
|
|
1866
|
+
const databasePath = join(root, 'controller.sqlite');
|
|
1867
|
+
const serviceContractPath = join(root, 'service-contract.json');
|
|
1868
|
+
try {
|
|
1869
|
+
const database = backupControlPlaneDatabase(config.controllerHome, databasePath);
|
|
1870
|
+
const servicePaths = forgeRuntimeServicePaths(config.controllerHome);
|
|
1871
|
+
const serviceConfig = readForgeRuntimeServiceConfig(servicePaths.configPath);
|
|
1872
|
+
const serviceContract = {
|
|
1873
|
+
schemaVersion: 1 as const,
|
|
1874
|
+
configPath: servicePaths.configPath,
|
|
1875
|
+
serviceConfig,
|
|
1876
|
+
// The label and fixed executable path are derived, never copied from a
|
|
1877
|
+
// mutable launchd plist. Recovery can rebuild that contract from this
|
|
1878
|
+
// declarative service configuration plus the immutable release.
|
|
1879
|
+
label: servicePaths.label,
|
|
1880
|
+
activeEntrypointPath: servicePaths.activeEntrypointPath,
|
|
1881
|
+
createdAt: new Date().toISOString(),
|
|
1882
|
+
};
|
|
1883
|
+
writeJson(serviceContractPath, serviceContract);
|
|
1884
|
+
return {
|
|
1885
|
+
schemaVersion: 1,
|
|
1886
|
+
attestationId,
|
|
1887
|
+
root,
|
|
1888
|
+
database: {
|
|
1889
|
+
path: databasePath,
|
|
1890
|
+
sha256: sha256FileBounded(databasePath),
|
|
1891
|
+
schemaVersion: database.schemaVersion,
|
|
1892
|
+
recordCount: database.recordCount,
|
|
1893
|
+
auditEventCount: database.auditEventCount,
|
|
1894
|
+
},
|
|
1895
|
+
serviceContract: { path: serviceContractPath, sha256: sha256FileBounded(serviceContractPath) },
|
|
1896
|
+
createdAt: new Date().toISOString(),
|
|
1897
|
+
};
|
|
1898
|
+
} catch (error) {
|
|
1899
|
+
rmSync(root, { recursive: true, force: true });
|
|
1900
|
+
throw error;
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
function pruneRetiredKnownGoodRecoveryBundles(config: RecoveryConfig, releases: readonly ReleaseEvidence[]): void {
|
|
1905
|
+
const root = knownGoodRecoveryBundleRoot(config.controllerHome);
|
|
1906
|
+
if (!existsSync(root)) return;
|
|
1907
|
+
const retained = new Set(releases.flatMap((release) => release.recoveryBundle ? [resolve(release.recoveryBundle.root)] : []));
|
|
1908
|
+
for (const entry of readdirSync(root, { withFileTypes: true })) {
|
|
1909
|
+
if (!entry.isDirectory()) continue;
|
|
1910
|
+
const path = resolve(root, entry.name);
|
|
1911
|
+
if (dirname(path) !== resolve(root) || retained.has(path)) continue;
|
|
1912
|
+
rmSync(path, { recursive: true, force: true });
|
|
1913
|
+
}
|
|
1914
|
+
}
|
|
1915
|
+
|
|
1916
|
+
/**
|
|
1917
|
+
* Reconcile Recovery's restore-point ledger with the only releases that the
|
|
1918
|
+
* committed Runtime authority can actually roll back to. Historical
|
|
1919
|
+
* attestations are evidence, not a second rollback authority, and must not
|
|
1920
|
+
* keep full SQLite bundles and immutable Runtime trees alive indefinitely.
|
|
1921
|
+
*/
|
|
1922
|
+
function reconcileKnownGoodRecoveryRetention(config: RecoveryConfig): void {
|
|
1923
|
+
const store = knownGood(config);
|
|
1924
|
+
if (store.schemaVersion !== 2 || store.releases.length === 0) return;
|
|
1925
|
+
const authority = releaseAuthority(config);
|
|
1926
|
+
if (!authority) throw new Error('KNOWN_GOOD_RETENTION_RUNTIME_AUTHORITY_MISSING');
|
|
1927
|
+
const rollbackReleases = [
|
|
1928
|
+
releaseEvidence(config.controllerHome, authority.active, authority),
|
|
1929
|
+
...(authority.previous ? [releaseEvidence(config.controllerHome, authority.previous, authority)] : []),
|
|
1930
|
+
].filter((release): release is ReleaseEvidence => Boolean(release));
|
|
1931
|
+
const retained: ReleaseEvidence[] = [];
|
|
1932
|
+
for (const entry of store.releases) {
|
|
1933
|
+
const rollbackEntry = rollbackReleases.find((release) => sameReleaseIdentity(release, entry));
|
|
1934
|
+
if (!rollbackEntry) continue;
|
|
1935
|
+
try { inspectKnownGoodRecoveryBundle(config.controllerHome, entry); }
|
|
1936
|
+
catch (error) {
|
|
1937
|
+
throw new Error(`KNOWN_GOOD_RETENTION_AUTHORITY_INVALID: ${error instanceof Error ? error.message : String(error)}`);
|
|
1938
|
+
}
|
|
1939
|
+
retained.push(entry);
|
|
1940
|
+
}
|
|
1941
|
+
if (retained.length === store.releases.length) return;
|
|
1942
|
+
writeJson(statePath(config), { schemaVersion: 2, releases: retained, updatedAt: new Date().toISOString() } satisfies KnownGoodStore);
|
|
1943
|
+
pruneRetiredKnownGoodRecoveryBundles(config, retained);
|
|
1944
|
+
}
|
|
1945
|
+
|
|
1946
|
+
interface ReleaseSessionStorageEstimate {
|
|
1947
|
+
requiredBytes: number;
|
|
1948
|
+
reserveBytes: number;
|
|
1949
|
+
}
|
|
1950
|
+
|
|
1951
|
+
function estimateReleaseSessionCandidateBytes(config: RecoveryConfig, manifestPath: string): ReleaseSessionStorageEstimate {
|
|
1952
|
+
const releaseRoot = dirname(resolve(manifestPath));
|
|
1953
|
+
const releaseMeasurement = measureReclaimablePath(releaseRoot, 100_000);
|
|
1954
|
+
const sourceRoot = config.primaryRuntimeSourceRoot?.trim();
|
|
1955
|
+
if (!sourceRoot) throw new Error('RELEASE_SESSION_STORAGE_SOURCE_UNAVAILABLE');
|
|
1956
|
+
const resolvedSourceRoot = resolve(sourceRoot);
|
|
1957
|
+
const sourceMeasurement = measureReclaimablePath(resolvedSourceRoot, 100_000);
|
|
1958
|
+
const dependencyRoot = join(resolvedSourceRoot, 'node_modules');
|
|
1959
|
+
const dependencyMeasurement = existsSync(dependencyRoot)
|
|
1960
|
+
? measureReclaimablePath(realpathSync(dependencyRoot), 1_000_000)
|
|
1961
|
+
: { bytes: 0, entries: 0, complete: true };
|
|
1962
|
+
const databasePath = join(resolve(config.controllerHome), 'control-plane.sqlite');
|
|
1963
|
+
let databaseBytes: number;
|
|
1964
|
+
try { databaseBytes = Math.max(0, statSync(databasePath).size); }
|
|
1965
|
+
catch (error) { throw new Error(`RELEASE_SESSION_STORAGE_DATABASE_UNAVAILABLE: ${error instanceof Error ? error.message : String(error)}`); }
|
|
1966
|
+
if (!releaseMeasurement.complete) throw new Error('RELEASE_SESSION_STORAGE_RELEASE_ESTIMATE_INCOMPLETE');
|
|
1967
|
+
if (!sourceMeasurement.complete) throw new Error('RELEASE_SESSION_STORAGE_SOURCE_ESTIMATE_INCOMPLETE');
|
|
1968
|
+
if (!dependencyMeasurement.complete) throw new Error('RELEASE_SESSION_STORAGE_DEPENDENCY_ESTIMATE_INCOMPLETE');
|
|
1969
|
+
// The candidate is rebuilt from the frozen checkout, not copied from the
|
|
1970
|
+
// active release. It creates a detached worktree, a dependency-inclusive
|
|
1971
|
+
// package snapshot, compiled artifacts, and a consistent SQLite snapshot.
|
|
1972
|
+
// Account for each independently rather than treating the active tree as a
|
|
1973
|
+
// proxy for the build's incremental footprint.
|
|
1974
|
+
const sourceBytes = releaseMeasurement.bytes
|
|
1975
|
+
+ sourceMeasurement.bytes
|
|
1976
|
+
+ dependencyMeasurement.bytes
|
|
1977
|
+
+ databaseBytes;
|
|
1978
|
+
if (!Number.isFinite(sourceBytes) || sourceBytes < 1) throw new Error('RELEASE_SESSION_STORAGE_ESTIMATE_INVALID');
|
|
1979
|
+
// Keep an equally sized reserve for compiler/materialization variance and
|
|
1980
|
+
// atomic promotion, rather than treating the global warning threshold as
|
|
1981
|
+
// rollback capacity.
|
|
1982
|
+
const measuredBytes = Math.max(1, Math.ceil(sourceBytes));
|
|
1983
|
+
return { requiredBytes: measuredBytes, reserveBytes: measuredBytes };
|
|
1984
|
+
}
|
|
1985
|
+
|
|
1312
1986
|
/** Explicitly records evidence only after the full independent verification passed. */
|
|
1313
1987
|
function persistVerifiedKnownGood(config: RecoveryConfig, verified: VerifyResult, performance: RuntimePerformanceEvidence): ReleaseEvidence {
|
|
1314
1988
|
const authority = releaseAuthority(config);
|
|
@@ -1332,13 +2006,19 @@ function persistVerifiedKnownGood(config: RecoveryConfig, verified: VerifyResult
|
|
|
1332
2006
|
releaseAuthorityRevision: authority.revision,
|
|
1333
2007
|
releaseFencingTokenSha256: createHash('sha256').update(authority.fencingToken).digest('hex'),
|
|
1334
2008
|
attestedAt: new Date().toISOString(),
|
|
2009
|
+
recoveryBundle: createKnownGoodRecoveryBundle(config),
|
|
1335
2010
|
};
|
|
1336
2011
|
const store = knownGood(config);
|
|
2012
|
+
const previous = authority.previous ? releaseEvidence(config.controllerHome, authority.previous, authority) : undefined;
|
|
1337
2013
|
const releases = [
|
|
1338
2014
|
attested,
|
|
1339
|
-
...store.releases.filter((entry) => entry
|
|
1340
|
-
]
|
|
1341
|
-
writeJson(statePath(config), { schemaVersion:
|
|
2015
|
+
...(previous ? store.releases.filter((entry) => sameReleaseIdentity(entry, previous)) : []),
|
|
2016
|
+
];
|
|
2017
|
+
writeJson(statePath(config), { schemaVersion: 2, releases, updatedAt: new Date().toISOString() } satisfies KnownGoodStore);
|
|
2018
|
+
// State is durable before cleanup. A crash between these lines leaves an
|
|
2019
|
+
// orphaned bundle, which this same owner removes on the next attestation;
|
|
2020
|
+
// it never invalidates a published recovery point.
|
|
2021
|
+
pruneRetiredKnownGoodRecoveryBundles(config, releases);
|
|
1342
2022
|
audit(config, 'known_good_attested', {
|
|
1343
2023
|
revision: active.revision,
|
|
1344
2024
|
artifactIdentity: active.artifactIdentity,
|
|
@@ -1363,6 +2043,82 @@ function runtimePerformanceIdentity(config: RecoveryConfig): RuntimePerformanceI
|
|
|
1363
2043
|
};
|
|
1364
2044
|
}
|
|
1365
2045
|
|
|
2046
|
+
export const RECOVERY_INTERNAL_PERFORMANCE_COMMAND = '__measure-runtime-performance';
|
|
2047
|
+
|
|
2048
|
+
export async function measureConfiguredRuntimePerformance(
|
|
2049
|
+
config: RecoveryConfig,
|
|
2050
|
+
dependencies: RuntimePerformanceDependencies = {},
|
|
2051
|
+
): Promise<RuntimePerformanceEvidence> {
|
|
2052
|
+
return await measureRuntimePerformance(() => runtimePerformanceIdentity(config), dependencies);
|
|
2053
|
+
}
|
|
2054
|
+
|
|
2055
|
+
function hasRuntimePerformanceTestSeam(dependencies: RuntimePerformanceDependencies): boolean {
|
|
2056
|
+
return dependencies.readCpu !== undefined
|
|
2057
|
+
|| dependencies.monotonicNow !== undefined
|
|
2058
|
+
|| dependencies.wallNow !== undefined
|
|
2059
|
+
|| dependencies.sleep !== undefined;
|
|
2060
|
+
}
|
|
2061
|
+
|
|
2062
|
+
async function measureRuntimePerformanceIsolated(
|
|
2063
|
+
config: RecoveryConfig,
|
|
2064
|
+
expectedIdentity: RuntimePerformanceIdentity,
|
|
2065
|
+
): Promise<RuntimePerformanceEvidence> {
|
|
2066
|
+
const recoveryBefore = readCurrentRecoveryRelease(config.controllerHome);
|
|
2067
|
+
if (!recoveryBefore) throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: current immutable Recovery release unavailable');
|
|
2068
|
+
const executable = join(recoveryBefore.releasePath, 'forge-recovery');
|
|
2069
|
+
if (!existsSync(executable)) throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: immutable Recovery sampler executable unavailable');
|
|
2070
|
+
|
|
2071
|
+
const result = await runBoundedChild(
|
|
2072
|
+
executable,
|
|
2073
|
+
[RECOVERY_INTERNAL_PERFORMANCE_COMMAND, '--controller-home', config.controllerHome],
|
|
2074
|
+
{
|
|
2075
|
+
timeoutMs: 100_000,
|
|
2076
|
+
maxOutputBytes: 32 * 1024,
|
|
2077
|
+
forwardSignals: false,
|
|
2078
|
+
env: runtimeAuthorityFreeEnvironment(process.env),
|
|
2079
|
+
},
|
|
2080
|
+
);
|
|
2081
|
+
if (result.status !== 0 || result.failureCode || result.timedOut) {
|
|
2082
|
+
const detail = result.failureCode ?? result.error ?? (result.stderr.trim() || `exit=${result.status}`);
|
|
2083
|
+
throw new Error(`RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler failed (${detail.slice(0, 240)})`);
|
|
2084
|
+
}
|
|
2085
|
+
|
|
2086
|
+
let envelope: unknown;
|
|
2087
|
+
try {
|
|
2088
|
+
envelope = JSON.parse(result.stdout);
|
|
2089
|
+
} catch {
|
|
2090
|
+
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler returned malformed evidence');
|
|
2091
|
+
}
|
|
2092
|
+
if (!envelope || typeof envelope !== 'object' || Array.isArray(envelope)) {
|
|
2093
|
+
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler returned invalid evidence envelope');
|
|
2094
|
+
}
|
|
2095
|
+
const parsed = envelope as { schemaVersion?: unknown; ok?: unknown; evidence?: unknown; error?: unknown };
|
|
2096
|
+
if (parsed.schemaVersion !== 1 || typeof parsed.ok !== 'boolean') {
|
|
2097
|
+
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler returned invalid evidence envelope');
|
|
2098
|
+
}
|
|
2099
|
+
if (!parsed.ok) {
|
|
2100
|
+
const detail = typeof parsed.error === 'string' ? parsed.error : '';
|
|
2101
|
+
if (/^RECOVERY_PERFORMANCE_(?:UNKNOWN|REJECTED):/.test(detail)) throw new Error(detail);
|
|
2102
|
+
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler returned an unclassified failure');
|
|
2103
|
+
}
|
|
2104
|
+
if (!parsed.evidence || typeof parsed.evidence !== 'object' || Array.isArray(parsed.evidence)) {
|
|
2105
|
+
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: isolated sampler omitted performance evidence');
|
|
2106
|
+
}
|
|
2107
|
+
const performance = parsed.evidence as RuntimePerformanceEvidence;
|
|
2108
|
+
assertRuntimePerformanceEvidence(performance, expectedIdentity);
|
|
2109
|
+
|
|
2110
|
+
const recoveryAfter = readCurrentRecoveryRelease(config.controllerHome);
|
|
2111
|
+
if (!recoveryAfter
|
|
2112
|
+
|| recoveryAfter.releaseRevision !== recoveryBefore.releaseRevision
|
|
2113
|
+
|| recoveryAfter.manifestSha256 !== recoveryBefore.manifestSha256) {
|
|
2114
|
+
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: Recovery release changed during performance observation');
|
|
2115
|
+
}
|
|
2116
|
+
if (!samePerformanceIdentity(expectedIdentity, runtimePerformanceIdentity(config))) {
|
|
2117
|
+
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: Runtime or release authority changed after performance observation');
|
|
2118
|
+
}
|
|
2119
|
+
return performance;
|
|
2120
|
+
}
|
|
2121
|
+
|
|
1366
2122
|
export async function attestKnownGood(
|
|
1367
2123
|
config: RecoveryConfig,
|
|
1368
2124
|
dependencies: RuntimePerformanceDependencies = {},
|
|
@@ -1376,17 +2132,20 @@ export async function attestKnownGood(
|
|
|
1376
2132
|
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: Runtime or release authority changed during functional verification');
|
|
1377
2133
|
}
|
|
1378
2134
|
|
|
1379
|
-
//
|
|
1380
|
-
//
|
|
1381
|
-
//
|
|
1382
|
-
//
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
2135
|
+
// Production observation executes in one short-lived child of the exact
|
|
2136
|
+
// immutable Recovery release so Gateway/watchdog work in the persistent
|
|
2137
|
+
// daemon cannot stretch the sampler's monotonic windows. Explicit dependency
|
|
2138
|
+
// hooks remain the in-process test seam only. Both paths stay read-only and
|
|
2139
|
+
// outside the Recovery mutation lock; exact Runtime identity remains fenced.
|
|
2140
|
+
const performance = hasRuntimePerformanceTestSeam(dependencies)
|
|
2141
|
+
? await measureRuntimePerformance(() => {
|
|
2142
|
+
const current = runtimePerformanceIdentity(config);
|
|
2143
|
+
if (!samePerformanceIdentity(verifiedIdentity, current)) {
|
|
2144
|
+
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: Runtime or release authority changed after functional verification');
|
|
2145
|
+
}
|
|
2146
|
+
return current;
|
|
2147
|
+
}, dependencies)
|
|
2148
|
+
: await measureRuntimePerformanceIsolated(config, verifiedIdentity);
|
|
1390
2149
|
|
|
1391
2150
|
const locked = await withLock(config, { action: 'attest_known_good' }, async () => {
|
|
1392
2151
|
const verified = await verifyStableRuntime(config);
|
|
@@ -1442,7 +2201,8 @@ async function rollbackPreviousLocked(config: RecoveryConfig, reason: string): P
|
|
|
1442
2201
|
}
|
|
1443
2202
|
const operationId = `recovery-rollback-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
|
1444
2203
|
try {
|
|
1445
|
-
const
|
|
2204
|
+
const rollbackResult = rollbackRuntimeReleaseWithResult(config.controllerHome, operationId);
|
|
2205
|
+
const committed = rollbackResult.authority;
|
|
1446
2206
|
if (
|
|
1447
2207
|
committed.active.releaseId !== target.revision
|
|
1448
2208
|
|| committed.active.artifactIdentity !== target.artifactIdentity
|
|
@@ -1455,11 +2215,16 @@ async function rollbackPreviousLocked(config: RecoveryConfig, reason: string): P
|
|
|
1455
2215
|
activeRevision: active.revision,
|
|
1456
2216
|
restoredRevision: target.revision,
|
|
1457
2217
|
restoredManifestSha256: target.manifestSha256,
|
|
2218
|
+
databaseDisposition: rollbackResult.databaseDisposition,
|
|
2219
|
+
liveAuditEventCount: rollbackResult.liveAuditEventCount,
|
|
2220
|
+
rollbackAuditEventCount: rollbackResult.rollbackAuditEventCount,
|
|
1458
2221
|
});
|
|
1459
2222
|
return {
|
|
1460
2223
|
ok: true,
|
|
1461
2224
|
operationId,
|
|
1462
|
-
detail:
|
|
2225
|
+
detail: rollbackResult.databaseDisposition === 'restored_backup'
|
|
2226
|
+
? 'whole-Runtime release and unchanged SQLite backup restored; Canonical Runtime remains stopped until its sole launcher starts it'
|
|
2227
|
+
: 'whole-Runtime release restored while newer live SQLite state was preserved; Canonical Runtime remains stopped until its sole launcher starts it',
|
|
1463
2228
|
};
|
|
1464
2229
|
} catch (error) {
|
|
1465
2230
|
const detail = error instanceof Error ? error.message : 'whole-Runtime rollback failed';
|
|
@@ -1508,65 +2273,226 @@ async function verifyLocalRuntime(
|
|
|
1508
2273
|
}, createRecoveryHttpTransport(config.controllerHome), options);
|
|
1509
2274
|
}
|
|
1510
2275
|
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1514
|
-
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
export const WATCHDOG_RUNTIME_STARTUP_GRACE_MS = 60_000;
|
|
1521
|
-
export const WATCHDOG_RUNTIME_RESTART_BUDGET_STABLE_MS = 5 * 60_000;
|
|
1522
|
-
|
|
1523
|
-
export function watchdogRuntimeStartupGraceMs(
|
|
1524
|
-
config: Pick<RecoveryConfig, 'primaryRuntimeService'>,
|
|
1525
|
-
): number {
|
|
1526
|
-
const configuredVerifyTimeout = config.primaryRuntimeService?.postRestartVerifyTimeoutMs;
|
|
1527
|
-
return Math.max(
|
|
1528
|
-
WATCHDOG_RUNTIME_STARTUP_GRACE_MS,
|
|
1529
|
-
Number.isFinite(configuredVerifyTimeout) ? Math.max(0, configuredVerifyTimeout ?? 0) : 0,
|
|
1530
|
-
);
|
|
1531
|
-
}
|
|
1532
|
-
|
|
1533
|
-
export function runtimeWithinWatchdogStartupGrace(
|
|
1534
|
-
input: { running: boolean; stale: boolean; snapshot?: { startedAt?: string } },
|
|
1535
|
-
nowMs = Date.now(),
|
|
1536
|
-
graceMs = WATCHDOG_RUNTIME_STARTUP_GRACE_MS,
|
|
1537
|
-
): boolean {
|
|
1538
|
-
const startedAtMs = input.snapshot?.startedAt ? Date.parse(input.snapshot.startedAt) : Number.NaN;
|
|
1539
|
-
return Boolean(
|
|
1540
|
-
input.running
|
|
1541
|
-
&& !input.stale
|
|
1542
|
-
&& Number.isFinite(startedAtMs)
|
|
1543
|
-
&& nowMs >= startedAtMs
|
|
1544
|
-
&& nowMs - startedAtMs < Math.max(0, graceMs),
|
|
1545
|
-
);
|
|
1546
|
-
}
|
|
1547
|
-
|
|
1548
|
-
export function runtimeRestartBudgetIdentity(release: Pick<ReleaseEvidence, 'revision' | 'artifactIdentity' | 'manifestSha256'> | undefined): string | undefined {
|
|
1549
|
-
if (!release?.revision || !release.artifactIdentity || !release.manifestSha256) return undefined;
|
|
1550
|
-
return `${release.revision}:${release.artifactIdentity}:${release.manifestSha256}`;
|
|
2276
|
+
/**
|
|
2277
|
+
* Release cutover must not create fresh sessions on the Connector whose quiet
|
|
2278
|
+
* window fences the activation. The Runtime/Recovery surfaces are sufficient
|
|
2279
|
+
* to verify the frozen Stable A and Candidate B identities; probing the
|
|
2280
|
+
* primary Connector here would refresh its activity timestamp and make the
|
|
2281
|
+
* cutover gate self-starve on every automatic retry.
|
|
2282
|
+
*/
|
|
2283
|
+
async function verifyLocalRuntimeForReleaseCutover(config: RecoveryConfig): Promise<VerifyResult> {
|
|
2284
|
+
return verifyLocalRuntime({ ...config, primaryConnectorService: undefined }, { probeMcpProtocol: false });
|
|
1551
2285
|
}
|
|
1552
2286
|
|
|
1553
2287
|
/**
|
|
1554
|
-
*
|
|
1555
|
-
*
|
|
1556
|
-
*
|
|
1557
|
-
*
|
|
2288
|
+
* Bounded observation owns health, not release verification. Keep this path
|
|
2289
|
+
* deliberately bounded to already-published Runtime authority/status plus local
|
|
2290
|
+
* HTTP transport checks needed for prompt targeted repair. Expensive execution
|
|
2291
|
+
* canaries, known-good bundle inspection, external transport probes and MCP
|
|
2292
|
+
* initialize/list/call stay in verifyStableRuntime/verifyLocalRuntime and run
|
|
2293
|
+
* only on the periodic verification deadline or an explicit strict boundary.
|
|
1558
2294
|
*/
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
2295
|
+
async function observeBoundedRuntimeHealth(
|
|
2296
|
+
config: RecoveryConfig,
|
|
2297
|
+
transport: RecoveryHttpTransport,
|
|
2298
|
+
options: {
|
|
2299
|
+
/**
|
|
2300
|
+
* Primary Connector probing is excluded from repair surfaces that must keep
|
|
2301
|
+
* working while the primary transport is the broken thing.
|
|
2302
|
+
*/
|
|
2303
|
+
includePrimaryConnectorLocal?: boolean;
|
|
2304
|
+
/**
|
|
2305
|
+
* Public/Connector transport probes are lightweight health evidence but
|
|
2306
|
+
* create Connector activity. Disable them while a ReleaseSession is waiting
|
|
2307
|
+
* for the cutover quiet window so watchdog observation cannot self-starve it.
|
|
2308
|
+
*/
|
|
2309
|
+
includePrimaryTransport?: boolean;
|
|
2310
|
+
} = {},
|
|
2311
|
+
): Promise<VerifyResult> {
|
|
2312
|
+
const observation = observeRuntimeStatus(config.controllerHome);
|
|
2313
|
+
const authority = releaseAuthority(config);
|
|
2314
|
+
const active = activeAuthorityRelease(config);
|
|
2315
|
+
const previous = previousAuthorityRelease(config);
|
|
2316
|
+
const runtimeHealthy = observation.running && observation.ready && !observation.stale;
|
|
2317
|
+
const coherent = Boolean(
|
|
2318
|
+
authority
|
|
2319
|
+
&& active
|
|
2320
|
+
&& observation.snapshot?.releaseId === active.revision
|
|
2321
|
+
&& observation.snapshot?.artifactIdentity === active.artifactIdentity
|
|
2322
|
+
&& authority.active.workerProtocolVersion === active.workerProtocolVersion
|
|
2323
|
+
);
|
|
2324
|
+
const probes: VerifyResult['probes'] = {
|
|
2325
|
+
runtime_status: {
|
|
2326
|
+
ok: observation.running && !observation.stale,
|
|
2327
|
+
detail: observation.running
|
|
2328
|
+
? observation.stale ? 'canonical Runtime status is stale' : 'canonical Runtime owner is live'
|
|
2329
|
+
: 'canonical Runtime is not running',
|
|
2330
|
+
},
|
|
2331
|
+
};
|
|
2332
|
+
const endpoint = observation.snapshot?.endpoint;
|
|
2333
|
+
probes.active_gateway = endpoint
|
|
2334
|
+
? await probe(transport, runtimeHealthEndpoint(endpoint))
|
|
2335
|
+
: { ok: false, detail: 'canonical Runtime endpoint is unavailable' };
|
|
2336
|
+
if (options.includePrimaryTransport !== false && config.publicMcpUrl) {
|
|
2337
|
+
probes.external_mcp_http = await probeExternalMcp(transport, config.publicMcpUrl);
|
|
2338
|
+
const connectorReadinessEndpoint = config.primaryConnectorService?.localMcpUrl?.trim() || config.publicMcpUrl;
|
|
2339
|
+
probes.primary_connector_ready = await probePublicGatewayReadiness(transport, connectorReadinessEndpoint, {
|
|
2340
|
+
acceptOAuthChallenge: packageConnectorAuthMode(config.controllerHome) === 'oauth',
|
|
2341
|
+
});
|
|
2342
|
+
}
|
|
2343
|
+
if (options.includePrimaryTransport !== false) {
|
|
2344
|
+
const primaryTunnel = configuredPrimaryPublicTunnel(config);
|
|
2345
|
+
if (primaryTunnel?.platform === 'openai-secure-tunnel') {
|
|
2346
|
+
const observed = await observeOpenAiTunnelRuntime(primaryTunnel);
|
|
2347
|
+
probes.primary_tunnel_runtime = { ok: observed.ok, detail: observed.detail, value: observed };
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
if (options.includePrimaryConnectorLocal !== false) {
|
|
2351
|
+
const primaryConnectorLocal = await probePrimaryConnectorLocal(config, transport);
|
|
2352
|
+
if (primaryConnectorLocal) probes.primary_connector_local = primaryConnectorLocal;
|
|
2353
|
+
}
|
|
2354
|
+
if (config.gateway) probes.recovery_gateway = await probe(transport, `http://${config.gateway.host}:${config.gateway.port}/health`);
|
|
2355
|
+
const watchdogHealth = observeRecoveryWatchdogHealth(
|
|
2356
|
+
config.controllerHome,
|
|
2357
|
+
config.installProfile === 'self-healing',
|
|
2358
|
+
);
|
|
2359
|
+
probes.recovery_watchdog = {
|
|
2360
|
+
ok: watchdogHealth.ok,
|
|
2361
|
+
detail: watchdogHealth.detail,
|
|
2362
|
+
value: {
|
|
2363
|
+
pulseAgeMs: watchdogHealth.pulseAgeMs,
|
|
2364
|
+
tickAgeMs: watchdogHealth.tickAgeMs,
|
|
2365
|
+
currentReleaseRevision: watchdogHealth.currentReleaseRevision,
|
|
2366
|
+
watchdogReleaseRevision: watchdogHealth.runtimeIdentity?.releaseRevision,
|
|
2367
|
+
watchdogPid: watchdogHealth.runtimeIdentity?.pid,
|
|
2368
|
+
},
|
|
2369
|
+
};
|
|
2370
|
+
// The dedicated Recovery transport is the only channel that can repair a
|
|
2371
|
+
// broken primary transport. It must be observed on every liveness tick, or a
|
|
2372
|
+
// dead Recovery tunnel stays invisible while the watchdog keeps reporting a
|
|
2373
|
+
// healthy system and never reaches the bounded tunnel-repair path.
|
|
2374
|
+
const recoveryTransport = await probeOpenAiRecoveryTunnel(config);
|
|
2375
|
+
if (recoveryTransport) probes.recovery_tunnel_runtime = recoveryTransport;
|
|
2376
|
+
const localChecks = Object.entries(probes)
|
|
2377
|
+
.filter(([name]) => !name.startsWith('recovery_'))
|
|
2378
|
+
.every(([, entry]) => entry.ok);
|
|
2379
|
+
return {
|
|
2380
|
+
ok: Boolean(runtimeHealthy && coherent && localChecks),
|
|
2381
|
+
at: new Date().toISOString(),
|
|
2382
|
+
runtime: {
|
|
2383
|
+
ok: runtimeHealthy,
|
|
2384
|
+
running: observation.running,
|
|
2385
|
+
ready: observation.ready,
|
|
2386
|
+
stale: observation.stale,
|
|
2387
|
+
reasonCodes: [...observation.reasonCodes],
|
|
2388
|
+
},
|
|
2389
|
+
releases: {
|
|
2390
|
+
active,
|
|
2391
|
+
previous,
|
|
2392
|
+
// This tier reports durable attestation identity only. Physical bundle
|
|
2393
|
+
// integrity is deliberately reserved for strict verification boundaries.
|
|
2394
|
+
knownGood: matchingKnownGoodAttestation(config, active),
|
|
2395
|
+
coherent,
|
|
2396
|
+
},
|
|
2397
|
+
probes,
|
|
2398
|
+
};
|
|
2399
|
+
}
|
|
2400
|
+
|
|
2401
|
+
async function observeWatchdogHealthTier(
|
|
2402
|
+
config: RecoveryConfig,
|
|
2403
|
+
transport = createRecoveryHttpTransport(config.controllerHome),
|
|
2404
|
+
): Promise<VerifyResult> {
|
|
2405
|
+
// The primary Connector quiet window is a cutover fence. Once the same
|
|
2406
|
+
// ReleaseSession is eligible for cutover, Recovery's five-second health
|
|
2407
|
+
// probe must stop opening Connector MCP sessions or it will continuously
|
|
2408
|
+
// refresh the very activity timestamp that the fence is waiting to drain.
|
|
2409
|
+
// An incomplete inventory fails closed by retaining the normal probe.
|
|
2410
|
+
const inventory = listCompleteReleaseSessions(config.controllerHome);
|
|
2411
|
+
const cutoverQuietWindowRequired = inventory.invalidSessionFiles.length === 0
|
|
2412
|
+
&& inventory.sessions.some((session) => (
|
|
2413
|
+
session.phase === 'cutover_eligible'
|
|
2414
|
+
|| session.phase === 'cutover_attempting'
|
|
2415
|
+
|| session.phase === 'cutover_committed'
|
|
2416
|
+
));
|
|
2417
|
+
const primaryTransportAttributionRequired = Boolean(
|
|
2418
|
+
config.primaryConnectorService || configuredPrimaryPublicTunnel(config),
|
|
2419
|
+
);
|
|
2420
|
+
return observeBoundedRuntimeHealth(config, transport, {
|
|
2421
|
+
includePrimaryConnectorLocal: !cutoverQuietWindowRequired,
|
|
2422
|
+
// A plain public URL is strict-verification evidence, not a five-second
|
|
2423
|
+
// liveness dependency. Probe primary transport on the cheap tier only when
|
|
2424
|
+
// Recovery has a managed Connector/tunnel whose failure attribution can
|
|
2425
|
+
// drive a bounded repair decision.
|
|
2426
|
+
includePrimaryTransport: !cutoverQuietWindowRequired && primaryTransportAttributionRequired,
|
|
2427
|
+
});
|
|
2428
|
+
}
|
|
2429
|
+
|
|
2430
|
+
function isExternalTunnelFailure(config: RecoveryConfig, verified: VerifyResult, localVerify: VerifyResult): boolean {
|
|
2431
|
+
const configured = configuredRecoveryTunnel(config);
|
|
2432
|
+
const localRecoveryGatewayHealthy = localVerify.probes.recovery_gateway?.ok ?? true;
|
|
2433
|
+
if (!configured || !localVerify.ok || !localRecoveryGatewayHealthy) return false;
|
|
2434
|
+
if (configured.platform === 'openai-secure-tunnel') {
|
|
2435
|
+
const observed = openAiTunnelObservationFromProbe(verified.probes.recovery_tunnel_runtime);
|
|
2436
|
+
// A locally healthy tunnel with degraded/unknown OpenAI control-plane reachability
|
|
2437
|
+
// is an upstream transport incident, not evidence that reconnecting the local
|
|
2438
|
+
// tunnel process is a valid repair. Keep it degraded and collect evidence.
|
|
2439
|
+
if (isUpstreamOpenAiControlPlaneIncident(observed)) return false;
|
|
2440
|
+
return verified.probes.recovery_tunnel_runtime?.ok !== true;
|
|
2441
|
+
}
|
|
2442
|
+
const external = verified.probes.recovery_external_http ?? verified.probes.external_mcp_http;
|
|
2443
|
+
return Boolean(configuredRecoveryPublicUrl(config) && external?.ok !== true);
|
|
2444
|
+
}
|
|
2445
|
+
|
|
2446
|
+
export const WATCHDOG_RUNTIME_STARTUP_GRACE_MS = 60_000;
|
|
2447
|
+
export const WATCHDOG_RUNTIME_RESTART_BUDGET_STABLE_MS = 5 * 60_000;
|
|
2448
|
+
|
|
2449
|
+
export function watchdogRuntimeStartupGraceMs(
|
|
2450
|
+
config: Pick<RecoveryConfig, 'primaryRuntimeService'>,
|
|
2451
|
+
): number {
|
|
2452
|
+
const configuredVerifyTimeout = config.primaryRuntimeService?.postRestartVerifyTimeoutMs;
|
|
2453
|
+
return Math.max(
|
|
2454
|
+
WATCHDOG_RUNTIME_STARTUP_GRACE_MS,
|
|
2455
|
+
Number.isFinite(configuredVerifyTimeout) ? Math.max(0, configuredVerifyTimeout ?? 0) : 0,
|
|
2456
|
+
);
|
|
2457
|
+
}
|
|
2458
|
+
|
|
2459
|
+
export function runtimeWithinWatchdogStartupGrace(
|
|
2460
|
+
input: { running: boolean; stale: boolean; snapshot?: { startedAt?: string } },
|
|
2461
|
+
nowMs = Date.now(),
|
|
2462
|
+
graceMs = WATCHDOG_RUNTIME_STARTUP_GRACE_MS,
|
|
2463
|
+
): boolean {
|
|
2464
|
+
const startedAtMs = input.snapshot?.startedAt ? Date.parse(input.snapshot.startedAt) : Number.NaN;
|
|
2465
|
+
return Boolean(
|
|
2466
|
+
input.running
|
|
2467
|
+
&& !input.stale
|
|
2468
|
+
&& Number.isFinite(startedAtMs)
|
|
2469
|
+
&& nowMs >= startedAtMs
|
|
2470
|
+
&& nowMs - startedAtMs < Math.max(0, graceMs),
|
|
2471
|
+
);
|
|
2472
|
+
}
|
|
2473
|
+
|
|
2474
|
+
export function runtimeRestartBudgetIdentity(release: Pick<ReleaseEvidence, 'revision' | 'artifactIdentity' | 'manifestSha256'> | undefined): string | undefined {
|
|
2475
|
+
if (!release?.revision || !release.artifactIdentity || !release.manifestSha256) return undefined;
|
|
2476
|
+
return `${release.revision}:${release.artifactIdentity}:${release.manifestSha256}`;
|
|
2477
|
+
}
|
|
2478
|
+
|
|
2479
|
+
/**
|
|
2480
|
+
* Bind primary Runtime restart accounting to the exact immutable Runtime release.
|
|
2481
|
+
* A missing identity is a legacy-state migration: retain its counters and bind
|
|
2482
|
+
* them to the currently observed release so a watchdog upgrade cannot mint a
|
|
2483
|
+
* fresh restart budget for an already failing Runtime.
|
|
2484
|
+
*/
|
|
2485
|
+
export function scopeWatchdogStateToRuntimeRelease(
|
|
2486
|
+
state: WatchdogState,
|
|
2487
|
+
release: Pick<ReleaseEvidence, 'revision' | 'artifactIdentity' | 'manifestSha256'> | undefined,
|
|
2488
|
+
): WatchdogState {
|
|
2489
|
+
const identity = runtimeRestartBudgetIdentity(release);
|
|
2490
|
+
if (!identity || state.runtimeRestartBudgetIdentity === identity) return state;
|
|
2491
|
+
if (!state.runtimeRestartBudgetIdentity) return { ...state, runtimeRestartBudgetIdentity: identity };
|
|
2492
|
+
return {
|
|
2493
|
+
...state,
|
|
2494
|
+
failures: 0,
|
|
2495
|
+
firstFailureAt: undefined,
|
|
1570
2496
|
rollbackUsed: false,
|
|
1571
2497
|
runtimeRestartAttempts: 0,
|
|
1572
2498
|
runtimeRestartFailures: 0,
|
|
@@ -1723,22 +2649,26 @@ export function decideWatchdog(input: {
|
|
|
1723
2649
|
|
|
1724
2650
|
export async function diagnose(config: RecoveryConfig): Promise<Record<string, unknown>> {
|
|
1725
2651
|
const verified = await verifyStableRuntime(config);
|
|
1726
|
-
return { verified, knownGood:
|
|
2652
|
+
return { verified, knownGood: inspectKnownGoodRecoverability(config), quarantine: json(quarantinePath(config)) ?? { releases: [] } };
|
|
1727
2653
|
}
|
|
1728
2654
|
|
|
1729
2655
|
export async function listReleases(config: RecoveryConfig): Promise<Record<string, unknown>> {
|
|
1730
2656
|
const observation = observeRuntimeStatus(config.controllerHome);
|
|
2657
|
+
const knownGoodRecoverability = inspectKnownGoodRecoverability(config);
|
|
1731
2658
|
return {
|
|
1732
2659
|
runtimeRunning: observation.running,
|
|
1733
2660
|
runtimeReady: observation.ready,
|
|
1734
2661
|
active: activeAuthorityRelease(config),
|
|
1735
2662
|
previous: previousAuthorityRelease(config),
|
|
1736
|
-
knownGood:
|
|
2663
|
+
knownGood: knownGoodRecoverability.available,
|
|
2664
|
+
knownGoodUnavailable: knownGoodRecoverability.unavailable,
|
|
2665
|
+
pinned: runtimePin(config)?.release,
|
|
1737
2666
|
};
|
|
1738
2667
|
}
|
|
1739
2668
|
|
|
1740
2669
|
interface CommandResult { ok: boolean; status: number | null; stdout: string; stderr: string; }
|
|
1741
|
-
|
|
2670
|
+
interface CommandOptions { cwd?: string; home?: string; maxOutputBytes?: number; }
|
|
2671
|
+
type CommandRunner = (commandName: string, args: string[], timeoutMs?: number, options?: CommandOptions) => Promise<CommandResult>;
|
|
1742
2672
|
interface LaunchdService { uid: number; domain: string; target: string; label: string; plistPath: string; }
|
|
1743
2673
|
|
|
1744
2674
|
/**
|
|
@@ -1758,6 +2688,28 @@ export function recoveryCommandPath(
|
|
|
1758
2688
|
return entries.includes(userBin) ? inheritedPath : [userBin, ...entries].join(delimiter);
|
|
1759
2689
|
}
|
|
1760
2690
|
|
|
2691
|
+
/**
|
|
2692
|
+
* Child environment for Recovery-owned commands.
|
|
2693
|
+
*
|
|
2694
|
+
* Keep the hosting Runtime's private FORGE_* authority out, but preserve the
|
|
2695
|
+
* account context: user-installed tools resolve their own per-account state from
|
|
2696
|
+
* HOME, and a persistent launchd/systemd service starts without an interactive
|
|
2697
|
+
* HOME to inherit. Without it, `tunnel-client` cannot see its own alias registry,
|
|
2698
|
+
* so Recovery both misreads a healthy dedicated tunnel as stopped and cannot
|
|
2699
|
+
* confirm the tunnel it just reconnected.
|
|
2700
|
+
*/
|
|
2701
|
+
export function recoveryCommandEnvironment(
|
|
2702
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
2703
|
+
accountHome = homedir(),
|
|
2704
|
+
): NodeJS.ProcessEnv {
|
|
2705
|
+
const home = env.HOME?.trim() || accountHome;
|
|
2706
|
+
return {
|
|
2707
|
+
...runtimeAuthorityFreeEnvironment(env),
|
|
2708
|
+
PATH: recoveryCommandPath(env.PATH ?? '', process.platform, home),
|
|
2709
|
+
HOME: home,
|
|
2710
|
+
};
|
|
2711
|
+
}
|
|
2712
|
+
|
|
1761
2713
|
export interface PublicTunnelRepairDependencies {
|
|
1762
2714
|
platform?: NodeJS.Platform;
|
|
1763
2715
|
currentUid?: () => Promise<number | undefined>;
|
|
@@ -1768,14 +2720,14 @@ export interface PublicTunnelRepairDependencies {
|
|
|
1768
2720
|
sleep?: (ms: number) => Promise<void>;
|
|
1769
2721
|
}
|
|
1770
2722
|
|
|
1771
|
-
function command(commandName: string, args: string[], timeoutMs = 10_000, options:
|
|
2723
|
+
function command(commandName: string, args: string[], timeoutMs = 10_000, options: CommandOptions = {}): Promise<CommandResult> {
|
|
1772
2724
|
return new Promise((resolveCommand) => {
|
|
1773
2725
|
const child = spawn(commandName, args, {
|
|
1774
2726
|
shell: false,
|
|
1775
2727
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
1776
2728
|
windowsHide: true,
|
|
1777
2729
|
cwd: options.cwd,
|
|
1778
|
-
env:
|
|
2730
|
+
env: recoveryCommandEnvironment(process.env, options.home),
|
|
1779
2731
|
});
|
|
1780
2732
|
const stdout: Buffer[] = [];
|
|
1781
2733
|
const stderr: Buffer[] = [];
|
|
@@ -1853,10 +2805,14 @@ async function ensureLaunchdServiceStarted(service: LaunchdService, runCommand:
|
|
|
1853
2805
|
}
|
|
1854
2806
|
|
|
1855
2807
|
export interface PrimaryRuntimeRecoveryDependencies {
|
|
2808
|
+
/** Explicit operator restart may recycle an already-healthy Runtime; watchdog/internal recovery keeps healthy no-op semantics. */
|
|
2809
|
+
forceHealthyRestart?: boolean;
|
|
1856
2810
|
platform?: NodeJS.Platform;
|
|
1857
2811
|
currentUid?: () => Promise<number | undefined>;
|
|
1858
2812
|
runCommand?: CommandRunner;
|
|
1859
2813
|
verifyLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
|
|
2814
|
+
/** Cheap readiness observation used while a restarted Runtime is converging; strict verification remains a separate acceptance boundary. */
|
|
2815
|
+
observeLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
|
|
1860
2816
|
/** Rebind the independently supervised OAuth Connector after a whole-Runtime release switch. */
|
|
1861
2817
|
repairPrimaryConnectorBinding?: (config: RecoveryConfig) => Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string }>;
|
|
1862
2818
|
runtimeRunning?: (config: RecoveryConfig) => boolean;
|
|
@@ -1865,13 +2821,31 @@ export interface PrimaryRuntimeRecoveryDependencies {
|
|
|
1865
2821
|
sleep?: (ms: number) => Promise<void>;
|
|
1866
2822
|
}
|
|
1867
2823
|
|
|
2824
|
+
export interface RuntimeReleaseKnownGoodDependencies extends RuntimePerformanceDependencies {
|
|
2825
|
+
/** Test/host seam for the existing whole-Runtime rollback transaction. */
|
|
2826
|
+
rollback?: PrimaryRuntimeRecoveryDependencies;
|
|
2827
|
+
/** Exact active-release live continuation proof; production defaults to the Supervisor single-writer ledger. */
|
|
2828
|
+
continuationProof?: (
|
|
2829
|
+
forgeHome: string,
|
|
2830
|
+
input: { repoId?: string; activeReleaseId: string; notBefore: string },
|
|
2831
|
+
) => Promise<WorkflowSupervisorContinuationProof | undefined>;
|
|
2832
|
+
}
|
|
2833
|
+
|
|
1868
2834
|
export interface RuntimeReleaseActivationGuard {
|
|
1869
2835
|
/** External caller identity for audit/lock attribution. */
|
|
1870
2836
|
requestId?: string;
|
|
2837
|
+
/** Internal Recovery-only mode: allow explicit activation of current.previous when it is durably pinned. */
|
|
2838
|
+
allowPreviousRelease?: boolean;
|
|
2839
|
+
/** Internal Recovery-only mode: restore Runtime authority without restoring an older SQLite backup on activation failure. */
|
|
2840
|
+
preserveDatabaseOnFailure?: boolean;
|
|
2841
|
+
/** Internal Recovery-only pin fence checked after acquiring the mutation lock. */
|
|
2842
|
+
requiredPinnedReleaseRevision?: string;
|
|
1871
2843
|
/** Authority snapshot observed when the caller decided to activate; null means the caller observed no authority. */
|
|
1872
2844
|
expectedAuthorityRevision?: number | null;
|
|
1873
2845
|
/** Active release observed when the caller decided to activate; null means the caller observed no active release. */
|
|
1874
2846
|
expectedActiveReleaseId?: string | null;
|
|
2847
|
+
/** Fresh portable source candidates may activate only after this exact Recovery ReleaseSession becomes cutover-eligible. */
|
|
2848
|
+
releaseSessionId?: string;
|
|
1875
2849
|
}
|
|
1876
2850
|
|
|
1877
2851
|
function configuredPrimaryRuntimeService(config: RecoveryConfig, platform: NodeJS.Platform = process.platform): PrimaryRuntimeServiceConfig {
|
|
@@ -2067,6 +3041,103 @@ async function probePrimaryConnectorLocal(
|
|
|
2067
3041
|
});
|
|
2068
3042
|
}
|
|
2069
3043
|
|
|
3044
|
+
export interface AutomaticReleaseCutoverReadiness {
|
|
3045
|
+
ready: boolean;
|
|
3046
|
+
activeSessions?: number;
|
|
3047
|
+
activePosts?: number;
|
|
3048
|
+
activeStreams?: number;
|
|
3049
|
+
initializing?: number;
|
|
3050
|
+
latestActivityAgeMs?: number;
|
|
3051
|
+
detail: string;
|
|
3052
|
+
}
|
|
3053
|
+
|
|
3054
|
+
const AUTOMATIC_RELEASE_CUTOVER_QUIET_MS = 30_000;
|
|
3055
|
+
const AUTOMATIC_RELEASE_CUTOVER_HEALTH_TIMEOUT_MS = 2_000;
|
|
3056
|
+
|
|
3057
|
+
function primaryConnectorHealthEndpoint(config: RecoveryConfig): string | undefined {
|
|
3058
|
+
const localMcpUrl = config.primaryConnectorService?.localMcpUrl?.trim();
|
|
3059
|
+
if (!localMcpUrl) return undefined;
|
|
3060
|
+
try {
|
|
3061
|
+
const url = new URL(localMcpUrl);
|
|
3062
|
+
url.pathname = '/health';
|
|
3063
|
+
url.search = '';
|
|
3064
|
+
url.hash = '';
|
|
3065
|
+
return url.toString();
|
|
3066
|
+
} catch {
|
|
3067
|
+
return undefined;
|
|
3068
|
+
}
|
|
3069
|
+
}
|
|
3070
|
+
|
|
3071
|
+
export async function probeReleaseCutoverReadiness(
|
|
3072
|
+
config: RecoveryConfig,
|
|
3073
|
+
options: { transport?: RecoveryHttpTransport; quietMs?: number; timeoutMs?: number } = {},
|
|
3074
|
+
): Promise<AutomaticReleaseCutoverReadiness> {
|
|
3075
|
+
const healthUrl = primaryConnectorHealthEndpoint(config);
|
|
3076
|
+
if (!healthUrl) return { ready: false, detail: 'primary Connector health endpoint is not configured' };
|
|
3077
|
+
const quietMs = options.quietMs ?? AUTOMATIC_RELEASE_CUTOVER_QUIET_MS;
|
|
3078
|
+
const timeoutMs = options.timeoutMs ?? AUTOMATIC_RELEASE_CUTOVER_HEALTH_TIMEOUT_MS;
|
|
3079
|
+
const transport = options.transport ?? createRecoveryHttpTransport(config.controllerHome);
|
|
3080
|
+
const controller = new AbortController();
|
|
3081
|
+
const timer = setTimeout(() => controller.abort('RECOVERY_AUTOMATIC_RELEASE_CUTOVER_HEALTH_TIMEOUT'), timeoutMs);
|
|
3082
|
+
try {
|
|
3083
|
+
const response = await transport.request({
|
|
3084
|
+
url: healthUrl,
|
|
3085
|
+
headers: { accept: 'application/json' },
|
|
3086
|
+
timeoutMs,
|
|
3087
|
+
signal: controller.signal,
|
|
3088
|
+
});
|
|
3089
|
+
if (!response.ok) return { ready: false, detail: `primary Connector health returned HTTP ${response.status}` };
|
|
3090
|
+
let payload: { sessions?: { active?: unknown; activePosts?: unknown; activeStreams?: unknown; initializing?: unknown; latestActivityAgeMs?: unknown } };
|
|
3091
|
+
try {
|
|
3092
|
+
payload = JSON.parse(response.body) as typeof payload;
|
|
3093
|
+
} catch {
|
|
3094
|
+
return { ready: false, detail: 'primary Connector health returned invalid JSON' };
|
|
3095
|
+
}
|
|
3096
|
+
const activeSessions = Number(payload.sessions?.active);
|
|
3097
|
+
const activePosts = Number(payload.sessions?.activePosts);
|
|
3098
|
+
const activeStreams = Number(payload.sessions?.activeStreams);
|
|
3099
|
+
const initializing = Number(payload.sessions?.initializing);
|
|
3100
|
+
const latestActivityAgeMs = Number(payload.sessions?.latestActivityAgeMs);
|
|
3101
|
+
if (
|
|
3102
|
+
!Number.isFinite(activeSessions)
|
|
3103
|
+
|| !Number.isFinite(activePosts)
|
|
3104
|
+
|| !Number.isFinite(activeStreams)
|
|
3105
|
+
|| !Number.isFinite(initializing)
|
|
3106
|
+
) {
|
|
3107
|
+
return { ready: false, detail: 'primary Connector health omitted bounded session activity counters' };
|
|
3108
|
+
}
|
|
3109
|
+
const evidence = {
|
|
3110
|
+
activeSessions,
|
|
3111
|
+
activePosts,
|
|
3112
|
+
...(Number.isFinite(activeStreams) ? { activeStreams } : {}),
|
|
3113
|
+
initializing,
|
|
3114
|
+
...(Number.isFinite(latestActivityAgeMs) ? { latestActivityAgeMs } : {}),
|
|
3115
|
+
};
|
|
3116
|
+
if (activePosts > 0 || activeStreams > 0 || initializing > 0) {
|
|
3117
|
+
return {
|
|
3118
|
+
ready: false,
|
|
3119
|
+
...evidence,
|
|
3120
|
+
detail: `primary Connector is busy: activePosts=${activePosts}; activeStreams=${activeStreams}; initializing=${initializing}`,
|
|
3121
|
+
};
|
|
3122
|
+
}
|
|
3123
|
+
if (activeSessions > 0 && (!Number.isFinite(latestActivityAgeMs) || latestActivityAgeMs < quietMs)) {
|
|
3124
|
+
const remainingMs = Number.isFinite(latestActivityAgeMs) ? Math.max(0, quietMs - latestActivityAgeMs) : quietMs;
|
|
3125
|
+
return { ready: false, ...evidence, detail: `waiting for ${remainingMs}ms more MCP quiet time before Runtime cutover` };
|
|
3126
|
+
}
|
|
3127
|
+
return {
|
|
3128
|
+
ready: true,
|
|
3129
|
+
...evidence,
|
|
3130
|
+
detail: activeSessions === 0
|
|
3131
|
+
? 'primary Connector has no live MCP sessions'
|
|
3132
|
+
: `primary Connector has been MCP-idle for ${latestActivityAgeMs}ms`,
|
|
3133
|
+
};
|
|
3134
|
+
} catch (error) {
|
|
3135
|
+
return { ready: false, detail: `primary Connector health is unavailable: ${error instanceof Error ? error.message : String(error)}` };
|
|
3136
|
+
} finally {
|
|
3137
|
+
clearTimeout(timer);
|
|
3138
|
+
}
|
|
3139
|
+
}
|
|
3140
|
+
|
|
2070
3141
|
function activePackageConnectorReleaseBinding(config: RecoveryConfig): PackageConnectorReleaseBinding | undefined {
|
|
2071
3142
|
const configured = config.primaryConnectorService;
|
|
2072
3143
|
if (!configured || !configured.localMcpUrl?.trim()) return undefined;
|
|
@@ -2082,7 +3153,15 @@ function activePackageConnectorReleaseBinding(config: RecoveryConfig): PackageCo
|
|
|
2082
3153
|
return undefined;
|
|
2083
3154
|
}
|
|
2084
3155
|
if (!existsSync(join(packageRoot, 'src', 'cli', 'index.ts')) || !existsSync(join(packageRoot, 'src', 'runtime', 'shared', 'node-ts-loader.mjs'))) return undefined;
|
|
2085
|
-
|
|
3156
|
+
const manifest = loadRuntimeReleaseManifest(authority.active.manifestPath, config.controllerHome);
|
|
3157
|
+
return {
|
|
3158
|
+
releaseId: authority.active.releaseId,
|
|
3159
|
+
releaseRoot,
|
|
3160
|
+
packageRoot,
|
|
3161
|
+
...(manifest.connectorEntrypoint === 'forge-mcp-gateway' && manifest.connectorArtifactIdentity
|
|
3162
|
+
? { connectorEntrypoint: manifest.connectorEntrypoint, connectorArtifactIdentity: manifest.connectorArtifactIdentity }
|
|
3163
|
+
: {}),
|
|
3164
|
+
};
|
|
2086
3165
|
}
|
|
2087
3166
|
|
|
2088
3167
|
export function resolveRecoveryPackageConnectorExecutable(
|
|
@@ -2263,7 +3342,7 @@ export async function restartPrimaryConnector(
|
|
|
2263
3342
|
tunnelRestarted = { ok: result.ok, attempted: true, detail: result.detail, serviceTarget: tunnel.target };
|
|
2264
3343
|
}
|
|
2265
3344
|
} else {
|
|
2266
|
-
tunnelRestarted = await ensureOpenAiTunnelRuntimeStarted(primaryTunnel, runCommand);
|
|
3345
|
+
tunnelRestarted = await ensureOpenAiTunnelRuntimeStarted(primaryTunnel, runCommand, tunnelClientWorkingDirectory(config.controllerHome));
|
|
2267
3346
|
}
|
|
2268
3347
|
if (!tunnelRestarted?.ok) {
|
|
2269
3348
|
const detail = tunnelRestarted?.detail ?? 'primary public tunnel service configuration is invalid or unavailable';
|
|
@@ -2509,19 +3588,9 @@ async function stopPrimaryRuntimeForReleaseTransition(input: {
|
|
|
2509
3588
|
runCommand: CommandRunner;
|
|
2510
3589
|
runtimeRunning: (config: RecoveryConfig) => boolean;
|
|
2511
3590
|
}): Promise<PrimaryRuntimeStoppedTransition> {
|
|
3591
|
+
const ownerBeforeStop: RuntimeOwnerRecord | null = readRuntimeOwner(input.config.controllerHome) ?? null;
|
|
2512
3592
|
const stopped = await stopPrimaryRuntimeServiceOwner(input.service, input.runCommand);
|
|
2513
3593
|
if (!stopped.ok) return { ok: false, detail: stopped.detail };
|
|
2514
|
-
const runtimeStopped = await waitForPrimaryRuntimeState({
|
|
2515
|
-
config: input.config,
|
|
2516
|
-
expectedRunning: false,
|
|
2517
|
-
timeoutMs: 20_000,
|
|
2518
|
-
now: input.now,
|
|
2519
|
-
wait: input.wait,
|
|
2520
|
-
runtimeRunning: input.runtimeRunning,
|
|
2521
|
-
});
|
|
2522
|
-
if (!runtimeStopped) {
|
|
2523
|
-
return { ok: false, detail: `primary Runtime owner remained live after bounded ${input.service.platform} stop` };
|
|
2524
|
-
}
|
|
2525
3594
|
const serviceStopped = await waitForPrimaryRuntimeServiceStopped({
|
|
2526
3595
|
owner: input.service,
|
|
2527
3596
|
timeoutMs: 20_000,
|
|
@@ -2538,6 +3607,35 @@ async function stopPrimaryRuntimeForReleaseTransition(input: {
|
|
|
2538
3607
|
};
|
|
2539
3608
|
}
|
|
2540
3609
|
|
|
3610
|
+
let runtimeStopped = await waitForPrimaryRuntimeState({
|
|
3611
|
+
config: input.config,
|
|
3612
|
+
expectedRunning: false,
|
|
3613
|
+
timeoutMs: 5_000,
|
|
3614
|
+
now: input.now,
|
|
3615
|
+
wait: input.wait,
|
|
3616
|
+
runtimeRunning: input.runtimeRunning,
|
|
3617
|
+
});
|
|
3618
|
+
if (!runtimeStopped) {
|
|
3619
|
+
const orphanTermination = await terminateVerifiedRuntimeOwner(input.config.controllerHome, ownerBeforeStop);
|
|
3620
|
+
if (!orphanTermination.ok) {
|
|
3621
|
+
return {
|
|
3622
|
+
ok: false,
|
|
3623
|
+
detail: `primary Runtime owner remained live after service shutdown and bounded identity-verified termination was refused/failed: ${orphanTermination.detail}`,
|
|
3624
|
+
};
|
|
3625
|
+
}
|
|
3626
|
+
runtimeStopped = await waitForPrimaryRuntimeState({
|
|
3627
|
+
config: input.config,
|
|
3628
|
+
expectedRunning: false,
|
|
3629
|
+
timeoutMs: 10_000,
|
|
3630
|
+
now: input.now,
|
|
3631
|
+
wait: input.wait,
|
|
3632
|
+
runtimeRunning: input.runtimeRunning,
|
|
3633
|
+
});
|
|
3634
|
+
if (!runtimeStopped) {
|
|
3635
|
+
return { ok: false, detail: 'primary Runtime owner remained live after verified orphan termination' };
|
|
3636
|
+
}
|
|
3637
|
+
}
|
|
3638
|
+
|
|
2541
3639
|
let portReleased = await waitForPrimaryRuntimePortReleased({
|
|
2542
3640
|
config: input.config,
|
|
2543
3641
|
timeoutMs: 20_000,
|
|
@@ -2563,46 +3661,148 @@ async function stopPrimaryRuntimeForReleaseTransition(input: {
|
|
|
2563
3661
|
staleListenerCleanup,
|
|
2564
3662
|
};
|
|
2565
3663
|
}
|
|
2566
|
-
return { ok: true, detail: `primary Runtime ${input.service.platform} service, owner, and TCP listener are stopped`, staleListenerCleanup };
|
|
2567
|
-
}
|
|
2568
3664
|
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
3665
|
+
let ownership = reconcileStoppedRuntimeOwnership(input.config.controllerHome);
|
|
3666
|
+
if (!ownership.ok) {
|
|
3667
|
+
// A Runtime can finish its listener shutdown before its owner process has
|
|
3668
|
+
// exited. The service and port fences above make this a safe, bounded
|
|
3669
|
+
// orphan-termination point; use the same process-identity proof as the
|
|
3670
|
+
// running-state path instead of treating the durable owner as permanent.
|
|
3671
|
+
const orphanTermination = await terminateVerifiedRuntimeOwner(input.config.controllerHome, ownerBeforeStop);
|
|
3672
|
+
if (!orphanTermination.ok) {
|
|
3673
|
+
return {
|
|
3674
|
+
ok: false,
|
|
3675
|
+
detail: `primary Runtime ownership did not quiesce after bounded ${input.service.platform} stop: ${orphanTermination.detail}`,
|
|
3676
|
+
staleListenerCleanup,
|
|
3677
|
+
};
|
|
3678
|
+
}
|
|
3679
|
+
ownership = reconcileStoppedRuntimeOwnership(input.config.controllerHome);
|
|
3680
|
+
if (!ownership.ok) {
|
|
3681
|
+
return {
|
|
3682
|
+
ok: false,
|
|
3683
|
+
detail: `primary Runtime ownership did not quiesce after verified orphan termination: ${ownership.detail}`,
|
|
3684
|
+
staleListenerCleanup,
|
|
3685
|
+
};
|
|
3686
|
+
}
|
|
2581
3687
|
}
|
|
2582
|
-
|
|
3688
|
+
const supervisorSocket = workflowSupervisorSocketPath(resolveWorkflowSupervisorForgeHome(input.config.controllerHome));
|
|
3689
|
+
const supervisor = await reconcileStoppedWorkflowSupervisorSocket(supervisorSocket);
|
|
3690
|
+
if (!supervisor.ok) {
|
|
3691
|
+
return {
|
|
3692
|
+
ok: false,
|
|
3693
|
+
detail: `Workflow Supervisor writer did not quiesce after Runtime stop: ${supervisor.detail}`,
|
|
3694
|
+
staleListenerCleanup,
|
|
3695
|
+
};
|
|
3696
|
+
}
|
|
3697
|
+
return {
|
|
3698
|
+
ok: true,
|
|
3699
|
+
detail: `primary Runtime ${input.service.platform} service, owner, TCP listener, and Workflow Supervisor writer are quiescent`,
|
|
3700
|
+
staleListenerCleanup,
|
|
3701
|
+
};
|
|
2583
3702
|
}
|
|
2584
3703
|
|
|
2585
|
-
|
|
2586
|
-
ok: boolean;
|
|
2587
|
-
detail: string;
|
|
2588
|
-
verify: VerifyResult;
|
|
2589
|
-
}
|
|
3704
|
+
const PRIMARY_RUNTIME_READINESS_POLL_INTERVAL_MS = 250;
|
|
2590
3705
|
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
3706
|
+
async function observePrimaryRuntimeActivationReadiness(config: RecoveryConfig): Promise<VerifyResult> {
|
|
3707
|
+
const observation = observeRuntimeStatus(config.controllerHome);
|
|
3708
|
+
const authority = releaseAuthority(config);
|
|
3709
|
+
const active = releaseEvidence(config.controllerHome, authority?.active, authority);
|
|
3710
|
+
const previous = releaseEvidence(config.controllerHome, authority?.previous, authority);
|
|
3711
|
+
const runtimeHealthy = observation.running && observation.ready && !observation.stale;
|
|
3712
|
+
const coherent = Boolean(
|
|
3713
|
+
authority
|
|
3714
|
+
&& active
|
|
3715
|
+
&& observation.snapshot?.releaseId === active.revision
|
|
3716
|
+
&& observation.snapshot?.artifactIdentity === active.artifactIdentity
|
|
3717
|
+
&& authority.active.workerProtocolVersion === active.workerProtocolVersion,
|
|
3718
|
+
);
|
|
3719
|
+
const probes: VerifyResult['probes'] = {
|
|
3720
|
+
runtime_status: {
|
|
3721
|
+
ok: observation.running && !observation.stale,
|
|
3722
|
+
detail: observation.running
|
|
3723
|
+
? observation.stale ? 'canonical Runtime status is stale' : 'canonical Runtime owner is live'
|
|
3724
|
+
: 'canonical Runtime is not running',
|
|
3725
|
+
},
|
|
3726
|
+
};
|
|
3727
|
+
const endpoint = observation.snapshot?.endpoint;
|
|
3728
|
+
probes.active_gateway = endpoint
|
|
3729
|
+
? await probe(createRecoveryHttpTransport(config.controllerHome), runtimeHealthEndpoint(endpoint), 750)
|
|
3730
|
+
: { ok: false, detail: 'canonical Runtime endpoint is unavailable' };
|
|
3731
|
+
return {
|
|
3732
|
+
ok: Boolean(runtimeHealthy && coherent && probes.active_gateway.ok),
|
|
3733
|
+
at: new Date().toISOString(),
|
|
3734
|
+
runtime: {
|
|
3735
|
+
ok: runtimeHealthy,
|
|
3736
|
+
running: observation.running,
|
|
3737
|
+
ready: observation.ready,
|
|
3738
|
+
stale: observation.stale,
|
|
3739
|
+
reasonCodes: [...observation.reasonCodes],
|
|
3740
|
+
},
|
|
3741
|
+
releases: { active, previous, coherent },
|
|
3742
|
+
probes,
|
|
3743
|
+
};
|
|
3744
|
+
}
|
|
3745
|
+
|
|
3746
|
+
async function waitForPrimaryRuntimeRecoveryHealth(input: {
|
|
3747
|
+
config: RecoveryConfig;
|
|
3748
|
+
timeoutMs: number;
|
|
3749
|
+
now: () => number;
|
|
3750
|
+
wait: (ms: number) => Promise<void>;
|
|
3751
|
+
observeLocal: (config: RecoveryConfig) => Promise<VerifyResult>;
|
|
3752
|
+
}): Promise<VerifyResult> {
|
|
3753
|
+
const deadline = input.now() + input.timeoutMs;
|
|
3754
|
+
let observed = await input.observeLocal(input.config);
|
|
3755
|
+
while (!observed.ok && input.now() < deadline) {
|
|
3756
|
+
await input.wait(PRIMARY_RUNTIME_READINESS_POLL_INTERVAL_MS);
|
|
3757
|
+
observed = await input.observeLocal(input.config);
|
|
3758
|
+
}
|
|
3759
|
+
return observed;
|
|
3760
|
+
}
|
|
3761
|
+
|
|
3762
|
+
async function verifyPrimaryRuntimeAfterStart(input: {
|
|
3763
|
+
config: RecoveryConfig;
|
|
3764
|
+
timeoutMs: number;
|
|
3765
|
+
now: () => number;
|
|
3766
|
+
wait: (ms: number) => Promise<void>;
|
|
3767
|
+
verifyLocal: (config: RecoveryConfig) => Promise<VerifyResult>;
|
|
3768
|
+
observeLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
|
|
3769
|
+
}): Promise<VerifyResult> {
|
|
3770
|
+
const readiness = await waitForPrimaryRuntimeRecoveryHealth({
|
|
3771
|
+
config: input.config,
|
|
3772
|
+
timeoutMs: input.timeoutMs,
|
|
3773
|
+
now: input.now,
|
|
3774
|
+
wait: input.wait,
|
|
3775
|
+
observeLocal: input.observeLocal ?? input.verifyLocal,
|
|
3776
|
+
});
|
|
3777
|
+
if (!readiness.ok) return readiness;
|
|
3778
|
+
// Readiness convergence is intentionally cheap. The acceptance gate remains
|
|
3779
|
+
// one strict whole-Runtime verification after the bounded observation phase.
|
|
3780
|
+
return input.verifyLocal(input.config);
|
|
3781
|
+
}
|
|
3782
|
+
|
|
3783
|
+
interface PrimaryRuntimeRebindStartResult {
|
|
3784
|
+
ok: boolean;
|
|
3785
|
+
detail: string;
|
|
3786
|
+
verify: VerifyResult;
|
|
3787
|
+
}
|
|
3788
|
+
|
|
3789
|
+
/**
|
|
3790
|
+
* Complete the only safe restart sequence after release authority changes:
|
|
3791
|
+
* rebind the release-pinned launchd contract, start the sole Runtime service,
|
|
3792
|
+
* and require whole-Runtime verification. A started-but-unverified Runtime is
|
|
3793
|
+
* stopped again so callers never leave a restart loop behind.
|
|
3794
|
+
*/
|
|
3795
|
+
async function rebindStartAndVerifyPrimaryRuntime(input: {
|
|
3796
|
+
config: RecoveryConfig;
|
|
3797
|
+
service: PrimaryRuntimeServiceOwner;
|
|
3798
|
+
runCommand: CommandRunner;
|
|
3799
|
+
now: () => number;
|
|
3800
|
+
wait: (ms: number) => Promise<void>;
|
|
3801
|
+
verifyLocal: (config: RecoveryConfig) => Promise<VerifyResult>;
|
|
3802
|
+
observeLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
|
|
3803
|
+
ensureRuntimeLaunchContract?: (controllerHome: string) => void;
|
|
2605
3804
|
beforeStart?: () => void;
|
|
3805
|
+
afterRuntimeReady?: () => Promise<{ ok: boolean; detail: string }>;
|
|
2606
3806
|
contractFailureContext?: string;
|
|
2607
3807
|
timeoutMs: number;
|
|
2608
3808
|
successDetail: string;
|
|
@@ -2648,13 +3848,42 @@ async function rebindStartAndVerifyPrimaryRuntime(input: {
|
|
|
2648
3848
|
if (!started.ok) {
|
|
2649
3849
|
return { ok: false, detail: started.detail, verify: await input.verifyLocal(input.config) };
|
|
2650
3850
|
}
|
|
2651
|
-
const
|
|
2652
|
-
|
|
3851
|
+
const observeLocal = input.observeLocal ?? input.verifyLocal;
|
|
3852
|
+
const runtimeOnlyConfig: RecoveryConfig = input.afterRuntimeReady
|
|
3853
|
+
? {
|
|
3854
|
+
...input.config,
|
|
3855
|
+
publicMcpUrl: undefined,
|
|
3856
|
+
primaryConnectorService: undefined,
|
|
3857
|
+
primaryPublicTunnelService: undefined,
|
|
3858
|
+
}
|
|
3859
|
+
: input.config;
|
|
3860
|
+
const runtimeHealth = await waitForPrimaryRuntimeRecoveryHealth({
|
|
3861
|
+
config: runtimeOnlyConfig,
|
|
2653
3862
|
timeoutMs: input.timeoutMs,
|
|
2654
3863
|
now: input.now,
|
|
2655
3864
|
wait: input.wait,
|
|
2656
|
-
|
|
3865
|
+
observeLocal,
|
|
2657
3866
|
});
|
|
3867
|
+
if (!runtimeHealth.ok) {
|
|
3868
|
+
await stopPrimaryRuntimeServiceOwner(input.service, input.runCommand);
|
|
3869
|
+
return {
|
|
3870
|
+
ok: false,
|
|
3871
|
+
detail: input.afterRuntimeReady
|
|
3872
|
+
? 'release transition candidate Runtime did not reach local readiness before Connector rebinding'
|
|
3873
|
+
: 'release transition candidate Runtime did not reach local readiness',
|
|
3874
|
+
verify: runtimeHealth,
|
|
3875
|
+
};
|
|
3876
|
+
}
|
|
3877
|
+
if (input.afterRuntimeReady) {
|
|
3878
|
+
const postReady = await input.afterRuntimeReady();
|
|
3879
|
+
if (!postReady.ok) {
|
|
3880
|
+
await stopPrimaryRuntimeServiceOwner(input.service, input.runCommand);
|
|
3881
|
+
return { ok: false, detail: postReady.detail, verify: await input.verifyLocal(input.config) };
|
|
3882
|
+
}
|
|
3883
|
+
}
|
|
3884
|
+
// Readiness has converged and any Connector rebind has completed. Strict
|
|
3885
|
+
// acceptance runs once here; there is no second readiness polling phase.
|
|
3886
|
+
const verify = await input.verifyLocal(input.config);
|
|
2658
3887
|
if (verify.ok) return { ok: true, detail: input.successDetail, verify };
|
|
2659
3888
|
await stopPrimaryRuntimeServiceOwner(input.service, input.runCommand);
|
|
2660
3889
|
return {
|
|
@@ -2669,8 +3898,10 @@ export async function restartPrimaryRuntime(
|
|
|
2669
3898
|
dependencies: PrimaryRuntimeRecoveryDependencies = {},
|
|
2670
3899
|
): Promise<PrimaryRuntimeRestartResult> {
|
|
2671
3900
|
const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
|
|
3901
|
+
const observeLocal = dependencies.observeLocal ?? (dependencies.verifyLocal ? dependencies.verifyLocal : observePrimaryRuntimeActivationReadiness);
|
|
2672
3902
|
const initial = await verifyLocal(config);
|
|
2673
|
-
|
|
3903
|
+
const forceHealthyRestart = dependencies.forceHealthyRestart === true;
|
|
3904
|
+
if (initial.ok && !forceHealthyRestart) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime is already healthy', verify: initial };
|
|
2674
3905
|
const platform = dependencies.platform ?? process.platform;
|
|
2675
3906
|
const uid = await (dependencies.currentUid ?? currentUid)();
|
|
2676
3907
|
const service = primaryRuntimeServiceOwner(config, platform, uid);
|
|
@@ -2681,8 +3912,52 @@ export async function restartPrimaryRuntime(
|
|
|
2681
3912
|
const wait = dependencies.sleep ?? sleep;
|
|
2682
3913
|
const locked = await withLock(config, { action: 'restart_primary_runtime' }, async () => {
|
|
2683
3914
|
const before = await verifyLocal(config);
|
|
2684
|
-
if (before.ok) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime recovered before restart', serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRestartResult;
|
|
2685
|
-
const
|
|
3915
|
+
if (before.ok && !forceHealthyRestart) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime recovered before restart', serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRestartResult;
|
|
3916
|
+
const runCommand = dependencies.runCommand ?? command;
|
|
3917
|
+
const runtimeRunning = dependencies.runtimeRunning ?? ((value: RecoveryConfig) => observeRuntimeStatus(value.controllerHome).running);
|
|
3918
|
+
const stopped = await stopPrimaryRuntimeForReleaseTransition({
|
|
3919
|
+
config,
|
|
3920
|
+
service,
|
|
3921
|
+
now,
|
|
3922
|
+
wait,
|
|
3923
|
+
runCommand,
|
|
3924
|
+
runtimeRunning,
|
|
3925
|
+
});
|
|
3926
|
+
if (!stopped.ok) {
|
|
3927
|
+
audit(config, 'primary_runtime_restart_quiescence_failed', { serviceTarget: service.target, detail: stopped.detail });
|
|
3928
|
+
return { ok: false, attempted: true, detail: stopped.detail, serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRestartResult;
|
|
3929
|
+
}
|
|
3930
|
+
const activeAuthority = releaseAuthority(config);
|
|
3931
|
+
if (activeAuthority) {
|
|
3932
|
+
const repairConnectorBinding = dependencies.repairPrimaryConnectorBinding
|
|
3933
|
+
?? ((value: RecoveryConfig) => repairPrimaryConnectorBinding(value, platform));
|
|
3934
|
+
const rebound = await rebindStartAndVerifyPrimaryRuntime({
|
|
3935
|
+
config,
|
|
3936
|
+
service,
|
|
3937
|
+
runCommand,
|
|
3938
|
+
now,
|
|
3939
|
+
wait,
|
|
3940
|
+
verifyLocal,
|
|
3941
|
+
observeLocal,
|
|
3942
|
+
ensureRuntimeLaunchContract: dependencies.ensureRuntimeLaunchContract,
|
|
3943
|
+
contractFailureContext: 'before restart',
|
|
3944
|
+
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 30_000,
|
|
3945
|
+
successDetail: 'Canonical Forge Runtime restarted and passed whole-Runtime verification',
|
|
3946
|
+
afterRuntimeReady: async () => {
|
|
3947
|
+
const connector = await repairConnectorBinding(config);
|
|
3948
|
+
return connector.ok
|
|
3949
|
+
? { ok: true, detail: connector.detail }
|
|
3950
|
+
: { ok: false, detail: `persistent Connector binding failed after Runtime readiness: ${connector.detail}` };
|
|
3951
|
+
},
|
|
3952
|
+
});
|
|
3953
|
+
if (rebound.ok) {
|
|
3954
|
+
audit(config, 'primary_runtime_restart_succeeded', { serviceTarget: service.target, release: rebound.verify.releases.active?.revision, launchContractRebound: true });
|
|
3955
|
+
return { ok: true, attempted: true, detail: rebound.detail, serviceTarget: service.target, verify: rebound.verify } satisfies PrimaryRuntimeRestartResult;
|
|
3956
|
+
}
|
|
3957
|
+
audit(config, 'primary_runtime_restart_unverified', { serviceTarget: service.target, reasonCodes: rebound.verify.runtime.reasonCodes, detail: rebound.detail, launchContractRebound: true });
|
|
3958
|
+
return { ok: false, attempted: true, detail: rebound.detail, serviceTarget: service.target, verify: rebound.verify } satisfies PrimaryRuntimeRestartResult;
|
|
3959
|
+
}
|
|
3960
|
+
const started = await ensurePrimaryRuntimeServiceStarted(service, runCommand, 'restart');
|
|
2686
3961
|
if (!started.ok) {
|
|
2687
3962
|
audit(config, 'primary_runtime_restart_failed', { serviceTarget: service.target, detail: started.detail });
|
|
2688
3963
|
return { ok: false, attempted: true, detail: started.detail, serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRestartResult;
|
|
@@ -2693,6 +3968,7 @@ export async function restartPrimaryRuntime(
|
|
|
2693
3968
|
now,
|
|
2694
3969
|
wait,
|
|
2695
3970
|
verifyLocal,
|
|
3971
|
+
observeLocal,
|
|
2696
3972
|
});
|
|
2697
3973
|
if (after.ok) {
|
|
2698
3974
|
audit(config, 'primary_runtime_restart_succeeded', { serviceTarget: service.target, release: after.releases.active?.revision });
|
|
@@ -2711,8 +3987,63 @@ export async function recoverPrimaryRuntime(
|
|
|
2711
3987
|
dependencies: PrimaryRuntimeRecoveryDependencies = {},
|
|
2712
3988
|
): Promise<PrimaryRuntimeRecoveryResult> {
|
|
2713
3989
|
const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
|
|
3990
|
+
const observeLocal = dependencies.observeLocal ?? (dependencies.verifyLocal ? dependencies.verifyLocal : observePrimaryRuntimeActivationReadiness);
|
|
2714
3991
|
const initial = await verifyLocal(config);
|
|
2715
3992
|
if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime recovered before rollback', verify: initial };
|
|
3993
|
+
|
|
3994
|
+
const releaseAuthoritySnapshot = releaseAuthority(config);
|
|
3995
|
+
if (releaseAuthoritySnapshot) {
|
|
3996
|
+
const inventory = listCompleteReleaseSessions(config.controllerHome);
|
|
3997
|
+
if (inventory.invalidSessionFiles.length > 0) {
|
|
3998
|
+
return {
|
|
3999
|
+
ok: false,
|
|
4000
|
+
attempted: false,
|
|
4001
|
+
noOp: true,
|
|
4002
|
+
detail: `RELEASE_SESSION_INVENTORY_INCOMPLETE: invalid=${inventory.invalidSessionFiles.join(',') || 'none'}`,
|
|
4003
|
+
verify: initial,
|
|
4004
|
+
};
|
|
4005
|
+
}
|
|
4006
|
+
const matching = inventory.sessions.filter((session) => (
|
|
4007
|
+
['cutover_attempting', 'cutover_committed', 'soaking'].includes(session.phase)
|
|
4008
|
+
&& session.transaction
|
|
4009
|
+
&& session.candidateRelease?.releaseId === releaseAuthoritySnapshot.active.releaseId
|
|
4010
|
+
&& session.candidateRelease.artifactIdentity === releaseAuthoritySnapshot.active.artifactIdentity
|
|
4011
|
+
));
|
|
4012
|
+
if (matching.length > 1) {
|
|
4013
|
+
return {
|
|
4014
|
+
ok: false,
|
|
4015
|
+
attempted: false,
|
|
4016
|
+
noOp: true,
|
|
4017
|
+
detail: `RELEASE_SESSION_MULTIPLE_ACTIVE_FOR_RUNTIME: ${matching.map((session) => session.sessionId).join(',')}`,
|
|
4018
|
+
verify: initial,
|
|
4019
|
+
};
|
|
4020
|
+
}
|
|
4021
|
+
const session = matching[0];
|
|
4022
|
+
if (session) {
|
|
4023
|
+
const sessionRollback = await rollbackConfiguredRuntimeReleaseSession(
|
|
4024
|
+
config,
|
|
4025
|
+
session.sessionId,
|
|
4026
|
+
dependencies,
|
|
4027
|
+
`recover-primary-runtime:${session.transaction!.operationId}`,
|
|
4028
|
+
);
|
|
4029
|
+
const after = await verifyLocal(config);
|
|
4030
|
+
const rollback: RollbackResult = {
|
|
4031
|
+
ok: sessionRollback.ok,
|
|
4032
|
+
...(sessionRollback.noOp === true ? { noOp: true } : {}),
|
|
4033
|
+
detail: sessionRollback.detail,
|
|
4034
|
+
verify: after,
|
|
4035
|
+
};
|
|
4036
|
+
return {
|
|
4037
|
+
ok: sessionRollback.ok,
|
|
4038
|
+
attempted: sessionRollback.attempted,
|
|
4039
|
+
...(sessionRollback.noOp === true ? { noOp: true } : {}),
|
|
4040
|
+
detail: sessionRollback.detail,
|
|
4041
|
+
rollback,
|
|
4042
|
+
verify: after,
|
|
4043
|
+
};
|
|
4044
|
+
}
|
|
4045
|
+
}
|
|
4046
|
+
|
|
2716
4047
|
const platform = dependencies.platform ?? process.platform;
|
|
2717
4048
|
const uid = await (dependencies.currentUid ?? currentUid)();
|
|
2718
4049
|
const service = primaryRuntimeServiceOwner(config, platform, uid);
|
|
@@ -2757,6 +4088,7 @@ export async function recoverPrimaryRuntime(
|
|
|
2757
4088
|
now,
|
|
2758
4089
|
wait,
|
|
2759
4090
|
verifyLocal,
|
|
4091
|
+
observeLocal,
|
|
2760
4092
|
ensureRuntimeLaunchContract: dependencies.ensureRuntimeLaunchContract,
|
|
2761
4093
|
contractFailureContext: 'after rollback',
|
|
2762
4094
|
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 45_000,
|
|
@@ -2829,17 +4161,17 @@ function validateRuntimeReleaseCandidate(
|
|
|
2829
4161
|
|
|
2830
4162
|
/**
|
|
2831
4163
|
* Activate an already staged and validated immutable Runtime release without
|
|
2832
|
-
* depending on the primary Runtime execution plane.
|
|
2833
|
-
*
|
|
2834
|
-
*
|
|
2835
|
-
*
|
|
2836
|
-
* restore the previous whole release and its SQLite backup before restarting.
|
|
4164
|
+
* depending on the primary Runtime execution plane. RuntimeReleaseAuthority
|
|
4165
|
+
* changes only physical active/previous identity plus the SQLite backup; when a
|
|
4166
|
+
* ReleaseSession owns the cutover, its transaction snapshot is the sole durable
|
|
4167
|
+
* semantic rollback authority through soak.
|
|
2837
4168
|
*/
|
|
2838
|
-
|
|
4169
|
+
async function activateRuntimeReleaseInternal(
|
|
2839
4170
|
config: RecoveryConfig,
|
|
2840
4171
|
candidateManifestPath: string,
|
|
2841
4172
|
dependencies: PrimaryRuntimeRecoveryDependencies = {},
|
|
2842
4173
|
guard: RuntimeReleaseActivationGuard = {},
|
|
4174
|
+
heldRecoveryLock?: RecoveryLock,
|
|
2843
4175
|
): Promise<RuntimeReleaseActivationResult> {
|
|
2844
4176
|
let candidate: { manifest: RuntimeReleaseManifest; releaseRoot: string; manifestPath: string };
|
|
2845
4177
|
try {
|
|
@@ -2849,6 +4181,56 @@ export async function activateRuntimeRelease(
|
|
|
2849
4181
|
audit(config, 'runtime_release_activation_candidate_invalid', { detail, ...(guard.requestId ? { requestId: guard.requestId } : {}) });
|
|
2850
4182
|
return { ok: false, attempted: false, noOp: true, detail };
|
|
2851
4183
|
}
|
|
4184
|
+
if (candidate.manifest.deploymentScope === 'portable' && !guard.requiredPinnedReleaseRevision) {
|
|
4185
|
+
const releaseSessionId = guard.releaseSessionId?.trim();
|
|
4186
|
+
if (!releaseSessionId) {
|
|
4187
|
+
const detail = 'RELEASE_SESSION_CUTOVER_REQUIRED: portable Runtime candidates cannot activate outside an exact ReleaseSession';
|
|
4188
|
+
audit(config, 'runtime_release_activation_release_session_required', {
|
|
4189
|
+
candidateRevision: candidate.manifest.releaseId,
|
|
4190
|
+
...(guard.requestId ? { requestId: guard.requestId } : {}),
|
|
4191
|
+
});
|
|
4192
|
+
return { ok: false, attempted: false, noOp: true, detail };
|
|
4193
|
+
}
|
|
4194
|
+
let session: ReleaseSession | undefined;
|
|
4195
|
+
try {
|
|
4196
|
+
session = readReleaseSession(config.controllerHome, releaseSessionId);
|
|
4197
|
+
} catch {
|
|
4198
|
+
session = undefined;
|
|
4199
|
+
}
|
|
4200
|
+
if (!session) {
|
|
4201
|
+
return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
|
|
4202
|
+
}
|
|
4203
|
+
if (session.phase !== 'cutover_attempting') {
|
|
4204
|
+
return {
|
|
4205
|
+
ok: false,
|
|
4206
|
+
attempted: false,
|
|
4207
|
+
noOp: true,
|
|
4208
|
+
detail: `RELEASE_SESSION_NOT_CUTOVER_ATTEMPTING: ${session.phase}`,
|
|
4209
|
+
};
|
|
4210
|
+
}
|
|
4211
|
+
if (resolve(session.stable.controllerHome) !== resolve(config.controllerHome)) {
|
|
4212
|
+
return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_STABLE_LANE_MISMATCH' };
|
|
4213
|
+
}
|
|
4214
|
+
const bound = session.candidateRelease;
|
|
4215
|
+
if (!bound) {
|
|
4216
|
+
return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED' };
|
|
4217
|
+
}
|
|
4218
|
+
const observedManifestSha256 = createHash('sha256').update(readFileSync(candidate.manifestPath)).digest('hex');
|
|
4219
|
+
const observedTreeSha256 = runtimeReleaseTreeSha256(candidate.releaseRoot);
|
|
4220
|
+
if (
|
|
4221
|
+
bound.releaseId !== candidate.manifest.releaseId
|
|
4222
|
+
|| bound.artifactIdentity !== candidate.manifest.artifactIdentity
|
|
4223
|
+
|| bound.manifestSha256 !== observedManifestSha256
|
|
4224
|
+
|| bound.treeSha256 !== observedTreeSha256
|
|
4225
|
+
) {
|
|
4226
|
+
return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_IDENTITY_MISMATCH' };
|
|
4227
|
+
}
|
|
4228
|
+
try {
|
|
4229
|
+
assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
|
|
4230
|
+
} catch (error) {
|
|
4231
|
+
return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) };
|
|
4232
|
+
}
|
|
4233
|
+
}
|
|
2852
4234
|
const platform = dependencies.platform ?? process.platform;
|
|
2853
4235
|
const uid = await (dependencies.currentUid ?? currentUid)();
|
|
2854
4236
|
const service = primaryRuntimeServiceOwner(config, platform, uid);
|
|
@@ -2860,17 +4242,25 @@ export async function activateRuntimeRelease(
|
|
|
2860
4242
|
const wait = dependencies.sleep ?? sleep;
|
|
2861
4243
|
const runtimeRunning = dependencies.runtimeRunning ?? ((value: RecoveryConfig) => observeRuntimeStatus(value.controllerHome).running);
|
|
2862
4244
|
const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
|
|
4245
|
+
const observeLocal = dependencies.observeLocal ?? (dependencies.verifyLocal ? dependencies.verifyLocal : observePrimaryRuntimeActivationReadiness);
|
|
2863
4246
|
const repairConnectorBinding = dependencies.repairPrimaryConnectorBinding
|
|
2864
4247
|
?? ((value: RecoveryConfig) => repairPrimaryConnectorBinding(value, platform));
|
|
2865
4248
|
const operationId = `recovery-activate-runtime-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
|
2866
4249
|
const lockRequestId = guard.requestId?.trim() || operationId;
|
|
2867
|
-
const
|
|
4250
|
+
const activateUnderRecoveryAuthority = async () => {
|
|
2868
4251
|
// Re-read authority only after acquiring the mutation lock. A caller may
|
|
2869
4252
|
// have selected its candidate before another activation completed; stale
|
|
2870
4253
|
// decisions must fail before bootout/publish rather than overwrite the newer
|
|
2871
4254
|
// active release.
|
|
2872
4255
|
const current = releaseAuthority(config);
|
|
2873
4256
|
const previousActive = current?.active;
|
|
4257
|
+
if (guard.requiredPinnedReleaseRevision) {
|
|
4258
|
+
const pinned = runtimePin(config)?.release;
|
|
4259
|
+
if (!pinned || pinned.revision !== guard.requiredPinnedReleaseRevision || pinned.revision !== candidate.manifest.releaseId) {
|
|
4260
|
+
const detail = `RUNTIME_PIN_AUTHORITY_CHANGED: expected ${guard.requiredPinnedReleaseRevision}, observed ${pinned?.revision ?? 'none'}`;
|
|
4261
|
+
return { ok: false, attempted: false, noOp: true, detail, operationId } satisfies RuntimeReleaseActivationResult;
|
|
4262
|
+
}
|
|
4263
|
+
}
|
|
2874
4264
|
const expectedRevision = guard.expectedAuthorityRevision;
|
|
2875
4265
|
const expectedActiveReleaseId = typeof guard.expectedActiveReleaseId === 'string' ? guard.expectedActiveReleaseId.trim() : guard.expectedActiveReleaseId;
|
|
2876
4266
|
if (expectedRevision !== undefined && (expectedRevision === null ? current !== undefined : current?.revision !== expectedRevision)) {
|
|
@@ -2958,7 +4348,7 @@ export async function activateRuntimeRelease(
|
|
|
2958
4348
|
} satisfies RuntimeReleaseActivationResult;
|
|
2959
4349
|
}
|
|
2960
4350
|
}
|
|
2961
|
-
if (current?.previous?.releaseId === candidate.manifest.releaseId) {
|
|
4351
|
+
if (!guard.allowPreviousRelease && current?.previous?.releaseId === candidate.manifest.releaseId) {
|
|
2962
4352
|
const detail = 'RUNTIME_RELEASE_REVERSE_ACTIVATION_REQUIRES_ROLLBACK: activate_runtime_release cannot replace the active release with current.previous; use rollback_previous or recover_primary_runtime';
|
|
2963
4353
|
audit(config, 'runtime_release_reverse_activation_rejected', {
|
|
2964
4354
|
operationId,
|
|
@@ -2969,9 +4359,23 @@ export async function activateRuntimeRelease(
|
|
|
2969
4359
|
});
|
|
2970
4360
|
return { ok: false, attempted: false, noOp: true, detail, operationId } satisfies RuntimeReleaseActivationResult;
|
|
2971
4361
|
}
|
|
2972
|
-
|
|
4362
|
+
// Snapshot Stable A while it is still serving. The release store keeps a
|
|
4363
|
+
// database-wide SQLite data_version observer open across this snapshot and
|
|
4364
|
+
// the stop boundary. Any intervening commit invalidates the speculative
|
|
4365
|
+
// snapshot and publish falls back to a stopped-state backup.
|
|
4366
|
+
let preparedDatabaseBackup;
|
|
4367
|
+
try {
|
|
4368
|
+
preparedDatabaseBackup = current
|
|
4369
|
+
? prepareRuntimeReleaseDatabaseBackup(config.controllerHome, operationId)
|
|
4370
|
+
: undefined;
|
|
4371
|
+
} catch (error) {
|
|
4372
|
+
const detail = error instanceof Error ? error.message : 'runtime release rollback database preparation failed';
|
|
4373
|
+
audit(config, 'runtime_release_activation_backup_prepare_failed', { serviceTarget: service.target, operationId, detail });
|
|
4374
|
+
return { ok: false, attempted: false, noOp: true, detail, serviceTarget: service.target, verify: await observeLocal(config) } satisfies RuntimeReleaseActivationResult;
|
|
4375
|
+
}
|
|
2973
4376
|
const stopped = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
|
|
2974
4377
|
if (!stopped.ok) {
|
|
4378
|
+
preparedDatabaseBackup?.discard();
|
|
2975
4379
|
const action = stopped.detail.startsWith('primary Runtime bootout failed')
|
|
2976
4380
|
? 'runtime_release_activation_stop_failed'
|
|
2977
4381
|
: 'runtime_release_activation_stop_unverified';
|
|
@@ -2989,7 +4393,13 @@ export async function activateRuntimeRelease(
|
|
|
2989
4393
|
}
|
|
2990
4394
|
let committed: RuntimeReleaseAuthority;
|
|
2991
4395
|
try {
|
|
2992
|
-
committed = publishRuntimeRelease(
|
|
4396
|
+
committed = publishRuntimeRelease(
|
|
4397
|
+
config.controllerHome,
|
|
4398
|
+
candidate.manifestPath,
|
|
4399
|
+
operationId,
|
|
4400
|
+
undefined,
|
|
4401
|
+
preparedDatabaseBackup,
|
|
4402
|
+
);
|
|
2993
4403
|
} catch (error) {
|
|
2994
4404
|
const detail = error instanceof Error ? error.message : 'runtime release authority publish failed';
|
|
2995
4405
|
audit(config, 'runtime_release_activation_publish_failed', { serviceTarget: service.target, operationId, detail });
|
|
@@ -3000,65 +4410,113 @@ export async function activateRuntimeRelease(
|
|
|
3000
4410
|
audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId });
|
|
3001
4411
|
return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
|
|
3002
4412
|
}
|
|
3003
|
-
|
|
3004
|
-
//
|
|
3005
|
-
//
|
|
3006
|
-
//
|
|
3007
|
-
|
|
3008
|
-
let storageMigration: ControllerHomeStorageMigration | undefined;
|
|
4413
|
+
|
|
4414
|
+
// Once the physical authority switch is durable, persist exact Stable A
|
|
4415
|
+
// rollback identity before Candidate B starts. A later start/Connector/verify
|
|
4416
|
+
// failure therefore never precedes ReleaseSession rollback authority.
|
|
4417
|
+
let releaseSessionTransactionPersisted = false;
|
|
3009
4418
|
let activationFailureDetail: string | undefined;
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
}
|
|
3042
|
-
}
|
|
4419
|
+
if (!guard.preserveDatabaseOnFailure && guard.releaseSessionId?.trim()) {
|
|
4420
|
+
try {
|
|
4421
|
+
const session = readReleaseSession(config.controllerHome, guard.releaseSessionId.trim());
|
|
4422
|
+
if (
|
|
4423
|
+
committed.operationId !== operationId
|
|
4424
|
+
|| !committed.previous?.databaseBackup
|
|
4425
|
+
|| !session
|
|
4426
|
+
|| session.phase !== 'cutover_attempting'
|
|
4427
|
+
) throw new Error('RELEASE_SESSION_TRANSACTION_CAPTURE_PRECONDITION_FAILED');
|
|
4428
|
+
recordReleaseSessionTransaction({
|
|
4429
|
+
controllerHome: config.controllerHome,
|
|
4430
|
+
sessionId: session.sessionId,
|
|
4431
|
+
expectedRevision: session.revision,
|
|
4432
|
+
transaction: {
|
|
4433
|
+
schemaVersion: 1,
|
|
4434
|
+
operationId,
|
|
4435
|
+
candidateReleaseId: candidate.manifest.releaseId,
|
|
4436
|
+
cutoverAuthorityRevision: committed.revision,
|
|
4437
|
+
rollbackRelease: committed.previous,
|
|
4438
|
+
startedAt: committed.committedAt,
|
|
4439
|
+
},
|
|
4440
|
+
});
|
|
4441
|
+
releaseSessionTransactionPersisted = true;
|
|
4442
|
+
} catch (error) {
|
|
4443
|
+
activationFailureDetail = `ReleaseSession rollback transaction capture failed before Runtime start: ${error instanceof Error ? error.message : String(error)}`;
|
|
4444
|
+
audit(config, 'release_session_transaction_capture_failed', {
|
|
4445
|
+
serviceTarget: service.target,
|
|
4446
|
+
operationId,
|
|
4447
|
+
requestId: lockRequestId,
|
|
4448
|
+
releaseSessionId: guard.releaseSessionId.trim(),
|
|
4449
|
+
detail: activationFailureDetail,
|
|
4450
|
+
});
|
|
4451
|
+
}
|
|
3043
4452
|
}
|
|
4453
|
+
|
|
4454
|
+
// The Connector is independently supervised, but its package snapshot proxies
|
|
4455
|
+
// the Canonical Runtime. Starting/rebinding it while the Runtime is stopped
|
|
4456
|
+
// creates a dependency cycle: Connector readiness waits on a Runtime that has
|
|
4457
|
+
// not started yet. Start the candidate first, require Runtime-only readiness,
|
|
4458
|
+
// then rebind the Connector and finally require whole-Runtime verification.
|
|
4459
|
+
let candidateConnectorBinding: { ok: boolean; attempted: boolean; noOp?: boolean; detail: string } | undefined;
|
|
4460
|
+
let storageMigration: ControllerHomeStorageMigration | undefined;
|
|
4461
|
+
const activated = activationFailureDetail
|
|
4462
|
+
? { ok: false, detail: activationFailureDetail, verify: await observeLocal(config) }
|
|
4463
|
+
: await rebindStartAndVerifyPrimaryRuntime({
|
|
4464
|
+
config,
|
|
4465
|
+
service,
|
|
4466
|
+
runCommand,
|
|
4467
|
+
now,
|
|
4468
|
+
wait,
|
|
4469
|
+
verifyLocal,
|
|
4470
|
+
observeLocal,
|
|
4471
|
+
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
|
|
4472
|
+
successDetail: 'requested Runtime release started and passed whole-Runtime verification',
|
|
4473
|
+
beforeStart: () => {
|
|
4474
|
+
storageMigration = migrateStoppedRepoLocalControllerHomeStorage(config.controllerHome, platform);
|
|
4475
|
+
if (storageMigration.migrated) {
|
|
4476
|
+
audit(config, 'runtime_controller_home_noindex_migrated', {
|
|
4477
|
+
serviceTarget: service.target,
|
|
4478
|
+
operationId,
|
|
4479
|
+
logicalHome: storageMigration.logicalHome,
|
|
4480
|
+
physicalHome: storageMigration.physicalHome,
|
|
4481
|
+
});
|
|
4482
|
+
}
|
|
4483
|
+
},
|
|
4484
|
+
afterRuntimeReady: async () => {
|
|
4485
|
+
candidateConnectorBinding = await repairConnectorBinding(config);
|
|
4486
|
+
if (candidateConnectorBinding.ok) return { ok: true, detail: candidateConnectorBinding.detail };
|
|
4487
|
+
const detail = `persistent Connector binding failed after Runtime readiness: ${candidateConnectorBinding.detail}`;
|
|
4488
|
+
activationFailureDetail = detail;
|
|
4489
|
+
audit(config, 'runtime_release_activation_connector_binding_failed', {
|
|
4490
|
+
serviceTarget: service.target,
|
|
4491
|
+
operationId,
|
|
4492
|
+
requestId: lockRequestId,
|
|
4493
|
+
detail: candidateConnectorBinding.detail,
|
|
4494
|
+
});
|
|
4495
|
+
return { ok: false, detail };
|
|
4496
|
+
},
|
|
4497
|
+
});
|
|
3044
4498
|
let after = activated.verify;
|
|
3045
4499
|
if (activated.ok && after.releases.active?.revision === candidate.manifest.releaseId) {
|
|
3046
4500
|
audit(config, 'runtime_release_activation_succeeded', {
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
4501
|
+
serviceTarget: service.target,
|
|
4502
|
+
operationId,
|
|
4503
|
+
requestId: lockRequestId,
|
|
4504
|
+
activeRevision: after.releases.active?.revision,
|
|
4505
|
+
expectedAuthorityRevision: guard.expectedAuthorityRevision,
|
|
4506
|
+
expectedActiveReleaseId: guard.expectedActiveReleaseId,
|
|
4507
|
+
controllerHomeStorageMigrated: storageMigration?.migrated === true,
|
|
4508
|
+
connectorBindingRepaired: candidateConnectorBinding?.attempted === true,
|
|
4509
|
+
releaseSessionTransactionPersisted,
|
|
4510
|
+
...(guard.releaseSessionId?.trim() ? { releaseSessionId: guard.releaseSessionId.trim() } : {}),
|
|
3055
4511
|
});
|
|
3056
4512
|
return {
|
|
3057
4513
|
ok: true,
|
|
3058
4514
|
attempted: true,
|
|
3059
|
-
detail:
|
|
3060
|
-
? 'requested Runtime release activated
|
|
3061
|
-
:
|
|
4515
|
+
detail: releaseSessionTransactionPersisted
|
|
4516
|
+
? 'requested Runtime release activated and verified; physical activation committed and ReleaseSession owns rollback authority through soak'
|
|
4517
|
+
: storageMigration?.migrated
|
|
4518
|
+
? 'requested Runtime release activated, Controller Home migrated to .noindex storage, persistent Connector rebound, and whole-Runtime verification passed'
|
|
4519
|
+
: 'requested Runtime release activated, persistent Connector rebound, and whole-Runtime verification passed',
|
|
3062
4520
|
serviceTarget: service.target,
|
|
3063
4521
|
operationId,
|
|
3064
4522
|
verify: after,
|
|
@@ -3079,8 +4537,9 @@ export async function activateRuntimeRelease(
|
|
|
3079
4537
|
audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId, detail: activationFailureDetail });
|
|
3080
4538
|
}
|
|
3081
4539
|
|
|
3082
|
-
// Activation failed: stop
|
|
3083
|
-
//
|
|
4540
|
+
// Activation failed: stop and restore the previous whole release. SQLite
|
|
4541
|
+
// rollback is allowed only when the live durable-mutation generation still
|
|
4542
|
+
// equals the captured backup; newer Controller Home state is preserved.
|
|
3084
4543
|
let rollback: RollbackResult;
|
|
3085
4544
|
const rollbackStop = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
|
|
3086
4545
|
if (!rollbackStop.ok) {
|
|
@@ -3092,8 +4551,25 @@ export async function activateRuntimeRelease(
|
|
|
3092
4551
|
};
|
|
3093
4552
|
} else {
|
|
3094
4553
|
try {
|
|
3095
|
-
const rollbackOperationId =
|
|
3096
|
-
|
|
4554
|
+
const rollbackOperationId = guard.preserveDatabaseOnFailure
|
|
4555
|
+
? `recovery-activate-runtime-rollback-${Date.now()}-${randomUUID().slice(0, 8)}`
|
|
4556
|
+
: operationId;
|
|
4557
|
+
let rollbackDatabaseDisposition: RuntimeDatabaseRollbackDisposition | 'preserved_by_guard' = 'preserved_by_guard';
|
|
4558
|
+
const physicallyRestored = guard.preserveDatabaseOnFailure && previousActive
|
|
4559
|
+
? publishRuntimeRelease(config.controllerHome, previousActive.manifestPath, rollbackOperationId)
|
|
4560
|
+
: (() => {
|
|
4561
|
+
const rollbackResult = rollbackRuntimeReleaseWithResult(config.controllerHome, rollbackOperationId);
|
|
4562
|
+
rollbackDatabaseDisposition = rollbackResult.databaseDisposition;
|
|
4563
|
+
return rollbackResult.authority;
|
|
4564
|
+
})();
|
|
4565
|
+
const restored = current && previousActive
|
|
4566
|
+
? reconcileFailedRuntimeReleaseActivationAuthority(
|
|
4567
|
+
config.controllerHome,
|
|
4568
|
+
current,
|
|
4569
|
+
candidate.manifest.releaseId,
|
|
4570
|
+
rollbackOperationId,
|
|
4571
|
+
)
|
|
4572
|
+
: physicallyRestored;
|
|
3097
4573
|
if (storageMigration?.migrated) {
|
|
3098
4574
|
rollbackStoppedRepoLocalControllerHomeStorage(storageMigration);
|
|
3099
4575
|
audit(config, 'runtime_controller_home_noindex_migration_rolled_back', {
|
|
@@ -3112,6 +4588,7 @@ export async function activateRuntimeRelease(
|
|
|
3112
4588
|
) {
|
|
3113
4589
|
throw new Error('RECOVERY_RUNTIME_RELEASE_ROLLBACK_AUTHORITY_MISMATCH');
|
|
3114
4590
|
}
|
|
4591
|
+
let rollbackConnectorBinding: { ok: boolean; attempted: boolean; noOp?: boolean; detail: string } | undefined;
|
|
3115
4592
|
const restarted = await rebindStartAndVerifyPrimaryRuntime({
|
|
3116
4593
|
config,
|
|
3117
4594
|
service,
|
|
@@ -3119,25 +4596,31 @@ export async function activateRuntimeRelease(
|
|
|
3119
4596
|
now,
|
|
3120
4597
|
wait,
|
|
3121
4598
|
verifyLocal,
|
|
4599
|
+
observeLocal,
|
|
3122
4600
|
contractFailureContext: 'after rollback',
|
|
3123
4601
|
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
|
|
3124
|
-
successDetail:
|
|
4602
|
+
successDetail: rollbackDatabaseDisposition === 'restored_backup'
|
|
4603
|
+
? 'previous whole-Runtime release and unchanged SQLite backup restored, restarted, rebound, and verified'
|
|
4604
|
+
: 'previous Runtime release restored while live SQLite state was preserved, restarted, rebound, and verified',
|
|
4605
|
+
afterRuntimeReady: async () => {
|
|
4606
|
+
rollbackConnectorBinding = await repairConnectorBinding(config);
|
|
4607
|
+
if (rollbackConnectorBinding.ok) return { ok: true, detail: rollbackConnectorBinding.detail };
|
|
4608
|
+
audit(config, 'runtime_release_activation_rollback_connector_binding_failed', {
|
|
4609
|
+
serviceTarget: service.target,
|
|
4610
|
+
operationId,
|
|
4611
|
+
rollbackOperationId,
|
|
4612
|
+
detail: rollbackConnectorBinding.detail,
|
|
4613
|
+
});
|
|
4614
|
+
return {
|
|
4615
|
+
ok: false,
|
|
4616
|
+
detail: `previous persistent Connector binding failed after rollback Runtime readiness: ${rollbackConnectorBinding.detail}`,
|
|
4617
|
+
};
|
|
4618
|
+
},
|
|
3125
4619
|
});
|
|
3126
|
-
const connectorBinding = restarted.ok ? await repairConnectorBinding(config) : undefined;
|
|
3127
|
-
if (connectorBinding && !connectorBinding.ok) {
|
|
3128
|
-
audit(config, 'runtime_release_activation_rollback_connector_binding_failed', {
|
|
3129
|
-
serviceTarget: service.target,
|
|
3130
|
-
operationId,
|
|
3131
|
-
rollbackOperationId,
|
|
3132
|
-
detail: connectorBinding.detail,
|
|
3133
|
-
});
|
|
3134
|
-
}
|
|
3135
4620
|
rollback = {
|
|
3136
|
-
ok: restarted.ok
|
|
4621
|
+
ok: restarted.ok,
|
|
3137
4622
|
operationId: rollbackOperationId,
|
|
3138
|
-
detail:
|
|
3139
|
-
? `${restarted.detail}; previous persistent Connector binding failed: ${connectorBinding.detail}`
|
|
3140
|
-
: restarted.detail,
|
|
4623
|
+
detail: restarted.detail,
|
|
3141
4624
|
verify: restarted.verify,
|
|
3142
4625
|
};
|
|
3143
4626
|
} catch (error) {
|
|
@@ -3164,80 +4647,1950 @@ export async function activateRuntimeRelease(
|
|
|
3164
4647
|
rollback,
|
|
3165
4648
|
verify: after,
|
|
3166
4649
|
} satisfies RuntimeReleaseActivationResult;
|
|
3167
|
-
}
|
|
4650
|
+
};
|
|
4651
|
+
|
|
4652
|
+
if (heldRecoveryLock) {
|
|
4653
|
+
const live = liveRecoveryMutationLock(config);
|
|
4654
|
+
if (
|
|
4655
|
+
!live
|
|
4656
|
+
|| live.instanceId !== heldRecoveryLock.instanceId
|
|
4657
|
+
|| live.pid !== heldRecoveryLock.pid
|
|
4658
|
+
|| live.processStartTime !== heldRecoveryLock.processStartTime
|
|
4659
|
+
) {
|
|
4660
|
+
return {
|
|
4661
|
+
ok: false,
|
|
4662
|
+
attempted: false,
|
|
4663
|
+
noOp: true,
|
|
4664
|
+
detail: 'RECOVERY_OPERATION_LOCK_AUTHORITY_CHANGED',
|
|
4665
|
+
serviceTarget: service.target,
|
|
4666
|
+
verify: await verifyStableRuntime(config),
|
|
4667
|
+
};
|
|
4668
|
+
}
|
|
4669
|
+
return activateUnderRecoveryAuthority();
|
|
4670
|
+
}
|
|
4671
|
+
|
|
4672
|
+
const locked = await withLock(
|
|
4673
|
+
config,
|
|
4674
|
+
{ action: 'activate_runtime_release', requestId: lockRequestId },
|
|
4675
|
+
async () => activateUnderRecoveryAuthority(),
|
|
4676
|
+
);
|
|
3168
4677
|
if (!locked.acquired) {
|
|
3169
4678
|
return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), serviceTarget: service.target, verify: await verifyStableRuntime(config) };
|
|
3170
4679
|
}
|
|
3171
4680
|
return locked.value;
|
|
3172
4681
|
}
|
|
3173
4682
|
|
|
3174
|
-
export
|
|
3175
|
-
|
|
3176
|
-
|
|
4683
|
+
export async function activateRuntimeRelease(
|
|
4684
|
+
config: RecoveryConfig,
|
|
4685
|
+
candidateManifestPath: string,
|
|
4686
|
+
dependencies: PrimaryRuntimeRecoveryDependencies = {},
|
|
4687
|
+
guard: RuntimeReleaseActivationGuard = {},
|
|
4688
|
+
): Promise<RuntimeReleaseActivationResult> {
|
|
4689
|
+
return activateRuntimeReleaseInternal(config, candidateManifestPath, dependencies, guard);
|
|
3177
4690
|
}
|
|
3178
4691
|
|
|
3179
|
-
export async function
|
|
4692
|
+
export async function activatePinnedRuntimeRelease(
|
|
3180
4693
|
config: RecoveryConfig,
|
|
3181
|
-
dependencies:
|
|
3182
|
-
|
|
3183
|
-
): Promise<
|
|
3184
|
-
|
|
3185
|
-
|
|
3186
|
-
|
|
3187
|
-
}
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3191
|
-
|
|
3192
|
-
|
|
3193
|
-
|
|
3194
|
-
|
|
3195
|
-
|
|
3196
|
-
|
|
3197
|
-
|
|
4694
|
+
dependencies: PrimaryRuntimeRecoveryDependencies = {},
|
|
4695
|
+
guard: RuntimeReleaseActivationGuard = {},
|
|
4696
|
+
): Promise<RuntimeReleaseActivationResult> {
|
|
4697
|
+
let pin: RuntimePinStore | undefined;
|
|
4698
|
+
try { pin = runtimePin(config); }
|
|
4699
|
+
catch (error) { return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) }; }
|
|
4700
|
+
if (!pin) return { ok: false, attempted: false, noOp: true, detail: 'RUNTIME_PIN_REQUIRED: no Runtime release is pinned' };
|
|
4701
|
+
let candidate: { manifest: RuntimeReleaseManifest; releaseRoot: string; manifestPath: string };
|
|
4702
|
+
try { candidate = validateRuntimeReleaseCandidate(config, pin.release.path); }
|
|
4703
|
+
catch (error) { return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) }; }
|
|
4704
|
+
if (candidate.manifest.deploymentScope === 'portable') {
|
|
4705
|
+
return {
|
|
4706
|
+
ok: false,
|
|
4707
|
+
attempted: false,
|
|
4708
|
+
noOp: true,
|
|
4709
|
+
detail: 'RUNTIME_PIN_PORTABLE_RELEASE_FORBIDDEN: portable source candidates require a verified Recovery ReleaseSession',
|
|
4710
|
+
};
|
|
4711
|
+
}
|
|
4712
|
+
let observed: ReleaseEvidence;
|
|
4713
|
+
try { observed = releaseEvidenceForPin(config, candidate); }
|
|
4714
|
+
catch (error) { return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : String(error) }; }
|
|
4715
|
+
if (observed.revision !== pin.release.revision || observed.artifactIdentity !== pin.release.artifactIdentity || observed.manifestSha256 !== pin.release.manifestSha256) {
|
|
4716
|
+
return { ok: false, attempted: false, noOp: true, detail: 'RUNTIME_PIN_IDENTITY_MISMATCH: pinned immutable release changed on disk' };
|
|
4717
|
+
}
|
|
4718
|
+
return activateRuntimeRelease(config, pin.release.path, dependencies, {
|
|
4719
|
+
...guard,
|
|
4720
|
+
allowPreviousRelease: true,
|
|
4721
|
+
preserveDatabaseOnFailure: true,
|
|
4722
|
+
requiredPinnedReleaseRevision: pin.release.revision,
|
|
4723
|
+
});
|
|
4724
|
+
}
|
|
4725
|
+
|
|
4726
|
+
|
|
4727
|
+
function configuredSourceRevision(sourceRoot: string): string {
|
|
4728
|
+
const env = {
|
|
4729
|
+
...runtimeAuthorityFreeEnvironment(process.env),
|
|
4730
|
+
PATH: recoveryCommandPath(),
|
|
4731
|
+
};
|
|
4732
|
+
// ReleaseSession source authority is the exact committed Git object selected
|
|
4733
|
+
// internally by Recovery. The configured checkout is only an object/dependency
|
|
4734
|
+
// provider: concurrent working-tree edits are intentionally non-authoritative
|
|
4735
|
+
// and are excluded by the detached immutable source snapshot used below.
|
|
4736
|
+
const head = spawnSync('git', ['-C', sourceRoot, 'rev-parse', '--verify', 'HEAD^{commit}'], {
|
|
4737
|
+
encoding: 'utf8',
|
|
4738
|
+
env,
|
|
4739
|
+
timeout: 10_000,
|
|
4740
|
+
});
|
|
4741
|
+
const revision = (head.stdout ?? '').trim();
|
|
4742
|
+
if (head.status !== 0 || !/^[a-f0-9]{40}$/i.test(revision)) {
|
|
4743
|
+
throw new Error('RELEASE_SESSION_SOURCE_REVISION_UNAVAILABLE');
|
|
4744
|
+
}
|
|
4745
|
+
return revision;
|
|
4746
|
+
}
|
|
4747
|
+
|
|
4748
|
+
export interface ConfiguredRuntimeReleaseSourceState {
|
|
4749
|
+
configured: boolean;
|
|
4750
|
+
sourceRevision?: string;
|
|
4751
|
+
activeSourceCommit?: string;
|
|
4752
|
+
activeReleaseId?: string;
|
|
4753
|
+
}
|
|
4754
|
+
|
|
4755
|
+
/**
|
|
4756
|
+
* Read the two immutable identities used only to decide whether automatic
|
|
4757
|
+
* release reconciliation is warranted. This does not create or advance a
|
|
4758
|
+
* ReleaseSession and therefore never becomes release intent authority.
|
|
4759
|
+
*/
|
|
4760
|
+
export function configuredRuntimeReleaseSourceState(config: RecoveryConfig): ConfiguredRuntimeReleaseSourceState {
|
|
4761
|
+
const sourceRoot = config.primaryRuntimeSourceRoot?.trim();
|
|
4762
|
+
if (!sourceRoot) return { configured: false };
|
|
4763
|
+
const authority = readRuntimeReleaseAuthority(config.controllerHome);
|
|
4764
|
+
if (!authority) throw new Error('RELEASE_AUTOMATION_ACTIVE_RELEASE_AUTHORITY_UNKNOWN');
|
|
4765
|
+
const manifest = loadRuntimeReleaseManifest(authority.active.manifestPath, config.controllerHome);
|
|
4766
|
+
return {
|
|
4767
|
+
configured: true,
|
|
4768
|
+
sourceRevision: configuredSourceRevision(sourceRoot),
|
|
4769
|
+
activeSourceCommit: manifest.sourceCommit?.trim() || undefined,
|
|
4770
|
+
activeReleaseId: authority.active.releaseId,
|
|
4771
|
+
};
|
|
4772
|
+
}
|
|
4773
|
+
|
|
4774
|
+
async function allocateCandidateLoopbackPort(stablePort: number): Promise<number> {
|
|
4775
|
+
for (let attempt = 0; attempt < 8; attempt += 1) {
|
|
4776
|
+
const port = await new Promise<number>((resolvePort, rejectPort) => {
|
|
4777
|
+
const server = createNetServer();
|
|
4778
|
+
server.unref();
|
|
4779
|
+
server.once('error', rejectPort);
|
|
4780
|
+
server.listen({ host: '127.0.0.1', port: 0, exclusive: true }, () => {
|
|
4781
|
+
const address = server.address();
|
|
4782
|
+
const selected = address && typeof address === 'object' ? address.port : 0;
|
|
4783
|
+
server.close((error) => error ? rejectPort(error) : resolvePort(selected));
|
|
4784
|
+
});
|
|
4785
|
+
});
|
|
4786
|
+
if (port > 0 && port !== stablePort) return port;
|
|
4787
|
+
}
|
|
4788
|
+
throw new Error('RELEASE_SESSION_CANDIDATE_PORT_UNAVAILABLE');
|
|
4789
|
+
}
|
|
4790
|
+
|
|
4791
|
+
function stableReleaseSessionIdentity(config: RecoveryConfig): ReleaseSessionStableRelease {
|
|
4792
|
+
const authority = releaseAuthority(config);
|
|
4793
|
+
if (!authority) throw new Error('RELEASE_SESSION_STABLE_RELEASE_AUTHORITY_REQUIRED');
|
|
4794
|
+
return {
|
|
4795
|
+
authorityRevision: authority.revision,
|
|
4796
|
+
releaseId: authority.active.releaseId,
|
|
4797
|
+
artifactIdentity: authority.active.artifactIdentity,
|
|
4798
|
+
manifestSha256: authority.active.manifestSha256,
|
|
4799
|
+
workerProtocolVersion: authority.active.workerProtocolVersion,
|
|
4800
|
+
releaseFencingTokenSha256: createHash('sha256').update(authority.fencingToken).digest('hex'),
|
|
4801
|
+
};
|
|
4802
|
+
}
|
|
4803
|
+
|
|
4804
|
+
function sameStableReleaseSessionIdentity(left: ReleaseSessionStableRelease, right: ReleaseSessionStableRelease): boolean {
|
|
4805
|
+
return left.authorityRevision === right.authorityRevision
|
|
4806
|
+
&& left.releaseId === right.releaseId
|
|
4807
|
+
&& left.artifactIdentity === right.artifactIdentity
|
|
4808
|
+
&& left.manifestSha256 === right.manifestSha256
|
|
4809
|
+
&& left.workerProtocolVersion === right.workerProtocolVersion
|
|
4810
|
+
&& left.releaseFencingTokenSha256 === right.releaseFencingTokenSha256;
|
|
4811
|
+
}
|
|
4812
|
+
|
|
4813
|
+
function assertStableReleaseSessionIdentityCurrent(config: RecoveryConfig, expected: ReleaseSessionStableRelease): void {
|
|
4814
|
+
const actual = stableReleaseSessionIdentity(config);
|
|
4815
|
+
if (
|
|
4816
|
+
actual.authorityRevision !== expected.authorityRevision
|
|
4817
|
+
|| actual.releaseId !== expected.releaseId
|
|
4818
|
+
|| actual.artifactIdentity !== expected.artifactIdentity
|
|
4819
|
+
|| actual.manifestSha256 !== expected.manifestSha256
|
|
4820
|
+
|| actual.workerProtocolVersion !== expected.workerProtocolVersion
|
|
4821
|
+
|| actual.releaseFencingTokenSha256 !== expected.releaseFencingTokenSha256
|
|
4822
|
+
) throw new Error('RELEASE_SESSION_STABLE_AUTHORITY_CHANGED');
|
|
4823
|
+
}
|
|
4824
|
+
|
|
4825
|
+
function candidateControllerHomeForSession(config: RecoveryConfig, sessionId: string): string {
|
|
4826
|
+
return join(dirname(resolve(config.controllerHome)), 'candidate-runtime-lanes', sessionId);
|
|
4827
|
+
}
|
|
4828
|
+
|
|
4829
|
+
export async function prepareConfiguredRuntimeReleaseSession(
|
|
4830
|
+
config: RecoveryConfig,
|
|
4831
|
+
dependencies: Pick<ConfiguredRuntimeActivationDependencies, 'stage'> = {},
|
|
4832
|
+
requestId?: string,
|
|
4833
|
+
): Promise<ConfiguredRuntimeActivationResult> {
|
|
4834
|
+
const sourceRoot = config.primaryRuntimeSourceRoot?.trim();
|
|
4835
|
+
if (!sourceRoot) return { ok: false, attempted: false, noOp: true, detail: 'primary Runtime source root is not configured in standalone Recovery' };
|
|
4836
|
+
const sourceRepositoryId = config.primaryRuntimeSourceRepositoryId?.trim();
|
|
4837
|
+
if (!sourceRepositoryId) return { ok: false, attempted: false, noOp: true, detail: 'primary Runtime source repository id is not configured in standalone Recovery' };
|
|
4838
|
+
|
|
4839
|
+
const verifiedStable = await verifyStableRuntime(config);
|
|
4840
|
+
if (!verifiedStable.ok) {
|
|
4841
|
+
return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_STABLE_RUNTIME_NOT_VERIFIED' };
|
|
4842
|
+
}
|
|
4843
|
+
|
|
4844
|
+
const locked = await withLock(config, {
|
|
4845
|
+
action: 'release_session_prepare',
|
|
4846
|
+
...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
|
|
4847
|
+
}, async () => {
|
|
4848
|
+
const stable = readStableExecutionLane(config.controllerHome);
|
|
4849
|
+
const stableRelease = stableReleaseSessionIdentity(config);
|
|
4850
|
+
const sourceRevision = configuredSourceRevision(sourceRoot);
|
|
4851
|
+
|
|
4852
|
+
// ReleaseSession owns semantic progression. This Recovery lock serializes only
|
|
4853
|
+
// physical preparation. Resume the exact source_frozen session after an
|
|
4854
|
+
// interruption instead of inventing a second release authority.
|
|
4855
|
+
const inventory = listCompleteReleaseSessions(config.controllerHome);
|
|
4856
|
+
if (inventory.invalidSessionFiles.length > 0) {
|
|
4857
|
+
return {
|
|
4858
|
+
ok: false as const,
|
|
4859
|
+
attempted: false,
|
|
4860
|
+
noOp: true,
|
|
4861
|
+
detail: `RELEASE_SESSION_INVENTORY_INCOMPLETE: invalid=${inventory.invalidSessionFiles.join(',') || 'none'}`,
|
|
4862
|
+
};
|
|
4863
|
+
}
|
|
4864
|
+
await reconcileOrphanCandidateLanes(config, stable, inventory.sessions);
|
|
4865
|
+
let resumableMatchingSession: ReleaseSession | undefined;
|
|
4866
|
+
for (const existing of [...inventory.sessions].sort((left, right) => left.updatedAt.localeCompare(right.updatedAt))) {
|
|
4867
|
+
// Candidate B becomes disposable once the semantic ReleaseSession says it
|
|
4868
|
+
// is retired. Cleanup is deliberately retried here because process death
|
|
4869
|
+
// can happen after the phase CAS but before the physical lane removal. The
|
|
4870
|
+
// remover is session/path fenced and idempotent, so this reconciles leaked
|
|
4871
|
+
// bytes without creating another retention authority.
|
|
4872
|
+
if (releaseSessionCandidateIsRetired(existing)) cleanupRetiredCandidateLane(config, existing);
|
|
4873
|
+
if (existing.phase === 'failed' || existing.phase === 'rolled_back' || existing.phase === 'known_good') continue;
|
|
4874
|
+
const preparationReusablePhase = existing.phase === 'source_frozen'
|
|
4875
|
+
|| existing.phase === 'built'
|
|
4876
|
+
|| existing.phase === 'static_verified'
|
|
4877
|
+
|| existing.phase === 'candidate_booted'
|
|
4878
|
+
|| existing.phase === 'candidate_verified'
|
|
4879
|
+
|| existing.phase === 'cutover_eligible';
|
|
4880
|
+
if (
|
|
4881
|
+
preparationReusablePhase
|
|
4882
|
+
&& existing.sourceRevision === sourceRevision
|
|
4883
|
+
&& sameStableReleaseSessionIdentity(existing.stableRelease, stableRelease)
|
|
4884
|
+
) {
|
|
4885
|
+
if (resumableMatchingSession) {
|
|
4886
|
+
return {
|
|
4887
|
+
ok: false as const,
|
|
4888
|
+
attempted: false,
|
|
4889
|
+
noOp: true,
|
|
4890
|
+
detail: `RELEASE_SESSION_MULTIPLE_ACTIVE: ${resumableMatchingSession.sessionId}:${resumableMatchingSession.phase},${existing.sessionId}:${existing.phase}`,
|
|
4891
|
+
};
|
|
4892
|
+
}
|
|
4893
|
+
resumableMatchingSession = existing;
|
|
4894
|
+
continue;
|
|
4895
|
+
}
|
|
4896
|
+
if (existing.phase === 'cutover_attempting' || existing.phase === 'cutover_committed') {
|
|
4897
|
+
return {
|
|
4898
|
+
ok: false as const,
|
|
4899
|
+
attempted: false,
|
|
4900
|
+
noOp: true,
|
|
4901
|
+
detail: `RELEASE_SESSION_PREPARE_REQUIRES_RECONCILIATION: ${existing.sessionId}:${existing.phase}`,
|
|
4902
|
+
releaseSession: existing,
|
|
4903
|
+
};
|
|
4904
|
+
}
|
|
4905
|
+
if (existing.phase === 'soaking') {
|
|
4906
|
+
const candidateIsCurrentStable = releaseSessionIsSoakingPredecessorOfStable(existing, stableRelease);
|
|
4907
|
+
if (candidateIsCurrentStable && existing.sourceRevision === sourceRevision) {
|
|
4908
|
+
return {
|
|
4909
|
+
ok: false as const,
|
|
4910
|
+
attempted: false,
|
|
4911
|
+
noOp: true,
|
|
4912
|
+
detail: `RELEASE_SESSION_PREPARE_REQUIRES_SOAK_RESOLUTION: ${existing.sessionId}`,
|
|
4913
|
+
releaseSession: existing,
|
|
4914
|
+
};
|
|
4915
|
+
}
|
|
4916
|
+
if (candidateIsCurrentStable) {
|
|
4917
|
+
// A newer source is allowed to prepare against the exact release that
|
|
4918
|
+
// this predecessor already cut over. The predecessor remains soaking
|
|
4919
|
+
// so its rollback authority survives until the successor commits.
|
|
4920
|
+
continue;
|
|
4921
|
+
}
|
|
4922
|
+
if (releaseSessionIsSoakingSupersededByStableAuthority(existing, stableRelease)) {
|
|
4923
|
+
const superseded = advanceReleaseSession({
|
|
4924
|
+
controllerHome: config.controllerHome,
|
|
4925
|
+
sessionId: existing.sessionId,
|
|
4926
|
+
expectedRevision: existing.revision,
|
|
4927
|
+
phase: 'failed',
|
|
4928
|
+
receipts: [{
|
|
4929
|
+
id: 'soak_superseded_by_runtime_authority',
|
|
4930
|
+
kind: 'soak',
|
|
4931
|
+
summary: `Soaking release ${existing.candidateRelease?.releaseId ?? 'unknown'} was superseded by active Runtime authority ${stableRelease.releaseId}@${stableRelease.authorityRevision}`.slice(0, 500),
|
|
4932
|
+
}],
|
|
4933
|
+
});
|
|
4934
|
+
audit(config, 'release_session_soak_superseded_by_runtime_authority', {
|
|
4935
|
+
sessionId: existing.sessionId,
|
|
4936
|
+
priorCutoverAuthorityRevision: existing.transaction?.cutoverAuthorityRevision,
|
|
4937
|
+
activeAuthorityRevision: stableRelease.authorityRevision,
|
|
4938
|
+
activeReleaseId: stableRelease.releaseId,
|
|
4939
|
+
});
|
|
4940
|
+
cleanupRetiredCandidateLane(config, superseded);
|
|
4941
|
+
continue;
|
|
4942
|
+
}
|
|
4943
|
+
return {
|
|
4944
|
+
ok: false as const,
|
|
4945
|
+
attempted: false,
|
|
4946
|
+
noOp: true,
|
|
4947
|
+
detail: `RELEASE_SESSION_SOAK_RUNTIME_IDENTITY_MISMATCH: ${existing.sessionId}`,
|
|
4948
|
+
releaseSession: existing,
|
|
4949
|
+
};
|
|
4950
|
+
}
|
|
4951
|
+
const superseded = await cancelReleaseSessionUnderLock(
|
|
4952
|
+
config,
|
|
4953
|
+
existing,
|
|
4954
|
+
'superseded by a newer ReleaseSession source freeze',
|
|
4955
|
+
);
|
|
4956
|
+
if (!superseded.ok) return superseded;
|
|
4957
|
+
}
|
|
4958
|
+
|
|
4959
|
+
if (resumableMatchingSession && resumableMatchingSession.phase !== 'source_frozen') {
|
|
4960
|
+
return {
|
|
4961
|
+
ok: true as const,
|
|
4962
|
+
attempted: false,
|
|
4963
|
+
noOp: true,
|
|
4964
|
+
detail: `matching ReleaseSession ${resumableMatchingSession.sessionId} already advanced to ${resumableMatchingSession.phase}; preparation is idempotent and must not supersede it`,
|
|
4965
|
+
releaseSession: resumableMatchingSession,
|
|
4966
|
+
};
|
|
4967
|
+
}
|
|
4968
|
+
|
|
4969
|
+
// Reconcile and reclaim only bytes that no current rollback authority needs.
|
|
4970
|
+
// This is one bounded lifecycle attempt, not a second GC owner, and it
|
|
4971
|
+
// retains fail-closed behavior whenever an authority cannot be proven.
|
|
4972
|
+
reconcileKnownGoodRecoveryRetention(config);
|
|
4973
|
+
const releaseRetention = cleanupControllerReleaseHistory(config.controllerHome, {
|
|
4974
|
+
graceMs: 0,
|
|
4975
|
+
stagingGraceMs: 0,
|
|
4976
|
+
maxRemovals: 16,
|
|
4977
|
+
});
|
|
4978
|
+
if (releaseRetention.errors.length > 0) {
|
|
4979
|
+
throw new Error(`RELEASE_SESSION_RETENTION_RECONCILIATION_FAILED: ${releaseRetention.errors.slice(0, 3).join('; ')}`);
|
|
4980
|
+
}
|
|
4981
|
+
|
|
4982
|
+
// Capacity is operation-specific: the candidate is a measured copy of the
|
|
4983
|
+
// current immutable Runtime tree plus a consistent SQLite snapshot. The
|
|
4984
|
+
// global warning threshold is observability, not rollback reserve; existing
|
|
4985
|
+
// active/previous authorities are already allocated and protected above.
|
|
4986
|
+
const candidateRoot = join(dirname(resolve(config.controllerHome)), 'candidate-runtime-lanes');
|
|
4987
|
+
const storageEstimate = estimateReleaseSessionCandidateBytes(config, releaseAuthority(config)!.active.manifestPath);
|
|
4988
|
+
assertStorageHeadroom(candidateRoot, {
|
|
4989
|
+
operation: 'release_session_prepare',
|
|
4990
|
+
requiredBytes: storageEstimate.requiredBytes,
|
|
4991
|
+
reserveBytes: storageEstimate.reserveBytes,
|
|
4992
|
+
});
|
|
4993
|
+
|
|
4994
|
+
let session: ReleaseSession;
|
|
4995
|
+
let candidateLane: ReturnType<typeof createCandidateExecutionLane>['candidate'];
|
|
4996
|
+
if (resumableMatchingSession) {
|
|
4997
|
+
session = resumableMatchingSession;
|
|
4998
|
+
candidateLane = session.candidate;
|
|
4999
|
+
} else {
|
|
5000
|
+
const nextSessionId = `release-${Date.now()}-${randomUUID().replaceAll('-', '').slice(0, 16)}`;
|
|
5001
|
+
const candidatePort = await allocateCandidateLoopbackPort(stable.port);
|
|
5002
|
+
candidateLane = createCandidateExecutionLane({
|
|
5003
|
+
stableControllerHome: stable.controllerHome,
|
|
5004
|
+
candidateControllerHome: candidateControllerHomeForSession(config, nextSessionId),
|
|
5005
|
+
candidatePort,
|
|
5006
|
+
sessionId: nextSessionId,
|
|
5007
|
+
}).candidate;
|
|
5008
|
+
session = createReleaseSession({
|
|
5009
|
+
controllerHome: config.controllerHome,
|
|
5010
|
+
sessionId: nextSessionId,
|
|
5011
|
+
stable,
|
|
5012
|
+
stableRelease,
|
|
5013
|
+
candidate: candidateLane,
|
|
5014
|
+
sourceRevision,
|
|
5015
|
+
});
|
|
5016
|
+
}
|
|
5017
|
+
const sessionId = session.sessionId;
|
|
5018
|
+
|
|
5019
|
+
try {
|
|
5020
|
+
assertStableReleaseSessionIdentityCurrent(config, stableRelease);
|
|
5021
|
+
const staged = withRuntimeReleaseSourceSnapshot({ sourceRoot, sourceRevision }, frozenSourceRoot =>
|
|
5022
|
+
(dependencies.stage ?? stageRuntimeReleaseFromCandidateSource)({
|
|
5023
|
+
controllerHome: candidateLane.controllerHome,
|
|
5024
|
+
sourceRoot: frozenSourceRoot,
|
|
5025
|
+
dependencyRoot: sourceRoot,
|
|
5026
|
+
sourceRepositoryId,
|
|
5027
|
+
}));
|
|
5028
|
+
assertRuntimeReleaseFiles(staged);
|
|
5029
|
+
if (staged.sourceCommit !== sourceRevision) throw new Error('RELEASE_SESSION_SOURCE_CHANGED_DURING_BUILD');
|
|
5030
|
+
const candidateRelease: ReleaseSessionCandidateRelease = {
|
|
5031
|
+
releaseId: staged.releaseId,
|
|
5032
|
+
manifestPath: staged.manifestPath,
|
|
5033
|
+
artifactIdentity: staged.artifactIdentity,
|
|
5034
|
+
manifestSha256: staged.manifestSha256,
|
|
5035
|
+
treeSha256: runtimeReleaseTreeSha256(staged.releasePath),
|
|
5036
|
+
sourceCommit: staged.sourceCommit,
|
|
5037
|
+
...(staged.sourceRepositoryId ? { sourceRepositoryId: staged.sourceRepositoryId } : {}),
|
|
5038
|
+
};
|
|
5039
|
+
assertStableReleaseSessionIdentityCurrent(config, stableRelease);
|
|
5040
|
+
session = advanceReleaseSession({
|
|
5041
|
+
controllerHome: config.controllerHome,
|
|
5042
|
+
sessionId,
|
|
5043
|
+
expectedRevision: session.revision,
|
|
5044
|
+
phase: 'built',
|
|
5045
|
+
candidateRelease,
|
|
5046
|
+
receipts: [{
|
|
5047
|
+
id: 'build',
|
|
5048
|
+
kind: 'build',
|
|
5049
|
+
summary: `portable candidate ${staged.releaseId} built under isolated Candidate B; Stable A unchanged`,
|
|
5050
|
+
}],
|
|
5051
|
+
});
|
|
5052
|
+
audit(config, 'release_session_candidate_built', {
|
|
5053
|
+
sessionId,
|
|
5054
|
+
sourceRevision,
|
|
5055
|
+
stableReleaseId: stableRelease.releaseId,
|
|
5056
|
+
stableAuthorityRevision: stableRelease.authorityRevision,
|
|
5057
|
+
candidateControllerHome: candidateLane.controllerHome,
|
|
5058
|
+
candidatePort: candidateLane.port,
|
|
5059
|
+
candidateReleaseId: staged.releaseId,
|
|
5060
|
+
candidateTreeSha256: candidateRelease.treeSha256,
|
|
5061
|
+
});
|
|
5062
|
+
return {
|
|
5063
|
+
ok: true as const,
|
|
5064
|
+
attempted: true,
|
|
5065
|
+
detail: 'configured Runtime source was built into an isolated Candidate B ReleaseSession; Stable A was not stopped or activated',
|
|
5066
|
+
staged,
|
|
5067
|
+
releaseSession: session,
|
|
5068
|
+
};
|
|
5069
|
+
} catch (error) {
|
|
5070
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
5071
|
+
try {
|
|
5072
|
+
session = advanceReleaseSession({
|
|
5073
|
+
controllerHome: config.controllerHome,
|
|
5074
|
+
sessionId,
|
|
5075
|
+
expectedRevision: session.revision,
|
|
5076
|
+
phase: 'failed',
|
|
5077
|
+
receipts: [{ id: 'build_failed', kind: 'build', summary: detail.slice(0, 500) }],
|
|
5078
|
+
});
|
|
5079
|
+
} catch { /* Preserve the original build failure. */ }
|
|
5080
|
+
cleanupRetiredCandidateLane(config, session);
|
|
5081
|
+
audit(config, 'release_session_candidate_build_failed', { sessionId, sourceRevision, detail });
|
|
5082
|
+
return { ok: false as const, attempted: true, detail, releaseSession: session };
|
|
5083
|
+
}
|
|
5084
|
+
});
|
|
5085
|
+
|
|
5086
|
+
if (!locked.acquired) {
|
|
5087
|
+
return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
|
|
5088
|
+
}
|
|
5089
|
+
return locked.value;
|
|
5090
|
+
}
|
|
5091
|
+
|
|
5092
|
+
|
|
5093
|
+
const RELEASE_SESSION_STATIC_GATES = [
|
|
5094
|
+
{ id: 'type', args: ['run', 'check:type'], timeoutMs: 10 * 60_000 },
|
|
5095
|
+
{ id: 'runtime_architecture', args: ['run', 'check:runtime-architecture'], timeoutMs: 5 * 60_000 },
|
|
5096
|
+
{ id: 'architecture_sync', args: ['run', 'check:architecture-sync'], timeoutMs: 5 * 60_000 },
|
|
5097
|
+
{ id: 'bootstrap', args: ['run', 'check:bootstrap-files'], timeoutMs: 5 * 60_000 },
|
|
5098
|
+
] as const;
|
|
5099
|
+
|
|
5100
|
+
export async function verifyConfiguredRuntimeReleaseSessionStaticGates(
|
|
5101
|
+
config: RecoveryConfig,
|
|
5102
|
+
sessionId: string,
|
|
5103
|
+
requestId?: string,
|
|
5104
|
+
): Promise<ConfiguredRuntimeActivationResult> {
|
|
5105
|
+
const sourceRoot = config.primaryRuntimeSourceRoot?.trim();
|
|
5106
|
+
if (!sourceRoot) return { ok: false, attempted: false, noOp: true, detail: 'primary Runtime source root is not configured in standalone Recovery' };
|
|
5107
|
+
const locked = await withLock(config, {
|
|
5108
|
+
action: 'release_session_static_verify',
|
|
5109
|
+
...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
|
|
5110
|
+
}, async () => {
|
|
5111
|
+
let session = readReleaseSession(config.controllerHome, sessionId);
|
|
5112
|
+
if (!session) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
|
|
5113
|
+
if (session.phase !== 'built') {
|
|
5114
|
+
return { ok: false as const, attempted: false, noOp: true, detail: `RELEASE_SESSION_STATIC_VERIFY_REQUIRES_BUILT: ${session.phase}`, releaseSession: session };
|
|
5115
|
+
}
|
|
5116
|
+
const candidate = session.candidateRelease;
|
|
5117
|
+
if (!candidate) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED', releaseSession: session };
|
|
5118
|
+
const frozenSourceRevision = session.sourceRevision;
|
|
5119
|
+
const stableRelease = session.stableRelease;
|
|
5120
|
+
try {
|
|
5121
|
+
assertStableReleaseSessionIdentityCurrent(config, stableRelease);
|
|
5122
|
+
const manifest = loadRuntimeReleaseManifest(candidate.manifestPath, session.candidate.controllerHome);
|
|
5123
|
+
if (
|
|
5124
|
+
manifest.deploymentScope !== 'portable'
|
|
5125
|
+
|| manifest.releaseId !== candidate.releaseId
|
|
5126
|
+
|| manifest.artifactIdentity !== candidate.artifactIdentity
|
|
5127
|
+
|| createHash('sha256').update(readFileSync(candidate.manifestPath)).digest('hex') !== candidate.manifestSha256
|
|
5128
|
+
|| runtimeReleaseTreeSha256(dirname(candidate.manifestPath)) !== candidate.treeSha256
|
|
5129
|
+
) throw new Error('RELEASE_SESSION_CANDIDATE_IDENTITY_MISMATCH');
|
|
5130
|
+
|
|
5131
|
+
const receipts = withRuntimeReleaseSourceSnapshot({
|
|
5132
|
+
sourceRoot,
|
|
5133
|
+
sourceRevision: frozenSourceRevision,
|
|
5134
|
+
}, frozenSourceRoot => {
|
|
5135
|
+
const gateReceipts: Array<{ id: string; kind: 'static_gate'; summary: string }> = [];
|
|
5136
|
+
for (const gate of RELEASE_SESSION_STATIC_GATES) {
|
|
5137
|
+
assertStableReleaseSessionIdentityCurrent(config, stableRelease);
|
|
5138
|
+
const startedAt = Date.now();
|
|
5139
|
+
const bunExecutable = resolveBunExecutable();
|
|
5140
|
+
const boundedRunner = join(frozenSourceRoot, 'scripts', 'run-bounded-command.mjs');
|
|
5141
|
+
const result = spawnSync(bunExecutable, [
|
|
5142
|
+
boundedRunner,
|
|
5143
|
+
'--timeout-ms', String(gate.timeoutMs),
|
|
5144
|
+
'--',
|
|
5145
|
+
bunExecutable,
|
|
5146
|
+
...gate.args,
|
|
5147
|
+
], {
|
|
5148
|
+
cwd: frozenSourceRoot,
|
|
5149
|
+
env: { ...runtimeAuthorityFreeEnvironment(process.env), PATH: recoveryCommandPath() },
|
|
5150
|
+
encoding: 'utf8',
|
|
5151
|
+
// The inner bounded runner owns the command process group and gets a
|
|
5152
|
+
// brief escalation window to terminate descendants before this
|
|
5153
|
+
// outer Recovery watchdog is allowed to intervene.
|
|
5154
|
+
timeout: gate.timeoutMs + 15_000,
|
|
5155
|
+
maxBuffer: 8 * 1024 * 1024,
|
|
5156
|
+
});
|
|
5157
|
+
const durationMs = Date.now() - startedAt;
|
|
5158
|
+
if (result.error || result.status !== 0) {
|
|
5159
|
+
const detail = result.error instanceof Error
|
|
5160
|
+
? result.error.message
|
|
5161
|
+
: (result.stderr || result.stdout || `exit ${result.status ?? 'unknown'}`).trim().slice(-2000);
|
|
5162
|
+
throw new Error(`RELEASE_SESSION_STATIC_GATE_FAILED:${gate.id}: ${detail}`);
|
|
5163
|
+
}
|
|
5164
|
+
gateReceipts.push({
|
|
5165
|
+
id: gate.id,
|
|
5166
|
+
kind: 'static_gate',
|
|
5167
|
+
summary: `${gate.id} passed on frozen source ${frozenSourceRevision} in ${durationMs}ms`,
|
|
5168
|
+
});
|
|
5169
|
+
}
|
|
5170
|
+
return gateReceipts;
|
|
5171
|
+
});
|
|
5172
|
+
assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
|
|
5173
|
+
session = advanceReleaseSession({
|
|
5174
|
+
controllerHome: config.controllerHome,
|
|
5175
|
+
sessionId,
|
|
5176
|
+
expectedRevision: session.revision,
|
|
5177
|
+
phase: 'static_verified',
|
|
5178
|
+
receipts,
|
|
5179
|
+
});
|
|
5180
|
+
audit(config, 'release_session_static_verified', {
|
|
5181
|
+
sessionId,
|
|
5182
|
+
sourceRevision: session.sourceRevision,
|
|
5183
|
+
candidateReleaseId: candidate.releaseId,
|
|
5184
|
+
gates: RELEASE_SESSION_STATIC_GATES.map((gate) => gate.id),
|
|
5185
|
+
});
|
|
5186
|
+
return {
|
|
5187
|
+
ok: true as const,
|
|
5188
|
+
attempted: true,
|
|
5189
|
+
detail: 'ReleaseSession static gates passed on the frozen source revision; Stable A remained unchanged',
|
|
5190
|
+
releaseSession: session,
|
|
5191
|
+
};
|
|
5192
|
+
} catch (error) {
|
|
5193
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
5194
|
+
try {
|
|
5195
|
+
session = advanceReleaseSession({
|
|
5196
|
+
controllerHome: config.controllerHome,
|
|
5197
|
+
sessionId,
|
|
5198
|
+
expectedRevision: session.revision,
|
|
5199
|
+
phase: 'failed',
|
|
5200
|
+
receipts: [{ id: 'static_failed', kind: 'static_gate', summary: detail.slice(0, 500) }],
|
|
5201
|
+
});
|
|
5202
|
+
} catch { /* preserve the original static verification failure */ }
|
|
5203
|
+
cleanupRetiredCandidateLane(config, session);
|
|
5204
|
+
audit(config, 'release_session_static_failed', { sessionId, detail });
|
|
5205
|
+
return { ok: false as const, attempted: true, detail, releaseSession: session };
|
|
5206
|
+
}
|
|
5207
|
+
});
|
|
5208
|
+
if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
|
|
5209
|
+
return locked.value;
|
|
5210
|
+
}
|
|
5211
|
+
|
|
5212
|
+
|
|
5213
|
+
const RELEASE_SESSION_CANDIDATE_VERIFY_TIMEOUT_MS = 60_000;
|
|
5214
|
+
|
|
5215
|
+
function candidateRecoveryConfig(session: ReleaseSession): RecoveryConfig {
|
|
5216
|
+
return {
|
|
5217
|
+
schemaVersion: 1,
|
|
5218
|
+
controllerHome: session.candidate.controllerHome,
|
|
5219
|
+
installProfile: 'manual',
|
|
5220
|
+
primaryRuntimeService: {
|
|
5221
|
+
...defaultPrimaryRuntimeServiceConfig(),
|
|
5222
|
+
postRestartVerifyTimeoutMs: RELEASE_SESSION_CANDIDATE_VERIFY_TIMEOUT_MS,
|
|
5223
|
+
},
|
|
5224
|
+
mainMcpTokenFile: session.candidate.authTokenFile,
|
|
5225
|
+
readOnlyTool: {
|
|
5226
|
+
name: STABLE_RECOVERY_READ_ONLY_TOOL.name,
|
|
5227
|
+
arguments: { ...STABLE_RECOVERY_READ_ONLY_TOOL.arguments },
|
|
5228
|
+
},
|
|
5229
|
+
};
|
|
5230
|
+
}
|
|
5231
|
+
|
|
5232
|
+
async function installReleaseSessionCandidateService(
|
|
5233
|
+
session: ReleaseSession,
|
|
5234
|
+
runCommand: CommandRunner = command,
|
|
5235
|
+
): Promise<void> {
|
|
5236
|
+
const candidateRelease = session.candidateRelease;
|
|
5237
|
+
if (!candidateRelease) throw new Error('RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED');
|
|
5238
|
+
const home = session.candidate.controllerHome;
|
|
5239
|
+
const serviceConfig = readForgeRuntimeServiceConfig(forgeRuntimeServicePaths(home).configPath);
|
|
5240
|
+
if (
|
|
5241
|
+
resolve(serviceConfig.controllerHome) !== resolve(home)
|
|
5242
|
+
|| serviceConfig.port !== session.candidate.port
|
|
5243
|
+
|| resolve(serviceConfig.authTokenFile) !== resolve(session.candidate.authTokenFile)
|
|
5244
|
+
) throw new Error('RELEASE_SESSION_CANDIDATE_SERVICE_CONTRACT_MISMATCH');
|
|
5245
|
+
|
|
5246
|
+
const operationId = `release-session-candidate-${session.sessionId}`;
|
|
5247
|
+
const authority = publishRuntimeRelease(home, candidateRelease.manifestPath, operationId);
|
|
5248
|
+
if (
|
|
5249
|
+
authority.active.releaseId !== candidateRelease.releaseId
|
|
5250
|
+
|| authority.active.artifactIdentity !== candidateRelease.artifactIdentity
|
|
5251
|
+
|| authority.active.manifestSha256 !== candidateRelease.manifestSha256
|
|
5252
|
+
) throw new Error('RELEASE_SESSION_CANDIDATE_AUTHORITY_MISMATCH');
|
|
5253
|
+
syncForgeRuntimeActiveEntrypoint(home);
|
|
5254
|
+
|
|
5255
|
+
if (process.platform === 'darwin') {
|
|
5256
|
+
await installForgeRuntimeService({
|
|
5257
|
+
config: serviceConfig,
|
|
5258
|
+
// Ignored in release mode; a non-empty bounded path keeps bootstrap fallback explicit.
|
|
5259
|
+
runnerPath: candidateRelease.manifestPath,
|
|
5260
|
+
});
|
|
5261
|
+
return;
|
|
5262
|
+
}
|
|
5263
|
+
if (process.platform === 'linux') {
|
|
5264
|
+
const env = runtimeAuthorityFreeEnvironment(process.env);
|
|
5265
|
+
writePackageRuntimeSystemdUserService(home, env);
|
|
5266
|
+
for (const argv of systemdRuntimeInstallCommands(forgeRuntimeServicePaths(home).label)) {
|
|
5267
|
+
const [executable, ...args] = argv;
|
|
5268
|
+
if (!executable) continue;
|
|
5269
|
+
const result = await runCommand(executable, args, 20_000);
|
|
5270
|
+
if (!result.ok) throw new Error(`RELEASE_SESSION_CANDIDATE_SYSTEMD_INSTALL_FAILED: ${result.stderr || result.stdout || executable}`);
|
|
5271
|
+
}
|
|
5272
|
+
return;
|
|
5273
|
+
}
|
|
5274
|
+
throw new Error(`RELEASE_SESSION_CANDIDATE_SERVICE_PLATFORM_UNSUPPORTED: ${process.platform}`);
|
|
5275
|
+
}
|
|
5276
|
+
|
|
5277
|
+
interface ReleaseSessionCandidateRetirement {
|
|
5278
|
+
ok: boolean;
|
|
5279
|
+
detail: string;
|
|
5280
|
+
}
|
|
5281
|
+
|
|
5282
|
+
function cleanupRetiredCandidateLane(config: RecoveryConfig, session: ReleaseSession): ReleaseSessionCandidateRetirement {
|
|
5283
|
+
try {
|
|
5284
|
+
const removed = removeRetiredCandidateExecutionLane(session.stable, session.candidate);
|
|
5285
|
+
if (removed) {
|
|
5286
|
+
audit(config, 'release_session_candidate_lane_cleaned', {
|
|
5287
|
+
sessionId: session.sessionId,
|
|
5288
|
+
candidateControllerHome: session.candidate.controllerHome,
|
|
5289
|
+
phase: session.phase,
|
|
5290
|
+
});
|
|
5291
|
+
}
|
|
5292
|
+
return {
|
|
5293
|
+
ok: true,
|
|
5294
|
+
detail: removed
|
|
5295
|
+
? 'Candidate B Controller Home was removed after terminal ReleaseSession cleanup'
|
|
5296
|
+
: 'Candidate B Controller Home was already absent after terminal ReleaseSession cleanup',
|
|
5297
|
+
};
|
|
5298
|
+
} catch (error) {
|
|
5299
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
5300
|
+
audit(config, 'release_session_candidate_lane_cleanup_failed', {
|
|
5301
|
+
sessionId: session.sessionId,
|
|
5302
|
+
candidateControllerHome: session.candidate.controllerHome,
|
|
5303
|
+
phase: session.phase,
|
|
5304
|
+
detail,
|
|
5305
|
+
});
|
|
5306
|
+
return { ok: false, detail };
|
|
5307
|
+
}
|
|
5308
|
+
}
|
|
5309
|
+
|
|
5310
|
+
async function stopCandidateServiceAtHome(
|
|
5311
|
+
home: string,
|
|
5312
|
+
runCommand: CommandRunner = command,
|
|
5313
|
+
): Promise<ReleaseSessionCandidateRetirement> {
|
|
5314
|
+
try {
|
|
5315
|
+
if (process.platform === 'darwin') {
|
|
5316
|
+
try {
|
|
5317
|
+
await uninstallForgeRuntimeService(home);
|
|
5318
|
+
} catch (error) {
|
|
5319
|
+
const uid = await currentUid();
|
|
5320
|
+
if (uid === undefined) {
|
|
5321
|
+
return { ok: false, detail: error instanceof Error ? error.message : String(error) };
|
|
5322
|
+
}
|
|
5323
|
+
const paths = forgeRuntimeServicePaths(home);
|
|
5324
|
+
const loaded = await runCommand('launchctl', ['print', `gui/${uid}/${paths.label}`], 5_000);
|
|
5325
|
+
if (loaded.ok) {
|
|
5326
|
+
return { ok: false, detail: error instanceof Error ? error.message : String(error) };
|
|
5327
|
+
}
|
|
5328
|
+
// An already-absent launchd job is an idempotent retirement. Remove only
|
|
5329
|
+
// its stale installed declaration, then verify the Runtime itself below.
|
|
5330
|
+
rmSync(paths.installedPlistPath, { force: true });
|
|
5331
|
+
}
|
|
5332
|
+
} else if (process.platform === 'linux') {
|
|
5333
|
+
const label = forgeRuntimeServicePaths(home).label;
|
|
5334
|
+
const unitName = systemdUserUnitName(label);
|
|
5335
|
+
const stopped = await runCommand('systemctl', ['--user', 'disable', '--now', unitName], 20_000);
|
|
5336
|
+
const stoppedDetail = `${stopped.stderr}\n${stopped.stdout}`;
|
|
5337
|
+
if (!stopped.ok && !/not loaded|not found|does not exist|no such file/i.test(stoppedDetail)) {
|
|
5338
|
+
return { ok: false, detail: `RELEASE_SESSION_CANDIDATE_SYSTEMD_STOP_FAILED: ${stoppedDetail.trim() || stopped.status}` };
|
|
5339
|
+
}
|
|
5340
|
+
rmSync(systemdUserUnitPath(unitName), { force: true });
|
|
5341
|
+
const reloaded = await runCommand('systemctl', ['--user', 'daemon-reload'], 20_000);
|
|
5342
|
+
if (!reloaded.ok) {
|
|
5343
|
+
return { ok: false, detail: `RELEASE_SESSION_CANDIDATE_SYSTEMD_RELOAD_FAILED: ${reloaded.stderr || reloaded.stdout || reloaded.status}` };
|
|
5344
|
+
}
|
|
5345
|
+
} else {
|
|
5346
|
+
return { ok: false, detail: `RELEASE_SESSION_CANDIDATE_SERVICE_PLATFORM_UNSUPPORTED: ${process.platform}` };
|
|
5347
|
+
}
|
|
5348
|
+
} catch (error) {
|
|
5349
|
+
return { ok: false, detail: error instanceof Error ? error.message : String(error) };
|
|
5350
|
+
}
|
|
5351
|
+
|
|
5352
|
+
const observation = observeRuntimeStatus(home);
|
|
5353
|
+
if (observation.running) {
|
|
5354
|
+
return { ok: false, detail: 'RELEASE_SESSION_CANDIDATE_STILL_RUNNING_AFTER_RETIREMENT' };
|
|
5355
|
+
}
|
|
5356
|
+
return { ok: true, detail: 'Candidate B persistent service is absent and its Runtime process is not running' };
|
|
5357
|
+
}
|
|
5358
|
+
|
|
5359
|
+
async function stopReleaseSessionCandidateService(
|
|
5360
|
+
session: ReleaseSession,
|
|
5361
|
+
runCommand: CommandRunner = command,
|
|
5362
|
+
): Promise<ReleaseSessionCandidateRetirement> {
|
|
5363
|
+
return stopCandidateServiceAtHome(session.candidate.controllerHome, runCommand);
|
|
5364
|
+
}
|
|
5365
|
+
|
|
5366
|
+
async function reconcileOrphanCandidateLanes(
|
|
5367
|
+
config: RecoveryConfig,
|
|
5368
|
+
stable: ReturnType<typeof readStableExecutionLane>,
|
|
5369
|
+
sessions: ReleaseSession[],
|
|
5370
|
+
): Promise<void> {
|
|
5371
|
+
const candidateRoot = resolve(dirname(resolve(config.controllerHome)), 'candidate-runtime-lanes');
|
|
5372
|
+
if (!existsSync(candidateRoot)) return;
|
|
5373
|
+
const knownHomes = new Set(sessions.map((session) => resolve(session.candidate.controllerHome)));
|
|
5374
|
+
const entries = readdirSync(candidateRoot, { withFileTypes: true })
|
|
5375
|
+
.filter((entry) => entry.isDirectory())
|
|
5376
|
+
.sort((left, right) => left.name.localeCompare(right.name));
|
|
5377
|
+
if (entries.length > 512) throw new Error(`RELEASE_SESSION_CANDIDATE_LANE_INVENTORY_INCOMPLETE: ${entries.length}`);
|
|
5378
|
+
|
|
5379
|
+
for (const entry of entries) {
|
|
5380
|
+
const candidateHome = resolve(candidateRoot, entry.name);
|
|
5381
|
+
if (knownHomes.has(candidateHome)) continue;
|
|
5382
|
+
try {
|
|
5383
|
+
// Reuse ReleaseSession's own id parser as the identity fence. A directory
|
|
5384
|
+
// whose name is not a valid session id is visible debt, not delete authority.
|
|
5385
|
+
if (readReleaseSession(config.controllerHome, entry.name)) continue;
|
|
5386
|
+
} catch (error) {
|
|
5387
|
+
audit(config, 'release_session_orphan_candidate_identity_invalid', {
|
|
5388
|
+
candidateControllerHome: candidateHome,
|
|
5389
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
5390
|
+
});
|
|
5391
|
+
continue;
|
|
5392
|
+
}
|
|
5393
|
+
|
|
5394
|
+
let candidateConfig;
|
|
5395
|
+
try {
|
|
5396
|
+
candidateConfig = readForgeRuntimeServiceConfig(forgeRuntimeServicePaths(candidateHome).configPath, { missingRepositoryRoot: 'omit' });
|
|
5397
|
+
} catch (error) {
|
|
5398
|
+
audit(config, 'release_session_orphan_candidate_service_contract_unproven', {
|
|
5399
|
+
sessionId: entry.name,
|
|
5400
|
+
candidateControllerHome: candidateHome,
|
|
5401
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
5402
|
+
});
|
|
5403
|
+
continue;
|
|
5404
|
+
}
|
|
5405
|
+
if (resolve(candidateConfig.controllerHome) !== candidateHome || candidateConfig.port === stable.port) {
|
|
5406
|
+
audit(config, 'release_session_orphan_candidate_service_contract_mismatch', {
|
|
5407
|
+
sessionId: entry.name,
|
|
5408
|
+
candidateControllerHome: candidateHome,
|
|
5409
|
+
configuredControllerHome: candidateConfig.controllerHome,
|
|
5410
|
+
configuredPort: candidateConfig.port,
|
|
5411
|
+
stablePort: stable.port,
|
|
5412
|
+
});
|
|
5413
|
+
continue;
|
|
5414
|
+
}
|
|
5415
|
+
|
|
5416
|
+
let running = true;
|
|
5417
|
+
try {
|
|
5418
|
+
running = observeRuntimeStatus(candidateHome).running;
|
|
5419
|
+
} catch (error) {
|
|
5420
|
+
audit(config, 'release_session_orphan_candidate_owner_unknown', {
|
|
5421
|
+
sessionId: entry.name,
|
|
5422
|
+
candidateControllerHome: candidateHome,
|
|
5423
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
5424
|
+
});
|
|
5425
|
+
continue;
|
|
5426
|
+
}
|
|
5427
|
+
if (running) {
|
|
5428
|
+
audit(config, 'release_session_orphan_candidate_active_owner', { sessionId: entry.name, candidateControllerHome: candidateHome });
|
|
5429
|
+
continue;
|
|
5430
|
+
}
|
|
5431
|
+
|
|
5432
|
+
const retirement = await stopCandidateServiceAtHome(candidateHome);
|
|
5433
|
+
if (!retirement.ok) {
|
|
5434
|
+
audit(config, 'release_session_orphan_candidate_service_retirement_failed', {
|
|
5435
|
+
sessionId: entry.name,
|
|
5436
|
+
candidateControllerHome: candidateHome,
|
|
5437
|
+
detail: retirement.detail,
|
|
5438
|
+
});
|
|
5439
|
+
continue;
|
|
5440
|
+
}
|
|
5441
|
+
try {
|
|
5442
|
+
const removed = removeRetiredCandidateExecutionLane(stable, { controllerHome: candidateHome, sessionId: entry.name });
|
|
5443
|
+
if (removed) {
|
|
5444
|
+
audit(config, 'release_session_orphan_candidate_lane_cleaned', { sessionId: entry.name, candidateControllerHome: candidateHome });
|
|
5445
|
+
}
|
|
5446
|
+
} catch (error) {
|
|
5447
|
+
audit(config, 'release_session_orphan_candidate_lane_cleanup_failed', {
|
|
5448
|
+
sessionId: entry.name,
|
|
5449
|
+
candidateControllerHome: candidateHome,
|
|
5450
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
5451
|
+
});
|
|
5452
|
+
}
|
|
5453
|
+
}
|
|
5454
|
+
}
|
|
5455
|
+
|
|
5456
|
+
function candidateCanaryReceipts(
|
|
5457
|
+
session: ReleaseSession,
|
|
5458
|
+
verified: VerifyResult,
|
|
5459
|
+
): Array<{ id: string; kind: 'candidate_canary'; summary: string }> {
|
|
5460
|
+
const observation = observeRuntimeStatus(session.candidate.controllerHome);
|
|
5461
|
+
const diagnostics = observation.snapshot?.readiness.diagnostics;
|
|
5462
|
+
const mcpProbeNames = ['mcp_initialize', 'mcp_initialized_notification', 'mcp_tools_list', 'mcp_read_only_call', 'mcp_session_close'];
|
|
5463
|
+
const failedMcp = mcpProbeNames.filter((name) => verified.probes[name]?.ok !== true);
|
|
5464
|
+
if (failedMcp.length > 0) throw new Error(`RELEASE_SESSION_CANDIDATE_MCP_CANARY_FAILED: ${failedMcp.join(',')}`);
|
|
5465
|
+
if (diagnostics?.scheduler.outcome !== 'pass') throw new Error('RELEASE_SESSION_CANDIDATE_SCHEDULER_CANARY_FAILED');
|
|
5466
|
+
if (diagnostics?.database.outcome !== 'pass' || diagnostics?.releaseCoherence.outcome !== 'pass' || diagnostics?.mcpEndToEnd.outcome !== 'pass') {
|
|
5467
|
+
throw new Error('RELEASE_SESSION_CANDIDATE_RUNTIME_DIAGNOSTICS_FAILED');
|
|
5468
|
+
}
|
|
5469
|
+
const supervisorSocket = workflowSupervisorSocketPath(resolveWorkflowSupervisorForgeHome(session.candidate.controllerHome));
|
|
5470
|
+
if (!existsSync(supervisorSocket)) throw new Error('RELEASE_SESSION_CANDIDATE_SUPERVISOR_CANARY_FAILED');
|
|
5471
|
+
return [
|
|
5472
|
+
{ id: 'recovery', kind: 'candidate_canary', summary: 'Candidate B stopped and restarted through Recovery service lifecycle and returned whole-Runtime healthy' },
|
|
5473
|
+
{ id: 'mcp', kind: 'candidate_canary', summary: 'Candidate B MCP initialize/tools-list/read-only-call/session-close passed using the Candidate credential' },
|
|
5474
|
+
{ id: 'scheduler', kind: 'candidate_canary', summary: 'Candidate B scheduler readiness diagnostic passed after recovery restart' },
|
|
5475
|
+
{ id: 'supervisor', kind: 'candidate_canary', summary: `Candidate B Workflow Supervisor socket is live at ${supervisorSocket}` },
|
|
5476
|
+
{ id: 'controller', kind: 'candidate_canary', summary: 'Candidate B controller read-only facade completed through MCP after restart' },
|
|
5477
|
+
];
|
|
5478
|
+
}
|
|
5479
|
+
|
|
5480
|
+
export async function bootAndVerifyConfiguredRuntimeReleaseSessionCandidate(
|
|
5481
|
+
config: RecoveryConfig,
|
|
5482
|
+
sessionId: string,
|
|
5483
|
+
requestId?: string,
|
|
5484
|
+
): Promise<ConfiguredRuntimeActivationResult> {
|
|
5485
|
+
const locked = await withLock(config, {
|
|
5486
|
+
action: 'release_session_candidate_boot',
|
|
5487
|
+
...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
|
|
5488
|
+
}, async () => {
|
|
5489
|
+
let session = readReleaseSession(config.controllerHome, sessionId);
|
|
5490
|
+
if (!session) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
|
|
5491
|
+
if (session.phase === 'candidate_booted') {
|
|
5492
|
+
const retirement = await stopReleaseSessionCandidateService(session);
|
|
5493
|
+
if (!retirement.ok) {
|
|
5494
|
+
return {
|
|
5495
|
+
ok: false as const,
|
|
5496
|
+
attempted: true,
|
|
5497
|
+
detail: `Candidate B boot/restart canary was interrupted and retirement failed: ${retirement.detail}`,
|
|
5498
|
+
releaseSession: session,
|
|
5499
|
+
};
|
|
5500
|
+
}
|
|
5501
|
+
session = advanceReleaseSession({
|
|
5502
|
+
controllerHome: config.controllerHome,
|
|
5503
|
+
sessionId,
|
|
5504
|
+
expectedRevision: session.revision,
|
|
5505
|
+
phase: 'failed',
|
|
5506
|
+
receipts: [{
|
|
5507
|
+
id: 'candidate_failed',
|
|
5508
|
+
kind: 'candidate_canary',
|
|
5509
|
+
summary: 'Candidate B boot/restart canary was interrupted; Candidate B was retired before terminalizing the ReleaseSession',
|
|
5510
|
+
}],
|
|
5511
|
+
});
|
|
5512
|
+
cleanupRetiredCandidateLane(config, session);
|
|
5513
|
+
return {
|
|
5514
|
+
ok: false as const,
|
|
5515
|
+
attempted: true,
|
|
5516
|
+
detail: 'Candidate B boot/restart canary was interrupted; Candidate B is retired and the ReleaseSession is terminal failed',
|
|
5517
|
+
releaseSession: session,
|
|
5518
|
+
};
|
|
5519
|
+
}
|
|
5520
|
+
if (session.phase !== 'static_verified') {
|
|
5521
|
+
return { ok: false as const, attempted: false, noOp: true, detail: `RELEASE_SESSION_CANDIDATE_BOOT_REQUIRES_STATIC_VERIFIED: ${session.phase}`, releaseSession: session };
|
|
5522
|
+
}
|
|
5523
|
+
const candidateRelease = session.candidateRelease;
|
|
5524
|
+
if (!candidateRelease) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED', releaseSession: session };
|
|
5525
|
+
const runCommand = command;
|
|
5526
|
+
const candidateConfig = candidateRecoveryConfig(session);
|
|
5527
|
+
try {
|
|
5528
|
+
assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
|
|
5529
|
+
if (runtimeReleaseTreeSha256(dirname(candidateRelease.manifestPath)) !== candidateRelease.treeSha256) {
|
|
5530
|
+
throw new Error('RELEASE_SESSION_CANDIDATE_TREE_CHANGED');
|
|
5531
|
+
}
|
|
5532
|
+
await installReleaseSessionCandidateService(session, runCommand);
|
|
5533
|
+
|
|
5534
|
+
let initialVerify = await verifyPrimaryRuntimeAfterStart({
|
|
5535
|
+
config: candidateConfig,
|
|
5536
|
+
timeoutMs: RELEASE_SESSION_CANDIDATE_VERIFY_TIMEOUT_MS,
|
|
5537
|
+
now: Date.now,
|
|
5538
|
+
wait: sleep,
|
|
5539
|
+
verifyLocal: verifyLocalRuntime,
|
|
5540
|
+
observeLocal: observePrimaryRuntimeActivationReadiness,
|
|
5541
|
+
});
|
|
5542
|
+
if (!initialVerify.ok) throw new Error(`RELEASE_SESSION_CANDIDATE_BOOT_UNVERIFIED: ${initialVerify.runtime.reasonCodes.join(',')}`);
|
|
5543
|
+
assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
|
|
5544
|
+
session = advanceReleaseSession({
|
|
5545
|
+
controllerHome: config.controllerHome,
|
|
5546
|
+
sessionId,
|
|
5547
|
+
expectedRevision: session.revision,
|
|
5548
|
+
phase: 'candidate_booted',
|
|
5549
|
+
receipts: [{ id: 'candidate_boot', kind: 'candidate_canary', summary: 'Candidate B booted and passed initial whole-Runtime verification while Stable A remained healthy' }],
|
|
5550
|
+
});
|
|
5551
|
+
|
|
5552
|
+
const platform = process.platform;
|
|
5553
|
+
const uid = await currentUid();
|
|
5554
|
+
const service = primaryRuntimeServiceOwner(candidateConfig, platform, uid);
|
|
5555
|
+
if (!service) throw new Error('RELEASE_SESSION_CANDIDATE_SERVICE_OWNER_MISSING');
|
|
5556
|
+
const stopped = await stopPrimaryRuntimeForReleaseTransition({
|
|
5557
|
+
config: candidateConfig,
|
|
5558
|
+
service,
|
|
5559
|
+
now: Date.now,
|
|
5560
|
+
wait: sleep,
|
|
5561
|
+
runCommand,
|
|
5562
|
+
runtimeRunning: (candidate) => observeRuntimeStatus(candidate.controllerHome).running,
|
|
5563
|
+
});
|
|
5564
|
+
if (!stopped.ok) throw new Error(`RELEASE_SESSION_CANDIDATE_RECOVERY_STOP_FAILED: ${stopped.detail}`);
|
|
5565
|
+
const restarted = await restartPrimaryRuntime(candidateConfig);
|
|
5566
|
+
if (!restarted.ok) throw new Error(`RELEASE_SESSION_CANDIDATE_RECOVERY_RESTART_FAILED: ${restarted.detail}`);
|
|
5567
|
+
initialVerify = restarted.verify;
|
|
5568
|
+
const receipts = candidateCanaryReceipts(session, initialVerify);
|
|
5569
|
+
assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
|
|
5570
|
+
session = advanceReleaseSession({
|
|
5571
|
+
controllerHome: config.controllerHome,
|
|
5572
|
+
sessionId,
|
|
5573
|
+
expectedRevision: session.revision,
|
|
5574
|
+
phase: 'candidate_verified',
|
|
5575
|
+
receipts,
|
|
5576
|
+
});
|
|
5577
|
+
session = advanceReleaseSession({
|
|
5578
|
+
controllerHome: config.controllerHome,
|
|
5579
|
+
sessionId,
|
|
5580
|
+
expectedRevision: session.revision,
|
|
5581
|
+
phase: 'cutover_eligible',
|
|
5582
|
+
});
|
|
5583
|
+
audit(config, 'release_session_candidate_verified', {
|
|
5584
|
+
sessionId,
|
|
5585
|
+
candidateReleaseId: candidateRelease.releaseId,
|
|
5586
|
+
candidateControllerHome: session.candidate.controllerHome,
|
|
5587
|
+
candidatePort: session.candidate.port,
|
|
5588
|
+
stableReleaseId: session.stableRelease.releaseId,
|
|
5589
|
+
});
|
|
5590
|
+
return {
|
|
5591
|
+
ok: true as const,
|
|
5592
|
+
attempted: true,
|
|
5593
|
+
detail: 'Candidate B booted in isolation, survived a Recovery restart canary, passed MCP/Scheduler/Supervisor/Controller canaries, and is cutover-eligible; Stable A remains active',
|
|
5594
|
+
releaseSession: session,
|
|
5595
|
+
};
|
|
5596
|
+
} catch (error) {
|
|
5597
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
5598
|
+
const retirement = await stopReleaseSessionCandidateService(session, runCommand);
|
|
5599
|
+
const terminalDetail = retirement.ok
|
|
5600
|
+
? detail
|
|
5601
|
+
: `${detail}; Candidate B retirement failed: ${retirement.detail}`;
|
|
5602
|
+
try {
|
|
5603
|
+
const latest = readReleaseSession(config.controllerHome, sessionId) ?? session;
|
|
5604
|
+
session = latest;
|
|
5605
|
+
if (retirement.ok && !['failed', 'rolled_back', 'known_good'].includes(latest.phase)) {
|
|
5606
|
+
session = advanceReleaseSession({
|
|
5607
|
+
controllerHome: config.controllerHome,
|
|
5608
|
+
sessionId,
|
|
5609
|
+
expectedRevision: latest.revision,
|
|
5610
|
+
phase: 'failed',
|
|
5611
|
+
receipts: [{ id: 'candidate_failed', kind: 'candidate_canary', summary: terminalDetail.slice(0, 500) }],
|
|
5612
|
+
});
|
|
5613
|
+
}
|
|
5614
|
+
if (retirement.ok && ['failed', 'rolled_back'].includes(session.phase)) {
|
|
5615
|
+
cleanupRetiredCandidateLane(config, session);
|
|
5616
|
+
}
|
|
5617
|
+
} catch { /* preserve original Candidate B failure */ }
|
|
5618
|
+
audit(config, 'release_session_candidate_failed', {
|
|
5619
|
+
sessionId,
|
|
5620
|
+
detail,
|
|
5621
|
+
candidateRetired: retirement.ok,
|
|
5622
|
+
candidateRetirementDetail: retirement.detail,
|
|
5623
|
+
});
|
|
5624
|
+
return { ok: false as const, attempted: true, detail: terminalDetail, releaseSession: session };
|
|
5625
|
+
}
|
|
5626
|
+
});
|
|
5627
|
+
if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
|
|
5628
|
+
return locked.value;
|
|
5629
|
+
}
|
|
5630
|
+
|
|
5631
|
+
|
|
5632
|
+
function terminalizeSupersededSoakingPredecessor(config: RecoveryConfig, successor: ReleaseSession): ReleaseSession | undefined {
|
|
5633
|
+
const inventory = listCompleteReleaseSessions(config.controllerHome);
|
|
5634
|
+
if (inventory.invalidSessionFiles.length > 0) {
|
|
5635
|
+
throw new Error(`RELEASE_SESSION_INVENTORY_INCOMPLETE: invalid=${inventory.invalidSessionFiles.join(',') || 'none'}`);
|
|
5636
|
+
}
|
|
5637
|
+
const predecessors = inventory.sessions.filter((existing) =>
|
|
5638
|
+
existing.sessionId !== successor.sessionId
|
|
5639
|
+
&& existing.sourceRevision !== successor.sourceRevision
|
|
5640
|
+
&& releaseSessionIsSoakingPredecessorOfStable(existing, successor.stableRelease));
|
|
5641
|
+
if (predecessors.length > 1) {
|
|
5642
|
+
throw new Error(`RELEASE_SESSION_MULTIPLE_SOAKING_PREDECESSORS: ${predecessors.map((entry) => entry.sessionId).join(',')}`);
|
|
5643
|
+
}
|
|
5644
|
+
const predecessor = predecessors[0];
|
|
5645
|
+
if (!predecessor) return undefined;
|
|
5646
|
+
const terminal = advanceReleaseSession({
|
|
5647
|
+
controllerHome: config.controllerHome,
|
|
5648
|
+
sessionId: predecessor.sessionId,
|
|
5649
|
+
expectedRevision: predecessor.revision,
|
|
5650
|
+
phase: 'failed',
|
|
5651
|
+
receipts: [{
|
|
5652
|
+
id: `successor_cutover:${successor.sessionId}`,
|
|
5653
|
+
kind: 'soak',
|
|
5654
|
+
summary: `superseded without known-good promotion after successor ${successor.sessionId} committed a cutover from Stable A ${successor.stableRelease.releaseId}`,
|
|
5655
|
+
}],
|
|
5656
|
+
});
|
|
5657
|
+
audit(config, 'release_session_soaking_predecessor_superseded', {
|
|
5658
|
+
predecessorSessionId: predecessor.sessionId,
|
|
5659
|
+
predecessorReleaseId: predecessor.candidateRelease?.releaseId,
|
|
5660
|
+
successorSessionId: successor.sessionId,
|
|
5661
|
+
successorReleaseId: successor.candidateRelease?.releaseId,
|
|
5662
|
+
});
|
|
5663
|
+
return terminal;
|
|
5664
|
+
}
|
|
5665
|
+
|
|
5666
|
+
export async function cutoverConfiguredRuntimeReleaseSession(
|
|
5667
|
+
config: RecoveryConfig,
|
|
5668
|
+
sessionId: string,
|
|
5669
|
+
requestId?: string,
|
|
5670
|
+
): Promise<ConfiguredRuntimeActivationResult> {
|
|
5671
|
+
const locked = await withLock(config, {
|
|
5672
|
+
action: 'release_session_cutover',
|
|
5673
|
+
...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
|
|
5674
|
+
}, async (releaseSessionLock) => {
|
|
5675
|
+
const initialSession = readReleaseSession(config.controllerHome, sessionId);
|
|
5676
|
+
if (!initialSession) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
|
|
5677
|
+
let session: ReleaseSession = initialSession;
|
|
5678
|
+
const candidateRelease = session.candidateRelease;
|
|
5679
|
+
if (!candidateRelease) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED', releaseSession: session };
|
|
5680
|
+
|
|
5681
|
+
const reconcileCutoverAttempt = async (
|
|
5682
|
+
reason: string,
|
|
5683
|
+
activation?: RuntimeReleaseActivationResult,
|
|
5684
|
+
): Promise<ConfiguredRuntimeActivationResult> => {
|
|
5685
|
+
// Activation may have durably captured the ReleaseSession rollback
|
|
5686
|
+
// transaction and advanced its CAS revision. Re-read the one semantic
|
|
5687
|
+
// authority before any post-cutover phase transition.
|
|
5688
|
+
session = readReleaseSession(config.controllerHome, sessionId) ?? session;
|
|
5689
|
+
const retirement = await stopReleaseSessionCandidateService(session);
|
|
5690
|
+
if (!retirement.ok) {
|
|
5691
|
+
audit(config, 'release_session_candidate_retirement_failed', {
|
|
5692
|
+
sessionId,
|
|
5693
|
+
phase: session.phase,
|
|
5694
|
+
candidateControllerHome: session.candidate.controllerHome,
|
|
5695
|
+
detail: retirement.detail,
|
|
5696
|
+
reconciliationReason: reason,
|
|
5697
|
+
});
|
|
5698
|
+
return {
|
|
5699
|
+
ok: false as const,
|
|
5700
|
+
attempted: true,
|
|
5701
|
+
detail: `${reason}; Candidate B retirement failed: ${retirement.detail}`,
|
|
5702
|
+
releaseSession: session,
|
|
5703
|
+
...(activation ? { activation } : {}),
|
|
5704
|
+
};
|
|
5705
|
+
}
|
|
5706
|
+
|
|
5707
|
+
const reconciliationTimeoutMs = Math.max(
|
|
5708
|
+
1_000,
|
|
5709
|
+
configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
|
|
5710
|
+
);
|
|
5711
|
+
const reconciliationDeadline = Date.now() + reconciliationTimeoutMs;
|
|
5712
|
+
let stableNow = observeRuntimeStatus(config.controllerHome);
|
|
5713
|
+
const stableIdentity = () => {
|
|
5714
|
+
const liveRuntimeIdentityVerified = stableNow.running && stableNow.ready && !stableNow.stale;
|
|
5715
|
+
return {
|
|
5716
|
+
candidateIsStable: Boolean(
|
|
5717
|
+
liveRuntimeIdentityVerified
|
|
5718
|
+
&& stableNow.snapshot?.releaseId === candidateRelease.releaseId
|
|
5719
|
+
&& stableNow.snapshot?.artifactIdentity === candidateRelease.artifactIdentity,
|
|
5720
|
+
),
|
|
5721
|
+
originalStableRestored: Boolean(
|
|
5722
|
+
liveRuntimeIdentityVerified
|
|
5723
|
+
&& stableNow.snapshot?.releaseId === session.stableRelease.releaseId
|
|
5724
|
+
&& stableNow.snapshot?.artifactIdentity === session.stableRelease.artifactIdentity,
|
|
5725
|
+
),
|
|
5726
|
+
};
|
|
5727
|
+
};
|
|
5728
|
+
let observedIdentity = stableIdentity();
|
|
5729
|
+
while (!observedIdentity.candidateIsStable && !observedIdentity.originalStableRestored && Date.now() < reconciliationDeadline) {
|
|
5730
|
+
await sleep(Math.min(500, Math.max(1, reconciliationDeadline - Date.now())));
|
|
5731
|
+
stableNow = observeRuntimeStatus(config.controllerHome);
|
|
5732
|
+
observedIdentity = stableIdentity();
|
|
5733
|
+
}
|
|
5734
|
+
const { candidateIsStable, originalStableRestored } = observedIdentity;
|
|
5735
|
+
|
|
5736
|
+
if (candidateIsStable) {
|
|
5737
|
+
if (session.phase === 'cutover_attempting') {
|
|
5738
|
+
session = advanceReleaseSession({
|
|
5739
|
+
controllerHome: config.controllerHome,
|
|
5740
|
+
sessionId,
|
|
5741
|
+
expectedRevision: session.revision,
|
|
5742
|
+
phase: 'cutover_committed',
|
|
5743
|
+
receipts: [{
|
|
5744
|
+
id: 'cutover',
|
|
5745
|
+
kind: 'cutover',
|
|
5746
|
+
summary: `Stable A runs verified Candidate B release ${candidateRelease.releaseId}; Candidate B retired before cutover commit`,
|
|
5747
|
+
}],
|
|
5748
|
+
});
|
|
5749
|
+
}
|
|
5750
|
+
// Stable A now owns the successor and its transaction can roll back to
|
|
5751
|
+
// the predecessor release. Only at this durable boundary may the older
|
|
5752
|
+
// soaking session surrender its independent rollback/acceptance path.
|
|
5753
|
+
terminalizeSupersededSoakingPredecessor(config, session);
|
|
5754
|
+
session = advanceReleaseSession({
|
|
5755
|
+
controllerHome: config.controllerHome,
|
|
5756
|
+
sessionId,
|
|
5757
|
+
expectedRevision: session.revision,
|
|
5758
|
+
phase: 'soaking',
|
|
5759
|
+
receipts: [{
|
|
5760
|
+
id: 'soak_started',
|
|
5761
|
+
kind: 'soak',
|
|
5762
|
+
summary: `cutover reconciled at ${new Date().toISOString()}; Candidate B is retired and known-good promotion remains gated on later stable verification`,
|
|
5763
|
+
}],
|
|
5764
|
+
});
|
|
5765
|
+
const cleanup = cleanupRetiredCandidateLane(config, session);
|
|
5766
|
+
audit(config, 'release_session_cutover_reconciled_committed', {
|
|
5767
|
+
sessionId,
|
|
5768
|
+
releaseId: candidateRelease.releaseId,
|
|
5769
|
+
candidateRetired: true,
|
|
5770
|
+
reason,
|
|
5771
|
+
});
|
|
5772
|
+
return {
|
|
5773
|
+
ok: true as const,
|
|
5774
|
+
attempted: true,
|
|
5775
|
+
detail: `ReleaseSession cutover reconciled to committed: Stable A runs the verified candidate artifact and Candidate B is retired${cleanup.ok ? '' : `; Candidate B cleanup failed: ${cleanup.detail}`}`,
|
|
5776
|
+
releaseSession: session,
|
|
5777
|
+
...(activation ? { activation } : {}),
|
|
5778
|
+
};
|
|
5779
|
+
}
|
|
5780
|
+
|
|
5781
|
+
if (originalStableRestored) {
|
|
5782
|
+
session = advanceReleaseSession({
|
|
5783
|
+
controllerHome: config.controllerHome,
|
|
5784
|
+
sessionId,
|
|
5785
|
+
expectedRevision: session.revision,
|
|
5786
|
+
phase: 'rolled_back',
|
|
5787
|
+
receipts: [{
|
|
5788
|
+
id: 'rollback',
|
|
5789
|
+
kind: 'rollback',
|
|
5790
|
+
summary: `cutover did not commit; exact Stable A ${session.stableRelease.releaseId} is active and verified, and Candidate B is retired`,
|
|
5791
|
+
}],
|
|
5792
|
+
});
|
|
5793
|
+
const cleanup = cleanupRetiredCandidateLane(config, session);
|
|
5794
|
+
audit(config, 'release_session_cutover_reconciled_rolled_back', {
|
|
5795
|
+
sessionId,
|
|
5796
|
+
restoredStableReleaseId: session.stableRelease.releaseId,
|
|
5797
|
+
candidateRetired: true,
|
|
5798
|
+
reason,
|
|
5799
|
+
});
|
|
5800
|
+
return {
|
|
5801
|
+
ok: false as const,
|
|
5802
|
+
attempted: true,
|
|
5803
|
+
detail: `ReleaseSession cutover reconciled to rolled_back: exact Stable A is active and Candidate B is retired${cleanup.ok ? '' : `; Candidate B cleanup failed: ${cleanup.detail}`}`,
|
|
5804
|
+
releaseSession: session,
|
|
5805
|
+
...(activation ? { activation } : {}),
|
|
5806
|
+
};
|
|
5807
|
+
}
|
|
5808
|
+
|
|
5809
|
+
session = advanceReleaseSession({
|
|
5810
|
+
controllerHome: config.controllerHome,
|
|
5811
|
+
sessionId,
|
|
5812
|
+
expectedRevision: session.revision,
|
|
5813
|
+
phase: 'failed',
|
|
5814
|
+
receipts: [{
|
|
5815
|
+
id: 'cutover_failed',
|
|
5816
|
+
kind: 'cutover',
|
|
5817
|
+
summary: `${reason}; Stable A outcome is not fully verified; Candidate B is retired`.slice(0, 500),
|
|
5818
|
+
}],
|
|
5819
|
+
});
|
|
5820
|
+
const cleanup = cleanupRetiredCandidateLane(config, session);
|
|
5821
|
+
audit(config, 'release_session_cutover_reconciled_failed', {
|
|
5822
|
+
sessionId,
|
|
5823
|
+
candidateReleaseId: candidateRelease.releaseId,
|
|
5824
|
+
candidateRetired: true,
|
|
5825
|
+
reason,
|
|
5826
|
+
observedActiveReleaseId: stableNow.snapshot?.releaseId,
|
|
5827
|
+
});
|
|
5828
|
+
return {
|
|
5829
|
+
ok: false as const,
|
|
5830
|
+
attempted: true,
|
|
5831
|
+
detail: `${reason}; Candidate B is retired but Stable A cutover/rollback outcome is not fully verified${cleanup.ok ? '' : `; Candidate B cleanup failed: ${cleanup.detail}`}`,
|
|
5832
|
+
releaseSession: session,
|
|
5833
|
+
...(activation ? { activation } : {}),
|
|
5834
|
+
};
|
|
5835
|
+
};
|
|
5836
|
+
|
|
5837
|
+
if (session.phase === 'cutover_attempting' || session.phase === 'cutover_committed') {
|
|
5838
|
+
return reconcileCutoverAttempt(`resuming ReleaseSession from ${session.phase} without a second activation attempt`);
|
|
5839
|
+
}
|
|
5840
|
+
if (session.phase !== 'cutover_eligible') {
|
|
5841
|
+
return { ok: false as const, attempted: false, noOp: true, detail: `RELEASE_SESSION_CUTOVER_REQUIRES_ELIGIBLE: ${session.phase}`, releaseSession: session };
|
|
5842
|
+
}
|
|
5843
|
+
|
|
5844
|
+
try {
|
|
5845
|
+
assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
|
|
5846
|
+
const stableBefore = await verifyLocalRuntimeForReleaseCutover(config);
|
|
5847
|
+
if (!stableBefore.ok) {
|
|
5848
|
+
const reasonCodes = stableBefore.runtime.reasonCodes.join(',') || 'unknown';
|
|
5849
|
+
audit(config, 'release_session_cutover_precondition_deferred', {
|
|
5850
|
+
sessionId,
|
|
5851
|
+
detail: 'stable Runtime verification is temporarily unavailable before cutover',
|
|
5852
|
+
reasonCodes,
|
|
5853
|
+
});
|
|
5854
|
+
return {
|
|
5855
|
+
ok: false as const,
|
|
5856
|
+
attempted: false,
|
|
5857
|
+
noOp: true,
|
|
5858
|
+
detail: `RELEASE_SESSION_STABLE_RUNTIME_VERIFICATION_DEFERRED: ${reasonCodes}`,
|
|
5859
|
+
releaseSession: session,
|
|
5860
|
+
};
|
|
5861
|
+
}
|
|
5862
|
+
if (
|
|
5863
|
+
stableBefore.releases.active?.revision !== session.stableRelease.releaseId
|
|
5864
|
+
|| stableBefore.releases.active?.artifactIdentity !== session.stableRelease.artifactIdentity
|
|
5865
|
+
) throw new Error('RELEASE_SESSION_STABLE_RUNTIME_IDENTITY_CHANGED');
|
|
5866
|
+
|
|
5867
|
+
const candidateConfig = candidateRecoveryConfig(session);
|
|
5868
|
+
const candidateBefore = await verifyLocalRuntimeForReleaseCutover(candidateConfig);
|
|
5869
|
+
if (
|
|
5870
|
+
!candidateBefore.ok
|
|
5871
|
+
|| candidateBefore.releases.active?.revision !== candidateRelease.releaseId
|
|
5872
|
+
|| candidateBefore.releases.active?.artifactIdentity !== candidateRelease.artifactIdentity
|
|
5873
|
+
) throw new Error('RELEASE_SESSION_CANDIDATE_NO_LONGER_VERIFIED');
|
|
5874
|
+
if (runtimeReleaseTreeSha256(dirname(candidateRelease.manifestPath)) !== candidateRelease.treeSha256) {
|
|
5875
|
+
throw new Error('RELEASE_SESSION_CANDIDATE_TREE_CHANGED');
|
|
5876
|
+
}
|
|
5877
|
+
|
|
5878
|
+
const promoted = promotePortableRuntimeRelease({
|
|
5879
|
+
sourceManifestPath: candidateRelease.manifestPath,
|
|
5880
|
+
targetControllerHome: config.controllerHome,
|
|
5881
|
+
expectedTreeSha256: candidateRelease.treeSha256,
|
|
5882
|
+
});
|
|
5883
|
+
if (
|
|
5884
|
+
promoted.releaseId !== candidateRelease.releaseId
|
|
5885
|
+
|| promoted.artifactIdentity !== candidateRelease.artifactIdentity
|
|
5886
|
+
|| promoted.manifestSha256 !== candidateRelease.manifestSha256
|
|
5887
|
+
|| promoted.treeSha256 !== candidateRelease.treeSha256
|
|
5888
|
+
) throw new Error('RELEASE_SESSION_PROMOTION_IDENTITY_MISMATCH');
|
|
5889
|
+
|
|
5890
|
+
// Promotion writes only a new immutable release tree. Stable A must still
|
|
5891
|
+
// be the exact frozen authority before we enter the one cutover attempt.
|
|
5892
|
+
assertStableReleaseSessionIdentityCurrent(config, session.stableRelease);
|
|
5893
|
+
const stableAfterPromotion = await verifyLocalRuntimeForReleaseCutover(config);
|
|
5894
|
+
if (!stableAfterPromotion.ok) {
|
|
5895
|
+
const reasonCodes = stableAfterPromotion.runtime.reasonCodes.join(',') || 'unknown';
|
|
5896
|
+
audit(config, 'release_session_cutover_post_promotion_verification_deferred', {
|
|
5897
|
+
sessionId,
|
|
5898
|
+
detail: 'stable Runtime verification is temporarily unavailable after immutable promotion and before activation',
|
|
5899
|
+
reasonCodes,
|
|
5900
|
+
});
|
|
5901
|
+
return {
|
|
5902
|
+
ok: false as const,
|
|
5903
|
+
attempted: false,
|
|
5904
|
+
noOp: true,
|
|
5905
|
+
detail: `RELEASE_SESSION_STABLE_POST_PROMOTION_VERIFICATION_DEFERRED: ${reasonCodes}`,
|
|
5906
|
+
releaseSession: session,
|
|
5907
|
+
};
|
|
5908
|
+
}
|
|
5909
|
+
if (
|
|
5910
|
+
stableAfterPromotion.releases.active?.revision !== session.stableRelease.releaseId
|
|
5911
|
+
|| stableAfterPromotion.releases.active?.artifactIdentity !== session.stableRelease.artifactIdentity
|
|
5912
|
+
) throw new Error('RELEASE_SESSION_STABLE_CHANGED_DURING_PROMOTION');
|
|
5913
|
+
|
|
5914
|
+
// Connector quietness is a cutover invariant, not an automatic-release
|
|
5915
|
+
// policy. Every caller, including explicit/local-recovery ReleaseSession
|
|
5916
|
+
// progression, must prove that no MCP request/stream is active and that
|
|
5917
|
+
// remaining sessions have been idle for the bounded quiet window before
|
|
5918
|
+
// Stable A ownership can move. Otherwise a perfectly valid release switch
|
|
5919
|
+
// can tear down the ChatGPT turn that requested it.
|
|
5920
|
+
let readiness = await probeReleaseCutoverReadiness(config);
|
|
5921
|
+
// The post-promotion verification above may itself create MCP activity
|
|
5922
|
+
// on the primary Connector. Give that activity a bounded opportunity
|
|
5923
|
+
// to become quiet before deferring the session; otherwise every retry
|
|
5924
|
+
// re-verifies the Runtime, refreshes latestActivityAgeMs, and can defer
|
|
5925
|
+
// forever without ever reaching the cutover attempt.
|
|
5926
|
+
const quietWaitDeadline = Date.now() + AUTOMATIC_RELEASE_CUTOVER_QUIET_MS + 5_000;
|
|
5927
|
+
while (
|
|
5928
|
+
!readiness.ready
|
|
5929
|
+
&& readiness.activePosts === 0
|
|
5930
|
+
&& readiness.activeStreams === 0
|
|
5931
|
+
&& readiness.initializing === 0
|
|
5932
|
+
&& Number.isFinite(readiness.latestActivityAgeMs)
|
|
5933
|
+
&& (readiness.latestActivityAgeMs ?? AUTOMATIC_RELEASE_CUTOVER_QUIET_MS) < AUTOMATIC_RELEASE_CUTOVER_QUIET_MS
|
|
5934
|
+
&& Date.now() < quietWaitDeadline
|
|
5935
|
+
) {
|
|
5936
|
+
const remainingMs = Math.max(
|
|
5937
|
+
50,
|
|
5938
|
+
AUTOMATIC_RELEASE_CUTOVER_QUIET_MS - (readiness.latestActivityAgeMs ?? 0) + 25,
|
|
5939
|
+
);
|
|
5940
|
+
await sleep(Math.min(2_000, remainingMs));
|
|
5941
|
+
readiness = await probeReleaseCutoverReadiness(config);
|
|
5942
|
+
}
|
|
5943
|
+
if (!readiness.ready) {
|
|
5944
|
+
audit(config, 'release_session_cutover_deferred_for_mcp_activity', {
|
|
5945
|
+
sessionId,
|
|
5946
|
+
detail: readiness.detail,
|
|
5947
|
+
activeSessions: readiness.activeSessions,
|
|
5948
|
+
activePosts: readiness.activePosts,
|
|
5949
|
+
activeStreams: readiness.activeStreams,
|
|
5950
|
+
initializing: readiness.initializing,
|
|
5951
|
+
latestActivityAgeMs: readiness.latestActivityAgeMs,
|
|
5952
|
+
requestKind: requestId?.startsWith('recovery-auto-release:') ? 'automatic' : 'explicit',
|
|
5953
|
+
});
|
|
5954
|
+
return {
|
|
5955
|
+
ok: true as const,
|
|
5956
|
+
attempted: false,
|
|
5957
|
+
noOp: true,
|
|
5958
|
+
detail: `RELEASE_SESSION_CUTOVER_DEFERRED_FOR_MCP_ACTIVITY: ${readiness.detail}`,
|
|
5959
|
+
releaseSession: session,
|
|
5960
|
+
};
|
|
5961
|
+
}
|
|
5962
|
+
|
|
5963
|
+
session = advanceReleaseSession({
|
|
5964
|
+
controllerHome: config.controllerHome,
|
|
5965
|
+
sessionId,
|
|
5966
|
+
expectedRevision: session.revision,
|
|
5967
|
+
phase: 'cutover_attempting',
|
|
5968
|
+
receipts: [{
|
|
5969
|
+
id: 'cutover_attempt',
|
|
5970
|
+
kind: 'cutover',
|
|
5971
|
+
summary: `byte-identical Candidate B ${candidateRelease.releaseId} promoted to Stable A release storage; beginning the single fenced cutover attempt`,
|
|
5972
|
+
}],
|
|
5973
|
+
});
|
|
5974
|
+
|
|
5975
|
+
const activation = await activateRuntimeReleaseInternal(config, promoted.manifestPath, {}, {
|
|
5976
|
+
...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
|
|
5977
|
+
expectedAuthorityRevision: session.stableRelease.authorityRevision,
|
|
5978
|
+
expectedActiveReleaseId: session.stableRelease.releaseId,
|
|
5979
|
+
releaseSessionId: session.sessionId,
|
|
5980
|
+
}, releaseSessionLock);
|
|
5981
|
+
|
|
5982
|
+
return reconcileCutoverAttempt(
|
|
5983
|
+
activation.ok
|
|
5984
|
+
? 'cutover activation completed; reconciling canonical Runtime identity before phase progression'
|
|
5985
|
+
: `cutover activation returned failure: ${activation.detail}`,
|
|
5986
|
+
activation,
|
|
5987
|
+
);
|
|
5988
|
+
} catch (error) {
|
|
5989
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
5990
|
+
const latest = readReleaseSession(config.controllerHome, sessionId) ?? session;
|
|
5991
|
+
session = latest;
|
|
5992
|
+
if (latest.phase === 'cutover_attempting' || latest.phase === 'cutover_committed') {
|
|
5993
|
+
return reconcileCutoverAttempt(`cutover interrupted after entering ${latest.phase}: ${detail}`);
|
|
5994
|
+
}
|
|
5995
|
+
if (latest.phase === 'cutover_eligible') {
|
|
5996
|
+
const retirement = await stopReleaseSessionCandidateService(latest);
|
|
5997
|
+
if (!retirement.ok) {
|
|
5998
|
+
audit(config, 'release_session_candidate_retirement_failed', {
|
|
5999
|
+
sessionId,
|
|
6000
|
+
phase: latest.phase,
|
|
6001
|
+
candidateControllerHome: latest.candidate.controllerHome,
|
|
6002
|
+
detail: retirement.detail,
|
|
6003
|
+
cutoverError: detail,
|
|
6004
|
+
});
|
|
6005
|
+
return {
|
|
6006
|
+
ok: false as const,
|
|
6007
|
+
attempted: false,
|
|
6008
|
+
noOp: true,
|
|
6009
|
+
detail: `${detail}; Candidate B retirement failed: ${retirement.detail}`,
|
|
6010
|
+
releaseSession: latest,
|
|
6011
|
+
};
|
|
6012
|
+
}
|
|
6013
|
+
try {
|
|
6014
|
+
session = advanceReleaseSession({
|
|
6015
|
+
controllerHome: config.controllerHome,
|
|
6016
|
+
sessionId,
|
|
6017
|
+
expectedRevision: latest.revision,
|
|
6018
|
+
phase: 'failed',
|
|
6019
|
+
receipts: [{
|
|
6020
|
+
id: 'cutover_precondition_failed',
|
|
6021
|
+
kind: 'cutover',
|
|
6022
|
+
summary: `${detail}; Candidate B retired`.slice(0, 500),
|
|
6023
|
+
}],
|
|
6024
|
+
});
|
|
6025
|
+
cleanupRetiredCandidateLane(config, session);
|
|
6026
|
+
} catch { /* preserve original cutover precondition failure */ }
|
|
6027
|
+
audit(config, 'release_session_cutover_precondition_failed', {
|
|
6028
|
+
sessionId,
|
|
6029
|
+
detail,
|
|
6030
|
+
candidateRetired: true,
|
|
6031
|
+
});
|
|
6032
|
+
}
|
|
6033
|
+
return { ok: false as const, attempted: false, noOp: true, detail, releaseSession: session };
|
|
6034
|
+
}
|
|
6035
|
+
});
|
|
6036
|
+
if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
|
|
6037
|
+
return locked.value;
|
|
6038
|
+
}
|
|
6039
|
+
|
|
6040
|
+
|
|
6041
|
+
async function cancelReleaseSessionUnderLock(
|
|
6042
|
+
config: RecoveryConfig,
|
|
6043
|
+
initialSession: ReleaseSession,
|
|
6044
|
+
reason: string,
|
|
6045
|
+
): Promise<ConfiguredRuntimeActivationResult> {
|
|
6046
|
+
let session = initialSession;
|
|
6047
|
+
if (session.phase === 'failed') {
|
|
6048
|
+
return { ok: true, attempted: false, noOp: true, detail: 'ReleaseSession is already terminal failed', releaseSession: session };
|
|
6049
|
+
}
|
|
6050
|
+
if (['cutover_attempting', 'cutover_committed', 'soaking'].includes(session.phase)) {
|
|
6051
|
+
return {
|
|
6052
|
+
ok: false,
|
|
6053
|
+
attempted: false,
|
|
6054
|
+
noOp: true,
|
|
6055
|
+
detail: `RELEASE_SESSION_CANCEL_AFTER_CUTOVER_FORBIDDEN: ${session.phase}; use exact ReleaseSession rollback/soak resolution`,
|
|
6056
|
+
releaseSession: session,
|
|
6057
|
+
};
|
|
6058
|
+
}
|
|
6059
|
+
if (session.phase === 'known_good' || session.phase === 'rolled_back') {
|
|
6060
|
+
return {
|
|
6061
|
+
ok: false,
|
|
6062
|
+
attempted: false,
|
|
6063
|
+
noOp: true,
|
|
6064
|
+
detail: `RELEASE_SESSION_CANCEL_TERMINAL: ${session.phase}`,
|
|
6065
|
+
releaseSession: session,
|
|
6066
|
+
};
|
|
6067
|
+
}
|
|
6068
|
+
|
|
6069
|
+
const priorPhase = session.phase;
|
|
6070
|
+
if (['candidate_booted', 'candidate_verified', 'cutover_eligible'].includes(session.phase)) {
|
|
6071
|
+
const retirement = await stopReleaseSessionCandidateService(session);
|
|
6072
|
+
if (!retirement.ok) {
|
|
6073
|
+
return {
|
|
6074
|
+
ok: false,
|
|
6075
|
+
attempted: true,
|
|
6076
|
+
detail: `RELEASE_SESSION_CANCEL_CANDIDATE_RETIRE_FAILED: ${retirement.detail}`,
|
|
6077
|
+
releaseSession: session,
|
|
6078
|
+
};
|
|
6079
|
+
}
|
|
6080
|
+
}
|
|
6081
|
+
|
|
6082
|
+
session = advanceReleaseSession({
|
|
6083
|
+
controllerHome: config.controllerHome,
|
|
6084
|
+
sessionId: session.sessionId,
|
|
6085
|
+
expectedRevision: session.revision,
|
|
6086
|
+
phase: 'failed',
|
|
6087
|
+
receipts: [{
|
|
6088
|
+
id: 'candidate_cancelled',
|
|
6089
|
+
kind: 'candidate_canary',
|
|
6090
|
+
summary: `Candidate B was retired before cutover: ${reason}`.slice(0, 500),
|
|
6091
|
+
}],
|
|
6092
|
+
});
|
|
6093
|
+
const cleanup = cleanupRetiredCandidateLane(config, session);
|
|
6094
|
+
audit(config, cleanup.ok ? 'release_session_cancelled' : 'release_session_cancelled_cleanup_failed', {
|
|
6095
|
+
sessionId: session.sessionId,
|
|
6096
|
+
priorPhase,
|
|
6097
|
+
candidateControllerHome: session.candidate.controllerHome,
|
|
6098
|
+
reason,
|
|
6099
|
+
cleanupDetail: cleanup.detail,
|
|
6100
|
+
});
|
|
6101
|
+
return {
|
|
6102
|
+
ok: cleanup.ok,
|
|
6103
|
+
attempted: true,
|
|
6104
|
+
detail: cleanup.ok
|
|
6105
|
+
? 'ReleaseSession Candidate B was terminalized failed and its isolated lane was retired before cutover; Stable A remained unchanged'
|
|
6106
|
+
: `ReleaseSession was terminalized failed but Candidate B lane cleanup failed: ${cleanup.detail}`,
|
|
6107
|
+
releaseSession: session,
|
|
6108
|
+
};
|
|
6109
|
+
}
|
|
6110
|
+
|
|
6111
|
+
export async function cancelConfiguredRuntimeReleaseSession(
|
|
6112
|
+
config: RecoveryConfig,
|
|
6113
|
+
sessionId: string,
|
|
6114
|
+
requestId?: string,
|
|
6115
|
+
): Promise<ConfiguredRuntimeActivationResult> {
|
|
6116
|
+
const locked = await withLock(config, {
|
|
6117
|
+
action: 'release_session_cancel',
|
|
6118
|
+
...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
|
|
3198
6119
|
}, async () => {
|
|
6120
|
+
const session = readReleaseSession(config.controllerHome, sessionId);
|
|
6121
|
+
if (!session) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
|
|
6122
|
+
return cancelReleaseSessionUnderLock(config, session, 'explicit Recovery cancellation');
|
|
6123
|
+
});
|
|
6124
|
+
if (!locked.acquired) {
|
|
6125
|
+
return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
|
|
6126
|
+
}
|
|
6127
|
+
return locked.value;
|
|
6128
|
+
}
|
|
6129
|
+
|
|
6130
|
+
|
|
6131
|
+
function releaseSessionRollbackTransaction(
|
|
6132
|
+
config: RecoveryConfig,
|
|
6133
|
+
session: ReleaseSession,
|
|
6134
|
+
): NonNullable<ReleaseSession['transaction']> {
|
|
6135
|
+
const authority = readRuntimeReleaseAuthority(config.controllerHome);
|
|
6136
|
+
const transaction = session.transaction;
|
|
6137
|
+
const candidateRelease = session.candidateRelease;
|
|
6138
|
+
const previous = authority?.previous;
|
|
6139
|
+
if (!authority || !transaction || !candidateRelease || !previous?.databaseBackup) {
|
|
6140
|
+
throw new Error('RELEASE_SESSION_ROLLBACK_TRANSACTION_REQUIRED');
|
|
6141
|
+
}
|
|
6142
|
+
if (
|
|
6143
|
+
transaction.candidateReleaseId !== candidateRelease.releaseId
|
|
6144
|
+
|| authority.active.releaseId !== candidateRelease.releaseId
|
|
6145
|
+
|| authority.active.artifactIdentity !== candidateRelease.artifactIdentity
|
|
6146
|
+
|| previous.releaseId !== transaction.rollbackRelease.releaseId
|
|
6147
|
+
|| previous.artifactIdentity !== transaction.rollbackRelease.artifactIdentity
|
|
6148
|
+
|| previous.manifestSha256 !== transaction.rollbackRelease.manifestSha256
|
|
6149
|
+
|| previous.databaseBackup.path !== transaction.rollbackRelease.databaseBackup?.path
|
|
6150
|
+
|| transaction.rollbackRelease.releaseId !== session.stableRelease.releaseId
|
|
6151
|
+
|| transaction.rollbackRelease.artifactIdentity !== session.stableRelease.artifactIdentity
|
|
6152
|
+
|| transaction.rollbackRelease.manifestSha256 !== session.stableRelease.manifestSha256
|
|
6153
|
+
) {
|
|
6154
|
+
throw new Error('RELEASE_SESSION_ROLLBACK_TRANSACTION_IDENTITY_MISMATCH');
|
|
6155
|
+
}
|
|
6156
|
+
return transaction;
|
|
6157
|
+
}
|
|
6158
|
+
|
|
6159
|
+
/**
|
|
6160
|
+
* A rollback can durably switch release authority before the process gets to
|
|
6161
|
+
* write the ReleaseSession terminal phase. This is sufficient proof only when
|
|
6162
|
+
* the authority operation, Stable A, Candidate B, and fenced transaction all
|
|
6163
|
+
* agree; a merely similar active/previous pair must not terminalize a session.
|
|
6164
|
+
*/
|
|
6165
|
+
function releaseSessionRollbackAlreadyCommitted(
|
|
6166
|
+
session: ReleaseSession,
|
|
6167
|
+
authority: RuntimeReleaseAuthority | undefined,
|
|
6168
|
+
): boolean {
|
|
6169
|
+
const candidateRelease = session.candidateRelease;
|
|
6170
|
+
const transaction = session.transaction;
|
|
6171
|
+
return Boolean(
|
|
6172
|
+
candidateRelease
|
|
6173
|
+
&& transaction
|
|
6174
|
+
&& authority?.operationId?.startsWith(`release-session-rollback:${session.sessionId}:`)
|
|
6175
|
+
&& authority.active.releaseId === session.stableRelease.releaseId
|
|
6176
|
+
&& authority.active.artifactIdentity === session.stableRelease.artifactIdentity
|
|
6177
|
+
&& authority.active.manifestSha256 === session.stableRelease.manifestSha256
|
|
6178
|
+
&& authority.previous?.releaseId === candidateRelease.releaseId
|
|
6179
|
+
&& authority.previous?.artifactIdentity === candidateRelease.artifactIdentity
|
|
6180
|
+
&& transaction.candidateReleaseId === candidateRelease.releaseId
|
|
6181
|
+
&& transaction.rollbackRelease.releaseId === session.stableRelease.releaseId
|
|
6182
|
+
&& transaction.rollbackRelease.artifactIdentity === session.stableRelease.artifactIdentity
|
|
6183
|
+
);
|
|
6184
|
+
}
|
|
6185
|
+
|
|
6186
|
+
export async function rollbackConfiguredRuntimeReleaseSession(
|
|
6187
|
+
config: RecoveryConfig,
|
|
6188
|
+
sessionId: string,
|
|
6189
|
+
dependencies: PrimaryRuntimeRecoveryDependencies = {},
|
|
6190
|
+
requestId?: string,
|
|
6191
|
+
): Promise<ConfiguredRuntimeActivationResult> {
|
|
6192
|
+
const locked = await withLock(config, {
|
|
6193
|
+
action: 'release_session_rollback',
|
|
6194
|
+
...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
|
|
6195
|
+
}, async () => {
|
|
6196
|
+
let session = readReleaseSession(config.controllerHome, sessionId);
|
|
6197
|
+
if (!session) return { ok: false as const, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
|
|
6198
|
+
if (session.phase === 'rolled_back') {
|
|
6199
|
+
return { ok: true as const, attempted: false, noOp: true, detail: 'ReleaseSession is already rolled back', releaseSession: session };
|
|
6200
|
+
}
|
|
6201
|
+
if (!['cutover_attempting', 'cutover_committed', 'soaking'].includes(session.phase)) {
|
|
6202
|
+
return {
|
|
6203
|
+
ok: false as const,
|
|
6204
|
+
attempted: false,
|
|
6205
|
+
noOp: true,
|
|
6206
|
+
detail: `RELEASE_SESSION_ROLLBACK_REQUIRES_CUTOVER: ${session.phase}`,
|
|
6207
|
+
releaseSession: session,
|
|
6208
|
+
};
|
|
6209
|
+
}
|
|
6210
|
+
|
|
6211
|
+
const candidateRelease = session.candidateRelease;
|
|
6212
|
+
const transactionBeforeRollback = session.transaction;
|
|
6213
|
+
const authorityBeforeRollback = readRuntimeReleaseAuthority(config.controllerHome);
|
|
6214
|
+
const rollbackAlreadyCommitted = releaseSessionRollbackAlreadyCommitted(
|
|
6215
|
+
session,
|
|
6216
|
+
authorityBeforeRollback,
|
|
6217
|
+
);
|
|
6218
|
+
if (rollbackAlreadyCommitted) {
|
|
6219
|
+
const live = observeRuntimeStatus(config.controllerHome);
|
|
6220
|
+
const stableIsLive = live.running && live.ready && !live.stale
|
|
6221
|
+
&& live.snapshot?.releaseId === session.stableRelease.releaseId
|
|
6222
|
+
&& live.snapshot?.artifactIdentity === session.stableRelease.artifactIdentity;
|
|
6223
|
+
if (!stableIsLive) {
|
|
6224
|
+
return {
|
|
6225
|
+
ok: false as const,
|
|
6226
|
+
attempted: false,
|
|
6227
|
+
noOp: true,
|
|
6228
|
+
detail: 'RELEASE_SESSION_ROLLBACK_COMMITTED_RUNTIME_NOT_YET_OBSERVED',
|
|
6229
|
+
releaseSession: session,
|
|
6230
|
+
};
|
|
6231
|
+
}
|
|
6232
|
+
session = advanceReleaseSession({
|
|
6233
|
+
controllerHome: config.controllerHome,
|
|
6234
|
+
sessionId,
|
|
6235
|
+
expectedRevision: session.revision,
|
|
6236
|
+
phase: 'rolled_back',
|
|
6237
|
+
receipts: [{
|
|
6238
|
+
id: 'rollback',
|
|
6239
|
+
kind: 'rollback',
|
|
6240
|
+
summary: `reconciled already-committed rollback ${authorityBeforeRollback!.operationId}; exact Stable A ${session.stableRelease.releaseId} is live and no rollback effect was replayed`,
|
|
6241
|
+
}],
|
|
6242
|
+
});
|
|
6243
|
+
audit(config, 'release_session_rollback_reconciled_committed', {
|
|
6244
|
+
sessionId,
|
|
6245
|
+
rollbackOperationId: authorityBeforeRollback!.operationId,
|
|
6246
|
+
restoredStableReleaseId: session.stableRelease.releaseId,
|
|
6247
|
+
candidateReleaseId: candidateRelease!.releaseId,
|
|
6248
|
+
});
|
|
6249
|
+
return {
|
|
6250
|
+
ok: true as const,
|
|
6251
|
+
attempted: false,
|
|
6252
|
+
noOp: true,
|
|
6253
|
+
detail: 'ReleaseSession reconciled the already-committed Stable A rollback from durable authority without replaying the rollback effect',
|
|
6254
|
+
releaseSession: session,
|
|
6255
|
+
};
|
|
6256
|
+
}
|
|
6257
|
+
|
|
6258
|
+
let transaction: NonNullable<ReleaseSession['transaction']>;
|
|
3199
6259
|
try {
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
return {
|
|
6260
|
+
transaction = releaseSessionRollbackTransaction(config, session);
|
|
6261
|
+
} catch (error) {
|
|
6262
|
+
return {
|
|
6263
|
+
ok: false as const,
|
|
6264
|
+
attempted: false,
|
|
6265
|
+
noOp: true,
|
|
6266
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
6267
|
+
releaseSession: session,
|
|
6268
|
+
};
|
|
6269
|
+
}
|
|
6270
|
+
|
|
6271
|
+
const platform = dependencies.platform ?? process.platform;
|
|
6272
|
+
const uid = await (dependencies.currentUid ?? currentUid)();
|
|
6273
|
+
const service = primaryRuntimeServiceOwner(config, platform, uid);
|
|
6274
|
+
if (!service) {
|
|
6275
|
+
return {
|
|
6276
|
+
ok: false as const,
|
|
6277
|
+
attempted: false,
|
|
6278
|
+
noOp: true,
|
|
6279
|
+
detail: `primary Forge Runtime ${configuredPrimaryRuntimeService(config, platform).platform} service is not installed for ${platform}`,
|
|
6280
|
+
releaseSession: session,
|
|
6281
|
+
};
|
|
6282
|
+
}
|
|
6283
|
+
const runCommand = dependencies.runCommand ?? command;
|
|
6284
|
+
const now = dependencies.now ?? Date.now;
|
|
6285
|
+
const wait = dependencies.sleep ?? sleep;
|
|
6286
|
+
const runtimeRunning = dependencies.runtimeRunning ?? ((value: RecoveryConfig) => observeRuntimeStatus(value.controllerHome).running);
|
|
6287
|
+
const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
|
|
6288
|
+
const observeLocal = dependencies.observeLocal ?? (dependencies.verifyLocal ? dependencies.verifyLocal : observePrimaryRuntimeActivationReadiness);
|
|
6289
|
+
const repairConnectorBinding = dependencies.repairPrimaryConnectorBinding
|
|
6290
|
+
?? ((value: RecoveryConfig) => repairPrimaryConnectorBinding(value, platform));
|
|
6291
|
+
|
|
6292
|
+
const stopped = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
|
|
6293
|
+
if (!stopped.ok) {
|
|
6294
|
+
return { ok: false as const, attempted: true, detail: stopped.detail, releaseSession: session };
|
|
6295
|
+
}
|
|
6296
|
+
|
|
6297
|
+
let restored: RuntimeReleaseAuthority;
|
|
6298
|
+
let databaseRollbackDisposition: RuntimeDatabaseRollbackDisposition = 'preserved_unversioned_backup';
|
|
6299
|
+
try {
|
|
6300
|
+
const rollbackResult = rollbackRuntimeReleaseWithResult(config.controllerHome, `release-session-rollback:${session.sessionId}:${Date.now()}`);
|
|
6301
|
+
restored = rollbackResult.authority;
|
|
6302
|
+
databaseRollbackDisposition = rollbackResult.databaseDisposition;
|
|
6303
|
+
if (
|
|
6304
|
+
restored.active.releaseId !== session.stableRelease.releaseId
|
|
6305
|
+
|| restored.active.artifactIdentity !== session.stableRelease.artifactIdentity
|
|
6306
|
+
|| restored.active.manifestSha256 !== session.stableRelease.manifestSha256
|
|
6307
|
+
) throw new Error('RELEASE_SESSION_ROLLBACK_STABLE_IDENTITY_MISMATCH');
|
|
3203
6308
|
} catch (error) {
|
|
3204
|
-
|
|
6309
|
+
const restart = await rebindStartAndVerifyPrimaryRuntime({
|
|
6310
|
+
config,
|
|
6311
|
+
service,
|
|
6312
|
+
runCommand,
|
|
6313
|
+
now,
|
|
6314
|
+
wait,
|
|
6315
|
+
verifyLocal,
|
|
6316
|
+
observeLocal,
|
|
6317
|
+
ensureRuntimeLaunchContract: dependencies.ensureRuntimeLaunchContract,
|
|
6318
|
+
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
|
|
6319
|
+
successDetail: 'ReleaseSession rollback transaction failed; current authoritative Runtime was restarted and verified',
|
|
6320
|
+
});
|
|
6321
|
+
return {
|
|
6322
|
+
ok: false as const,
|
|
6323
|
+
attempted: true,
|
|
6324
|
+
detail: `${error instanceof Error ? error.message : String(error)}; authoritative Runtime restart: ${restart.detail}`,
|
|
6325
|
+
releaseSession: session,
|
|
6326
|
+
};
|
|
3205
6327
|
}
|
|
6328
|
+
|
|
6329
|
+
let rollbackConnectorBinding: { ok: boolean; attempted: boolean; noOp?: boolean; detail: string } | undefined;
|
|
6330
|
+
const restarted = await rebindStartAndVerifyPrimaryRuntime({
|
|
6331
|
+
config,
|
|
6332
|
+
service,
|
|
6333
|
+
runCommand,
|
|
6334
|
+
now,
|
|
6335
|
+
wait,
|
|
6336
|
+
verifyLocal,
|
|
6337
|
+
observeLocal,
|
|
6338
|
+
ensureRuntimeLaunchContract: dependencies.ensureRuntimeLaunchContract,
|
|
6339
|
+
contractFailureContext: 'after ReleaseSession rollback',
|
|
6340
|
+
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
|
|
6341
|
+
successDetail: databaseRollbackDisposition === 'restored_backup'
|
|
6342
|
+
? 'exact Stable A whole-Runtime release and unchanged SQLite backup restored, restarted, rebound, and verified'
|
|
6343
|
+
: 'exact Stable A Runtime release restored while newer live SQLite state was preserved, restarted, rebound, and verified',
|
|
6344
|
+
afterRuntimeReady: async () => {
|
|
6345
|
+
rollbackConnectorBinding = await repairConnectorBinding(config);
|
|
6346
|
+
return rollbackConnectorBinding.ok
|
|
6347
|
+
? { ok: true, detail: rollbackConnectorBinding.detail }
|
|
6348
|
+
: { ok: false, detail: `Stable A persistent Connector binding failed after ReleaseSession rollback: ${rollbackConnectorBinding.detail}` };
|
|
6349
|
+
},
|
|
6350
|
+
});
|
|
6351
|
+
|
|
6352
|
+
const latest = readReleaseSession(config.controllerHome, sessionId) ?? session;
|
|
6353
|
+
session = advanceReleaseSession({
|
|
6354
|
+
controllerHome: config.controllerHome,
|
|
6355
|
+
sessionId,
|
|
6356
|
+
expectedRevision: latest.revision,
|
|
6357
|
+
phase: restarted.ok ? 'rolled_back' : 'failed',
|
|
6358
|
+
receipts: [{
|
|
6359
|
+
id: restarted.ok ? 'rollback' : 'rollback_failed',
|
|
6360
|
+
kind: 'rollback',
|
|
6361
|
+
summary: restarted.ok
|
|
6362
|
+
? databaseRollbackDisposition === 'restored_backup'
|
|
6363
|
+
? `exact Stable A ${latest.stableRelease.releaseId} and its unchanged SQLite backup restored from ReleaseSession activation transaction`
|
|
6364
|
+
: `exact Stable A ${latest.stableRelease.releaseId} restored while newer live SQLite state was preserved across rollback`
|
|
6365
|
+
: `Stable A authority restored but Runtime verification failed after rollback: ${restarted.detail}`.slice(0, 500),
|
|
6366
|
+
}],
|
|
6367
|
+
});
|
|
6368
|
+
audit(config, restarted.ok ? 'release_session_rolled_back' : 'release_session_rollback_restart_failed', {
|
|
6369
|
+
sessionId,
|
|
6370
|
+
releaseSessionTransactionOperationId: transaction.operationId,
|
|
6371
|
+
restoredStableReleaseId: restored.active.releaseId,
|
|
6372
|
+
connectorBindingRepaired: rollbackConnectorBinding?.ok === true,
|
|
6373
|
+
databaseRollbackDisposition,
|
|
6374
|
+
detail: restarted.detail,
|
|
6375
|
+
});
|
|
6376
|
+
return {
|
|
6377
|
+
ok: restarted.ok,
|
|
6378
|
+
attempted: true,
|
|
6379
|
+
detail: restarted.ok
|
|
6380
|
+
? databaseRollbackDisposition === 'restored_backup'
|
|
6381
|
+
? 'ReleaseSession rollback restored the exact frozen Stable A release, unchanged SQLite backup, service binding, and verified Runtime'
|
|
6382
|
+
: 'ReleaseSession rollback restored the exact frozen Stable A release while preserving newer live SQLite state, service binding, and verified Runtime'
|
|
6383
|
+
: `ReleaseSession restored Stable A authority but failed post-rollback Runtime verification: ${restarted.detail}`,
|
|
6384
|
+
releaseSession: session,
|
|
6385
|
+
};
|
|
3206
6386
|
});
|
|
3207
|
-
if (!
|
|
3208
|
-
return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(
|
|
6387
|
+
if (!locked.acquired) {
|
|
6388
|
+
return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner) };
|
|
3209
6389
|
}
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
6390
|
+
return locked.value;
|
|
6391
|
+
}
|
|
6392
|
+
|
|
6393
|
+
|
|
6394
|
+
export async function promoteConfiguredRuntimeReleaseSessionKnownGood(
|
|
6395
|
+
config: RecoveryConfig,
|
|
6396
|
+
sessionId: string,
|
|
6397
|
+
dependencies: RuntimeReleaseKnownGoodDependencies = {},
|
|
6398
|
+
requestId?: string,
|
|
6399
|
+
): Promise<ConfiguredRuntimeActivationResult> {
|
|
6400
|
+
const initial = readReleaseSession(config.controllerHome, sessionId);
|
|
6401
|
+
if (!initial) return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_MISSING' };
|
|
6402
|
+
if (initial.phase !== 'soaking') {
|
|
6403
|
+
return { ok: false, attempted: false, noOp: true, detail: `RELEASE_SESSION_KNOWN_GOOD_REQUIRES_SOAKING: ${initial.phase}`, releaseSession: initial };
|
|
3213
6404
|
}
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
6405
|
+
// A crash or caller timeout may happen after rollback authority is committed
|
|
6406
|
+
// but before its ReleaseSession receipt is written. Reconcile that terminal
|
|
6407
|
+
// fact before candidate identity or any expensive whole-Runtime acceptance
|
|
6408
|
+
// work; Candidate B is no longer active, so it can never become known-good.
|
|
6409
|
+
if (releaseSessionRollbackAlreadyCommitted(initial, readRuntimeReleaseAuthority(config.controllerHome))) {
|
|
6410
|
+
return rollbackConfiguredRuntimeReleaseSession(
|
|
6411
|
+
config,
|
|
6412
|
+
sessionId,
|
|
6413
|
+
dependencies.rollback ?? {},
|
|
6414
|
+
requestId?.trim() ? `${requestId.trim()}:reconcile-committed-rollback` : undefined,
|
|
6415
|
+
);
|
|
6416
|
+
}
|
|
6417
|
+
const candidateRelease = initial.candidateRelease;
|
|
6418
|
+
if (!candidateRelease) return { ok: false, attempted: false, noOp: true, detail: 'RELEASE_SESSION_CANDIDATE_RELEASE_REQUIRED', releaseSession: initial };
|
|
6419
|
+
|
|
6420
|
+
let attested: ReleaseEvidence;
|
|
6421
|
+
let continuationProof: WorkflowSupervisorContinuationProof;
|
|
6422
|
+
try {
|
|
6423
|
+
const active = activeAuthorityRelease(config);
|
|
6424
|
+
if (
|
|
6425
|
+
!active
|
|
6426
|
+
|| active.revision !== candidateRelease.releaseId
|
|
6427
|
+
|| active.artifactIdentity !== candidateRelease.artifactIdentity
|
|
6428
|
+
|| active.manifestSha256 !== candidateRelease.manifestSha256
|
|
6429
|
+
) throw new Error('RELEASE_SESSION_SOAK_RUNTIME_IDENTITY_MISMATCH');
|
|
6430
|
+
|
|
6431
|
+
const activeAuthority = readRuntimeReleaseAuthority(config.controllerHome);
|
|
6432
|
+
if (
|
|
6433
|
+
!activeAuthority
|
|
6434
|
+
|| activeAuthority.active.releaseId !== candidateRelease.releaseId
|
|
6435
|
+
|| activeAuthority.active.artifactIdentity !== candidateRelease.artifactIdentity
|
|
6436
|
+
|| activeAuthority.active.manifestSha256 !== candidateRelease.manifestSha256
|
|
6437
|
+
) throw new Error('RELEASE_SESSION_SOAK_RUNTIME_IDENTITY_MISMATCH');
|
|
6438
|
+
try {
|
|
6439
|
+
const proof = await (dependencies.continuationProof ?? getWorkflowSupervisorContinuationProof)(
|
|
6440
|
+
resolveWorkflowSupervisorForgeHome(config.controllerHome),
|
|
6441
|
+
{
|
|
6442
|
+
...(candidateRelease.sourceRepositoryId ? { repoId: candidateRelease.sourceRepositoryId } : {}),
|
|
6443
|
+
activeReleaseId: candidateRelease.releaseId,
|
|
6444
|
+
notBefore: activeAuthority.committedAt,
|
|
6445
|
+
},
|
|
6446
|
+
);
|
|
6447
|
+
if (!proof) throw new Error('RELEASE_SESSION_AUTONOMOUS_CONTINUATION_PROOF_MISSING');
|
|
6448
|
+
continuationProof = proof;
|
|
6449
|
+
} catch (error) {
|
|
6450
|
+
if (error instanceof Error && error.message === 'RELEASE_SESSION_AUTONOMOUS_CONTINUATION_PROOF_MISSING') throw error;
|
|
6451
|
+
throw new Error(`RELEASE_SESSION_AUTONOMOUS_CONTINUATION_PROOF_UNAVAILABLE: ${error instanceof Error ? error.message : String(error)}`);
|
|
6452
|
+
}
|
|
6453
|
+
|
|
6454
|
+
const existingAttestation = matchingKnownGood(config, active);
|
|
6455
|
+
if (existingAttestation) {
|
|
6456
|
+
// A known-good attestation is durable release authority, not a transient
|
|
6457
|
+
// observation. Crash/retry reconciliation must reuse it instead of
|
|
6458
|
+
// replaying performance observation and public MCP probes. The live
|
|
6459
|
+
// Runtime still has to own the exact same release and remain locally
|
|
6460
|
+
// live/ready/non-stale before the ReleaseSession can terminalize.
|
|
6461
|
+
if (!liveRuntimeOwnsRelease(config, existingAttestation)) {
|
|
6462
|
+
throw new Error('RELEASE_SESSION_ATTESTED_RUNTIME_NOT_CURRENT');
|
|
6463
|
+
}
|
|
6464
|
+
attested = existingAttestation;
|
|
6465
|
+
} else {
|
|
6466
|
+
const before = await verifyStableRuntime(config);
|
|
6467
|
+
if (!before.ok) throw new Error('RELEASE_SESSION_SOAK_RUNTIME_VERIFY_FAILED');
|
|
6468
|
+
if (!sameReleaseEvidenceIdentity(before.releases.active, active)) {
|
|
6469
|
+
throw new Error('RELEASE_SESSION_SOAK_RUNTIME_IDENTITY_MISMATCH');
|
|
6470
|
+
}
|
|
6471
|
+
attested = await attestKnownGood(config, dependencies);
|
|
6472
|
+
}
|
|
6473
|
+
if (
|
|
6474
|
+
attested.revision !== candidateRelease.releaseId
|
|
6475
|
+
|| attested.artifactIdentity !== candidateRelease.artifactIdentity
|
|
6476
|
+
|| attested.manifestSha256 !== candidateRelease.manifestSha256
|
|
6477
|
+
|| !attested.recoveryBundle
|
|
6478
|
+
) throw new Error('RELEASE_SESSION_KNOWN_GOOD_ATTESTATION_IDENTITY_MISMATCH');
|
|
6479
|
+
inspectKnownGoodRecoveryBundle(config.controllerHome, attested);
|
|
6480
|
+
} catch (error) {
|
|
6481
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
6482
|
+
audit(config, 'release_session_known_good_failed', { sessionId, detail });
|
|
6483
|
+
|
|
6484
|
+
// A hard runaway-CPU rejection is terminal acceptance evidence, not a
|
|
6485
|
+
// transient observation failure. Keeping Candidate B live in `soaking`
|
|
6486
|
+
// caused the automatic release driver to launch another full 60-second
|
|
6487
|
+
// sampler on every reconciliation cycle. Restore the exact frozen Stable A
|
|
6488
|
+
// through the existing ReleaseSession rollback transaction instead. The
|
|
6489
|
+
// coordinator will then treat this source revision as already attempted and
|
|
6490
|
+
// will not autonomously replay it until source authority changes.
|
|
6491
|
+
if (/^RECOVERY_PERFORMANCE_REJECTED:/.test(detail)) {
|
|
6492
|
+
const rollback = await rollbackConfiguredRuntimeReleaseSession(
|
|
6493
|
+
config,
|
|
6494
|
+
sessionId,
|
|
6495
|
+
dependencies.rollback ?? {},
|
|
6496
|
+
requestId?.trim() ? `${requestId.trim()}:performance-rejected` : undefined,
|
|
6497
|
+
);
|
|
6498
|
+
audit(config, 'release_session_known_good_performance_rejected', {
|
|
6499
|
+
sessionId,
|
|
6500
|
+
detail,
|
|
6501
|
+
rollbackOk: rollback.ok,
|
|
6502
|
+
rollbackDetail: rollback.detail,
|
|
6503
|
+
});
|
|
6504
|
+
return {
|
|
6505
|
+
ok: false,
|
|
6506
|
+
attempted: true,
|
|
6507
|
+
detail: rollback.ok
|
|
6508
|
+
? `${detail}; Candidate B rejected by the Recovery runaway-CPU gate and Stable A restored`
|
|
6509
|
+
: `${detail}; automatic Stable A rollback failed: ${rollback.detail}`,
|
|
6510
|
+
releaseSession: rollback.releaseSession ?? readReleaseSession(config.controllerHome, sessionId) ?? initial,
|
|
6511
|
+
};
|
|
6512
|
+
}
|
|
6513
|
+
|
|
6514
|
+
return { ok: false, attempted: true, detail, releaseSession: readReleaseSession(config.controllerHome, sessionId) ?? initial };
|
|
6515
|
+
}
|
|
6516
|
+
|
|
6517
|
+
const locked = await withLock(config, {
|
|
6518
|
+
action: 'release_session_known_good',
|
|
3225
6519
|
...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
6520
|
+
}, async () => {
|
|
6521
|
+
let session = readReleaseSession(config.controllerHome, sessionId);
|
|
6522
|
+
if (!session) return { ok: false as const, attempted: true, detail: 'RELEASE_SESSION_MISSING' };
|
|
6523
|
+
if (session.phase !== 'soaking') {
|
|
6524
|
+
return { ok: false as const, attempted: true, detail: `RELEASE_SESSION_CHANGED_DURING_KNOWN_GOOD_ATTESTATION: ${session.phase}`, releaseSession: session };
|
|
6525
|
+
}
|
|
6526
|
+
// The expensive independent verification and performance observation are
|
|
6527
|
+
// already embodied by `attested`. Re-running the full network probe suite
|
|
6528
|
+
// here created a split-brain transaction: the durable recovery bundle could
|
|
6529
|
+
// be published successfully, then one transient Gateway/MCP timeout left the
|
|
6530
|
+
// ReleaseSession permanently soaking. Under the mutation lock, only fence
|
|
6531
|
+
// the identity that could invalidate that attestation.
|
|
6532
|
+
if (!liveRuntimeOwnsRelease(config, attested)) {
|
|
6533
|
+
return { ok: false as const, attempted: true, detail: 'RELEASE_SESSION_RUNTIME_CHANGED_AFTER_KNOWN_GOOD_ATTESTATION', releaseSession: session };
|
|
6534
|
+
}
|
|
6535
|
+
try {
|
|
6536
|
+
releaseSessionRollbackTransaction(config, session);
|
|
6537
|
+
} catch (error) {
|
|
6538
|
+
return {
|
|
6539
|
+
ok: false as const,
|
|
6540
|
+
attempted: true,
|
|
6541
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
6542
|
+
releaseSession: session,
|
|
6543
|
+
};
|
|
6544
|
+
}
|
|
6545
|
+
session = advanceReleaseSession({
|
|
6546
|
+
controllerHome: config.controllerHome,
|
|
6547
|
+
sessionId,
|
|
6548
|
+
expectedRevision: session.revision,
|
|
6549
|
+
phase: 'known_good',
|
|
6550
|
+
receipts: [
|
|
6551
|
+
{
|
|
6552
|
+
id: 'supervisor_continuation_proof',
|
|
6553
|
+
kind: 'soak',
|
|
6554
|
+
summary: `Supervisor task ${continuationProof.taskId} proved ${continuationProof.actions.join(' -> ')} on active release ${continuationProof.activeReleaseId} across Runtime instances ${continuationProof.runtimeInstanceIds.join(',')}`,
|
|
6555
|
+
},
|
|
6556
|
+
{
|
|
6557
|
+
id: 'known_good',
|
|
6558
|
+
kind: 'known_good',
|
|
6559
|
+
summary: `release ${attested.revision} passed soak/performance observation and owns recoverable bundle ${attested.recoveryBundle!.attestationId}`,
|
|
6560
|
+
},
|
|
6561
|
+
],
|
|
6562
|
+
});
|
|
6563
|
+
audit(config, 'release_session_known_good', {
|
|
6564
|
+
sessionId,
|
|
6565
|
+
releaseId: attested.revision,
|
|
6566
|
+
attestationId: attested.recoveryBundle!.attestationId,
|
|
6567
|
+
releaseAuthorityRevision: attested.releaseAuthorityRevision,
|
|
6568
|
+
releaseSessionTransactionOperationId: session.transaction?.operationId,
|
|
6569
|
+
});
|
|
6570
|
+
return {
|
|
6571
|
+
ok: true as const,
|
|
6572
|
+
attempted: true,
|
|
6573
|
+
detail: 'ReleaseSession became known-good only after full Runtime verification, performance observation, and recoverable release+SQLite+service bundle attestation',
|
|
6574
|
+
releaseSession: session,
|
|
6575
|
+
};
|
|
3232
6576
|
});
|
|
3233
|
-
return {
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
6577
|
+
if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), releaseSession: initial };
|
|
6578
|
+
return locked.value;
|
|
6579
|
+
}
|
|
6580
|
+
|
|
6581
|
+
export interface ConfiguredRuntimeActivationDependencies {
|
|
6582
|
+
stage?: typeof stageRuntimeReleaseFromCandidateSource;
|
|
6583
|
+
activate?: (config: RecoveryConfig, manifestPath: string, guard: RuntimeReleaseActivationGuard) => Promise<RuntimeReleaseActivationResult>;
|
|
6584
|
+
}
|
|
6585
|
+
|
|
6586
|
+
export async function stageAndActivateConfiguredRuntimeRelease(
|
|
6587
|
+
config: RecoveryConfig,
|
|
6588
|
+
dependencies: ConfiguredRuntimeActivationDependencies = {},
|
|
6589
|
+
requestId?: string,
|
|
6590
|
+
): Promise<ConfiguredRuntimeActivationResult> {
|
|
6591
|
+
// Frozen-client compatibility alias. It deliberately stops at Candidate B
|
|
6592
|
+
// build/session preparation. Activation is a separate ReleaseSession phase.
|
|
6593
|
+
return prepareConfiguredRuntimeReleaseSession(config, dependencies, requestId);
|
|
3241
6594
|
}
|
|
3242
6595
|
|
|
3243
6596
|
function tunnelLaunchdService(configured: RecoveryTunnelServiceConfig | undefined, uid: number): LaunchdService | undefined {
|
|
@@ -3276,9 +6629,26 @@ function tunnelRepairAllowed(config: RecoveryConfig, now: number): boolean {
|
|
|
3276
6629
|
return typeof prior?.lastAttemptAt !== 'number' || now - prior.lastAttemptAt >= cooldownMs;
|
|
3277
6630
|
}
|
|
3278
6631
|
|
|
6632
|
+
async function verifyRecoveryTunnelRepairSurface(config: RecoveryConfig): Promise<VerifyResult> {
|
|
6633
|
+
// Recovery tunnel repair is a bootstrap control-plane operation. It must not
|
|
6634
|
+
// depend on the primary public MCP/Connector transport that Recovery exists
|
|
6635
|
+
// to repair around, and it must stay bounded: the repair runs inside the same
|
|
6636
|
+
// process that serves the Recovery gateway, so strict whole-Runtime or
|
|
6637
|
+
// known-good bundle verification here starves that gateway, delays the tunnel's
|
|
6638
|
+
// own OAuth discovery, and turns a repair into a multi-minute Recovery outage.
|
|
6639
|
+
// Observe canonical Runtime authority/status plus the local Recovery gateway
|
|
6640
|
+
// and the dedicated Recovery transport only.
|
|
6641
|
+
return observeBoundedRuntimeHealth(config, createRecoveryHttpTransport(config.controllerHome), {
|
|
6642
|
+
includePrimaryConnectorLocal: false,
|
|
6643
|
+
});
|
|
6644
|
+
}
|
|
6645
|
+
|
|
3279
6646
|
export async function repairPublicTunnel(config: RecoveryConfig, dependencies: PublicTunnelRepairDependencies = {}): Promise<PublicTunnelRepairResult> {
|
|
3280
|
-
const verify = dependencies.verify ??
|
|
3281
|
-
|
|
6647
|
+
const verify = dependencies.verify ?? verifyRecoveryTunnelRepairSurface;
|
|
6648
|
+
// The repair runs in-process with the Recovery gateway it protects, so the
|
|
6649
|
+
// local precondition uses the same bounded surface instead of strict
|
|
6650
|
+
// whole-Runtime verification.
|
|
6651
|
+
const verifyLocal = dependencies.verifyLocal ?? verifyRecoveryTunnelRepairSurface;
|
|
3282
6652
|
const now = dependencies.now ?? Date.now;
|
|
3283
6653
|
const wait = dependencies.sleep ?? sleep;
|
|
3284
6654
|
const runCommand = dependencies.runCommand ?? command;
|
|
@@ -3387,7 +6757,9 @@ export async function repairPublicTunnel(config: RecoveryConfig, dependencies: P
|
|
|
3387
6757
|
audit(config, 'public_tunnel_restart_failed', { serviceLabel, serviceTarget, detail });
|
|
3388
6758
|
return { ok: false, attempted: false, noOp: true, detail, serviceLabel, serviceTarget, verify: before, localVerify: localBefore };
|
|
3389
6759
|
}
|
|
3390
|
-
const connected = await runCommand('tunnel-client', args, Math.max(30_000, configured.postRestartVerifyTimeoutMs ?? 20_000)
|
|
6760
|
+
const connected = await runCommand('tunnel-client', args, Math.max(30_000, configured.postRestartVerifyTimeoutMs ?? 20_000), {
|
|
6761
|
+
cwd: tunnelClientWorkingDirectory(config.controllerHome),
|
|
6762
|
+
});
|
|
3391
6763
|
if (!connected.ok) {
|
|
3392
6764
|
const detail = `OpenAI Secure MCP Tunnel runtime connect failed: ${connected.stderr || connected.stdout || connected.status}`;
|
|
3393
6765
|
audit(config, 'public_tunnel_restart_failed', { serviceLabel, serviceTarget, detail });
|
|
@@ -3453,8 +6825,8 @@ export interface RecoveryWatchdogRestartDependencies extends RecoveryRoleRestart
|
|
|
3453
6825
|
probeWatchdog?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
|
|
3454
6826
|
}
|
|
3455
6827
|
|
|
3456
|
-
function recoveryRoleLaunchdService(config: RecoveryConfig, uid: number,
|
|
3457
|
-
const label =
|
|
6828
|
+
function recoveryRoleLaunchdService(config: RecoveryConfig, uid: number, _role: 'gateway'): LaunchdService {
|
|
6829
|
+
const label = RECOVERY_DAEMON_LABEL;
|
|
3458
6830
|
const generated = join(recoveryRoot(config), 'launchd', `${label}.plist`);
|
|
3459
6831
|
const installed = join(homedir(), 'Library', 'LaunchAgents', `${label}.plist`);
|
|
3460
6832
|
return {
|
|
@@ -3471,19 +6843,14 @@ async function probeRecoveryGateway(config: RecoveryConfig): Promise<{ ok: boole
|
|
|
3471
6843
|
return probe(createRecoveryHttpTransport(config.controllerHome), `http://${config.gateway.host}:${config.gateway.port}/health`);
|
|
3472
6844
|
}
|
|
3473
6845
|
|
|
3474
|
-
async function probeRecoveryWatchdog(config: RecoveryConfig): Promise<{ ok: boolean; detail: string }> {
|
|
3475
|
-
const health = observeRecoveryWatchdogHealth(config.controllerHome);
|
|
3476
|
-
return { ok: health.ok, detail: health.detail };
|
|
3477
|
-
}
|
|
3478
|
-
|
|
3479
6846
|
async function restartRecoveryRole(input: {
|
|
3480
6847
|
config: RecoveryConfig;
|
|
3481
|
-
role: 'gateway'
|
|
3482
|
-
action: 'restart_recovery_gateway'
|
|
6848
|
+
role: 'gateway';
|
|
6849
|
+
action: 'restart_recovery_gateway';
|
|
3483
6850
|
check: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
|
|
3484
6851
|
dependencies: RecoveryRoleRestartDependencies;
|
|
3485
6852
|
}): Promise<RecoveryGatewayRestartResult> {
|
|
3486
|
-
const display =
|
|
6853
|
+
const display = 'Recovery daemon';
|
|
3487
6854
|
if ((input.dependencies.platform ?? process.platform) !== 'darwin') return { ok: false, attempted: false, noOp: true, detail: `${display} launchd restart is only supported on macOS` };
|
|
3488
6855
|
const initial = await input.check(input.config);
|
|
3489
6856
|
if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: `${display} is already healthy` };
|
|
@@ -3491,7 +6858,7 @@ async function restartRecoveryRole(input: {
|
|
|
3491
6858
|
if (uid === undefined) return { ok: false, attempted: false, noOp: true, detail: `${display} launchd UID is unavailable` };
|
|
3492
6859
|
const service = recoveryRoleLaunchdService(input.config, uid, input.role);
|
|
3493
6860
|
if (!existsSync(service.plistPath)) return { ok: false, attempted: false, noOp: true, detail: `${display} launchd plist is missing: ${service.plistPath}`, serviceTarget: service.target };
|
|
3494
|
-
const eventPrefix =
|
|
6861
|
+
const eventPrefix = 'recovery_daemon';
|
|
3495
6862
|
const locked = await withLock(input.config, { action: input.action }, async () => {
|
|
3496
6863
|
const before = await input.check(input.config);
|
|
3497
6864
|
if (before.ok) return { ok: true, attempted: false, noOp: true, detail: `${display} recovered before restart`, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
|
|
@@ -3533,19 +6900,6 @@ export async function restartRecoveryGateway(
|
|
|
3533
6900
|
});
|
|
3534
6901
|
}
|
|
3535
6902
|
|
|
3536
|
-
export async function restartRecoveryWatchdog(
|
|
3537
|
-
config: RecoveryConfig,
|
|
3538
|
-
dependencies: RecoveryWatchdogRestartDependencies = {},
|
|
3539
|
-
): Promise<RecoveryWatchdogRestartResult> {
|
|
3540
|
-
return restartRecoveryRole({
|
|
3541
|
-
config,
|
|
3542
|
-
role: 'watchdog',
|
|
3543
|
-
action: 'restart_recovery_watchdog',
|
|
3544
|
-
check: dependencies.probeWatchdog ?? probeRecoveryWatchdog,
|
|
3545
|
-
dependencies,
|
|
3546
|
-
});
|
|
3547
|
-
}
|
|
3548
|
-
|
|
3549
6903
|
function pidAlive(pid: number | undefined): boolean {
|
|
3550
6904
|
if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 0) return false;
|
|
3551
6905
|
try { process.kill(pid, 0); return true; } catch { return false; }
|
|
@@ -3565,7 +6919,7 @@ export function gatewayToken(config: RecoveryConfig): string | undefined {
|
|
|
3565
6919
|
export function initializeStandaloneRecovery(
|
|
3566
6920
|
controllerHome: string,
|
|
3567
6921
|
port = 8787,
|
|
3568
|
-
extensions: Partial<Pick<RecoveryConfig, 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryPublicTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryRuntimeSourceRepositoryId' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
|
|
6922
|
+
extensions: Partial<Pick<RecoveryConfig, 'installProfile' | 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryPublicTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryRuntimeSourceRepositoryId' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
|
|
3569
6923
|
): RecoveryConfig {
|
|
3570
6924
|
const root = resolve(controllerHome);
|
|
3571
6925
|
const tokenPath = join(root, 'recovery', 'config', 'gateway-token.json');
|
|
@@ -3604,27 +6958,17 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
3604
6958
|
now,
|
|
3605
6959
|
watchdogRuntimeStartupGraceMs(config),
|
|
3606
6960
|
);
|
|
3607
|
-
//
|
|
3608
|
-
//
|
|
3609
|
-
//
|
|
3610
|
-
//
|
|
3611
|
-
//
|
|
3612
|
-
//
|
|
3613
|
-
|
|
3614
|
-
|
|
3615
|
-
|
|
3616
|
-
|
|
3617
|
-
|
|
3618
|
-
createRecoveryHttpTransport(config.controllerHome),
|
|
3619
|
-
{ probeMcpProtocol: fullVerifyDue },
|
|
3620
|
-
);
|
|
3621
|
-
let localVerify = await verifyLocalRuntime(config, { probeMcpProtocol: fullVerifyDue });
|
|
3622
|
-
if (!localVerify.ok && !fullVerifyDue && !runtimeStartupGrace) {
|
|
3623
|
-
verified = await verifyStableRuntime(config);
|
|
3624
|
-
localVerify = await verifyLocalRuntime(config);
|
|
3625
|
-
fullVerificationPerformed = true;
|
|
3626
|
-
}
|
|
3627
|
-
const lastFullVerifyAt = fullVerificationPerformed ? now : scopedPrior.lastFullVerifyAt;
|
|
6961
|
+
// Watchdog owns bounded health observation only. Never run strict Runtime or
|
|
6962
|
+
// known-good bundle verification from startup or the five-second tick, even
|
|
6963
|
+
// when the bounded tier degrades: those checks can hash/inspect multi-GB
|
|
6964
|
+
// recovery evidence and would make liveness depend on release-acceptance cost.
|
|
6965
|
+
// Explicit verify/attest/release/rollback operations remain the sole strict
|
|
6966
|
+
// recoverability boundaries. Keep the legacy timestamp as non-authoritative
|
|
6967
|
+
// compatibility state so existing persisted WatchdogState remains readable.
|
|
6968
|
+
const health = await observeWatchdogHealthTier(config);
|
|
6969
|
+
const verified = health;
|
|
6970
|
+
const localVerify = health;
|
|
6971
|
+
const lastFullVerifyAt = scopedPrior.lastFullVerifyAt ?? now;
|
|
3628
6972
|
const activeMutation = liveRecoveryMutationLock(config);
|
|
3629
6973
|
if (activeMutation) {
|
|
3630
6974
|
const attributable = recoveryLockOwnerAttributable(activeMutation);
|
|
@@ -3654,7 +6998,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
3654
6998
|
};
|
|
3655
6999
|
return { state, decision: { action: 'degraded', reason }, verify: verified };
|
|
3656
7000
|
}
|
|
3657
|
-
if (runtimeStartupGrace && !localVerify.ok) {
|
|
7001
|
+
if (runtimeStartupGrace && !localVerify.runtime.ok) {
|
|
3658
7002
|
const state: WatchdogState = {
|
|
3659
7003
|
...scopedPrior,
|
|
3660
7004
|
failures: 0,
|
|
@@ -3670,8 +7014,24 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
3670
7014
|
verify: verified,
|
|
3671
7015
|
};
|
|
3672
7016
|
}
|
|
7017
|
+
// Recovery is only usable when the transport the Recovery connector actually
|
|
7018
|
+
// reaches is itself verified. Treat the configured Recovery transport as part
|
|
7019
|
+
// of the recovery-health gate so a dead tunnel degrades the watchdog and
|
|
7020
|
+
// reaches the bounded `repair_public_tunnel` path instead of reporting healthy.
|
|
7021
|
+
const recoveryTransportHealthy = configuredRecoveryTunnel(config)?.platform === 'openai-secure-tunnel'
|
|
7022
|
+
? verified.probes.recovery_tunnel_runtime?.ok === true
|
|
7023
|
+
: verified.probes.recovery_external_http?.ok !== false;
|
|
3673
7024
|
const recoveryHealthy = verified.probes.recovery_gateway?.ok !== false
|
|
3674
|
-
&&
|
|
7025
|
+
&& recoveryTransportHealthy;
|
|
7026
|
+
const recoveryTunnelObservation = openAiTunnelObservationFromProbe(verified.probes.recovery_tunnel_runtime);
|
|
7027
|
+
if (isUpstreamOpenAiControlPlaneIncident(recoveryTunnelObservation) && scopedPrior.failures === 0) {
|
|
7028
|
+
await persistOpenAiTransportIncidentEvidence({
|
|
7029
|
+
config,
|
|
7030
|
+
verify: verified,
|
|
7031
|
+
runtimeObservation,
|
|
7032
|
+
recoveryTunnel: recoveryTunnelObservation!,
|
|
7033
|
+
});
|
|
7034
|
+
}
|
|
3675
7035
|
const primaryRuntimeHealthy = canonicalRuntimeSafeForTargetedConnectorRecovery(localVerify);
|
|
3676
7036
|
const primaryConnectorConfigured = Boolean(config.primaryConnectorService);
|
|
3677
7037
|
const primaryConnectorLocalFailed = verified.probes.primary_connector_local?.ok === false;
|