@moretea-labs/forge 1.6.0 → 1.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -0
- package/CHANGELOG.md +39 -0
- package/CLAUDE.md +4 -0
- package/README.md +2 -2
- package/README.zh-CN.md +1 -1
- package/SKILL.md +2 -2
- package/THIRD_PARTY_NOTICES.md +1 -1
- package/agents/openai.yaml +1 -1
- package/assets/hooks/hook-input.sh +11 -8
- package/assets/hooks/lib/session-state.sh +10 -7
- package/assets/hooks/lib/workflow-state.sh +13 -177
- package/assets/hooks/post-tool-observer.sh +8 -2
- package/assets/hooks/prompt-guard.sh +5 -81
- package/assets/hooks/session-start-context.sh +56 -1
- package/assets/partials/05-workflow.partial.md +1 -1
- package/assets/partials-agents/02-operating-mode.partial.md +2 -0
- package/assets/partials-agents/03-orchestration.partial.md +1 -1
- package/assets/plugin-registry.v1.json +3 -3
- package/assets/reference-configs/document-generation.md +1 -1
- package/assets/reference-configs/harness-overview.md +1 -1
- package/assets/reference-configs/sprint-contracts.md +3 -3
- package/assets/skill-version.json +2 -2
- package/assets/templates/contract.template.md +2 -7
- package/assets/templates/helpers/capture-plan.sh +2 -2
- package/assets/templates/helpers/check-architecture-sync.sh +17 -0
- package/assets/templates/helpers/check-task-workflow.sh +4 -4
- package/assets/templates/helpers/contract-worktree.sh +1 -21
- package/assets/templates/helpers/ensure-task-workflow.sh +11 -36
- package/assets/templates/helpers/inspect-project-state.ts +8 -6
- package/assets/templates/helpers/migrate-project-template.sh +22 -45
- package/assets/templates/helpers/migrate-workflow-docs.ts +9 -7
- package/assets/templates/helpers/new-plan.sh +1 -1
- package/assets/templates/helpers/plan-to-todo.sh +12 -37
- package/assets/templates/helpers/ship-worktrees.sh +2 -29
- package/assets/templates/helpers/verify-contract.sh +1 -111
- package/assets/templates/helpers/verify-sprint.sh +3 -180
- package/assets/templates/helpers/workflow-contract.ts +5 -1
- package/assets/templates/plan.template.md +1 -1
- package/assets/templates/review.template.md +9 -34
- package/assets/workflow-contract.v1.json +10 -19
- package/docs/README.md +10 -10
- package/docs/architecture/CURRENT.md +178 -0
- package/docs/architecture/EVOLUTION.md +42 -0
- package/docs/architecture/current/README.md +4 -156
- package/docs/architecture/decisions/20260803-release-identity-binding-and-exit-policy.md +2 -2
- package/docs/architecture/decisions/20260823-immediate-controller-relay.md +53 -0
- package/docs/architecture/decisions/20260826-browser-runtime-v2.md +73 -0
- package/docs/architecture/history/README.md +7 -0
- package/docs/architecture/{chatgpt-handoff-facade.md → history/chatgpt-handoff-facade.md} +4 -0
- package/docs/architecture/{global-hook-runtime.md → history/global-hook-runtime.md} +7 -3
- package/docs/architecture/{ios-semantic-automation-provider-v2.md → history/ios-semantic-automation-provider-v2.md} +6 -2
- package/docs/architecture/history.md +3 -34
- package/docs/architecture/index.md +15 -98
- package/docs/architecture/modules/controller-runtime/transport-lifecycle.md +1 -1
- package/docs/architecture/reviews/2026-08-22-semantic-harness-convergence-acceptance.md +39 -0
- package/docs/architecture/versions/1.6.md +38 -0
- package/docs/architecture/versions/1.7.md +41 -0
- package/docs/forge-chatgpt-browser-engine.md +2 -2
- package/docs/forge-chatgpt-mcp-setup.md +4 -28
- package/docs/forge-chatgpt-supervised-automation.md +3 -3
- package/docs/forge-github-issue-launcher.md +1 -1
- package/docs/forge-plugin-management.md +1 -1
- package/docs/forge-runtime-self-healing-loop.md +1 -1
- package/docs/operations/controller-browser-plugin.md +26 -6
- package/docs/operations/controller-performance-and-502.md +143 -0
- package/docs/operations/controller-reliability-runbook.md +1 -1
- package/docs/operations/features.md +2 -2
- package/docs/operations/github-repository.md +1 -1
- package/docs/operations/homebrew.md +2 -2
- package/docs/operations/mcp-tool-exposure.md +5 -3
- package/docs/wiki/Architecture.md +6 -6
- package/docs/wiki/Implementation.md +2 -2
- package/docs/wiki/Integrations.md +2 -2
- package/docs/wiki/Work-Lifecycle.md +1 -1
- package/package.json +33 -11
- package/scripts/activate-source-baseline.ts +17 -13
- package/scripts/architecture-event.ts +2 -820
- package/scripts/architecture-queue.sh +2 -645
- package/scripts/archive-architecture-request.sh +2 -242
- package/scripts/archive-workflow.sh +1 -291
- package/scripts/benchmark-background-check-overlap.ts +228 -0
- package/scripts/benchmark-mcp-host.sh +117 -0
- package/scripts/benchmark-mcp-transport.ts +170 -0
- package/scripts/benchmark-quality-harness.ts +146 -0
- package/scripts/benchmark-route-session-concurrency.ts +207 -6
- package/scripts/benchmark-thin-harness-mcp-e2e.ts +7 -1
- package/scripts/benchmark-typescript-navigation.ts +186 -0
- package/scripts/benchmark-work-efficiency.ts +334 -0
- package/scripts/capability-config.ts +2 -392
- package/scripts/capability-resolver.ts +2 -503
- package/scripts/capture-plan.sh +1 -332
- package/scripts/check-agent-tooling.sh +2 -1558
- package/scripts/check-architecture-sync.sh +2 -371
- package/scripts/check-brain-manifest.sh +2 -253
- package/scripts/check-context-files.sh +2 -47
- package/scripts/check-deploy-sql-order.sh +2 -115
- package/scripts/check-mcp-compatibility.ts +14 -0
- package/scripts/check-runtime-architecture.mjs +349 -79
- package/scripts/check-skill-version.ts +3 -191
- package/scripts/check-source-duplication.mjs +61 -0
- package/scripts/check-stable-baseline.ts +92 -0
- package/scripts/check-task-sync.sh +2 -63
- package/scripts/check-task-workflow.sh +2 -1000
- package/scripts/codex-handoff-resume.sh +2 -267
- package/scripts/context-contract-sync.sh +2 -514
- package/scripts/contract-run.ts +2 -463
- package/scripts/contract-worktree.sh +1 -722
- package/scripts/ensure-task-workflow.sh +1 -1225
- package/scripts/host-temporary-cloud-mcp-e2e.sh +265 -0
- package/scripts/inspect-project-state.ts +4 -279
- package/scripts/install-standalone-recovery.ts +24 -0
- package/scripts/lib/project-init-lib.sh +22 -46
- package/scripts/lib/run-forge-helper.sh +19 -0
- package/scripts/lib/run-forge-helper.ts +26 -0
- package/scripts/migrate-project-template.sh +1 -1189
- package/scripts/migrate-workflow-docs.ts +4 -412
- package/scripts/new-plan.sh +1 -180
- package/scripts/new-spec.sh +1 -47
- package/scripts/new-sprint.sh +1 -46
- package/scripts/plan-to-todo.sh +1 -796
- package/scripts/prepare-codex-handoff.sh +2 -201
- package/scripts/prepare-handoff.sh +1 -78
- package/scripts/refresh-current-status.sh +2 -467
- package/scripts/run-governed-gate.ts +5 -1
- package/scripts/select-agent-context-blocks.sh +2 -64
- package/scripts/ship-worktrees.sh +1 -558
- package/scripts/smoke-ios-physical-device.ts +4 -2
- package/scripts/smoke-runtime-recovery.ts +1 -35
- package/scripts/smoke-schedule-engine.ts +1 -20
- package/scripts/sprint-backlog.sh +1 -834
- package/scripts/stage-runtime-release.ts +2 -1
- package/scripts/summarize-failures.sh +2 -123
- package/scripts/switch-plan.sh +1 -150
- package/scripts/sync-brain-docs.sh +2 -379
- package/scripts/sync-generated-authority.mjs +175 -0
- package/scripts/verify-contract.sh +2 -703
- package/scripts/verify-forge-runtime.sh +0 -1
- package/scripts/verify-sprint.sh +1 -677
- package/scripts/workflow-contract.ts +4 -205
- package/scripts/workstream-sync.sh +2 -261
- package/src/cli/agent-jobs/executor-health.ts +1 -1
- package/src/cli/commands/init-hook.ts +3 -3
- package/src/cli/commands/mcp.ts +9 -5
- package/src/cli/commands/plugin.ts +12 -3
- package/src/cli/commands/recovery.ts +25 -3
- package/src/cli/commands/runtime.ts +55 -3
- package/src/cli/commands/security.ts +12 -1
- package/src/cli/commands/setup-profile.ts +31 -21
- package/src/cli/controller/check-runner.ts +113 -5
- package/src/cli/controller/composite-result.ts +1 -1
- package/src/cli/controller/context/focus.ts +51 -0
- package/src/cli/controller/context/instruction-context.ts +114 -0
- package/src/cli/controller/context/known-paths.ts +76 -0
- package/src/cli/controller/context/query-planning.ts +110 -0
- package/src/cli/controller/context/rule-resolution.ts +532 -0
- package/src/cli/controller/context/source-materializer.ts +278 -0
- package/src/cli/controller/context/structural.ts +16 -0
- package/src/cli/controller/context/types.ts +286 -0
- package/src/cli/controller/context-pack.ts +828 -487
- package/src/cli/controller/launch-agents.ts +77 -2
- package/src/cli/controller/operational-plan.ts +2 -2
- package/src/cli/controller/work-mode.ts +6 -4
- package/src/cli/editing/edit-session.ts +60 -7
- package/src/cli/github/contracts.ts +16 -0
- package/src/cli/github/github.ts +135 -16
- package/src/cli/github/session.ts +2 -1
- package/src/cli/installer/managed-entries.ts +15 -1
- package/src/cli/local-bridge/console-automations.ts +54 -78
- package/src/cli/local-bridge/console-view-models.ts +1 -35
- package/src/cli/local-bridge/facade-api.ts +37 -264
- package/src/cli/local-bridge/frontend/src/App.tsx +3 -3
- package/src/cli/local-bridge/frontend/src/features/automations/AutomationsView.tsx +23 -13
- package/src/cli/local-bridge/frontend/src/features/capabilities/CapabilitiesView.tsx +37 -4
- package/src/cli/local-bridge/frontend/src/features/overview/OverviewView.tsx +6 -4
- package/src/cli/local-bridge/frontend/src/lib/api.ts +1 -6
- package/src/cli/local-bridge/frontend/src/lib/routes.ts +1 -1
- package/src/cli/local-bridge/frontend/src/shell/Sidebar.tsx +2 -2
- package/src/cli/local-bridge/frontend/src/styles.css +1 -1
- package/src/cli/local-bridge/frontend/src/types.ts +5 -8
- package/src/cli/local-bridge/job-store.ts +1 -490
- package/src/cli/local-bridge/server.ts +2 -515
- package/src/cli/local-bridge/ui-dist/app.css +1 -1
- package/src/cli/local-bridge/ui-dist/app.js +3703 -3757
- package/src/cli/mcp/auth.ts +5 -5
- package/src/cli/mcp/instructions.ts +3 -2
- package/src/cli/mcp/legacy-tool-service.ts +10 -222
- package/src/cli/mcp/multi-repository.ts +16 -5
- package/src/cli/mcp/oauth.ts +175 -26
- package/src/cli/mcp/post-finalize-work-attribution.ts +150 -0
- package/src/cli/mcp/repository-tools.ts +282 -146
- package/src/cli/mcp/server.ts +586 -116
- package/src/cli/mcp/tool-capability-cache.ts +1 -1
- package/src/cli/mcp/tool-contract.ts +14 -0
- package/src/cli/mcp/toolset-selection.ts +1 -1
- package/src/cli/mcp/toolset.ts +44 -7
- package/src/cli/mcp/transports/http.ts +124 -14
- package/src/cli/mcp/transports/session-registry.ts +43 -5
- package/src/cli/mcp/types.ts +3 -3
- package/src/cli/repositories/command-classifier.ts +103 -21
- package/src/cli/repositories/command-executor.ts +78 -526
- package/src/cli/repositories/command-process.ts +108 -0
- package/src/cli/repositories/command-scope.ts +99 -41
- package/src/cli/repositories/controller-home.ts +7 -8
- package/src/cli/repositories/git-command-executor.ts +2 -0
- package/src/cli/repositories/locks.ts +53 -16
- package/src/cli/repositories/repository-snapshot.ts +124 -0
- package/src/cli/repositories/selected-path-actions.ts +17 -9
- package/src/cli/repositories/structured-git.ts +118 -6
- package/src/cli/repository/inspector.ts +563 -65
- package/src/cli/repository/session-cache.ts +58 -17
- package/src/cli/v81-entry.ts +1 -4
- package/src/effects/fs-transaction.ts +85 -17
- package/src/runtime/context/codegraph-read-provider.ts +233 -2
- package/src/runtime/context/codegraph-sidecar.cjs +74 -40
- package/src/runtime/context/generic-lsp-provider.ts +355 -0
- package/src/runtime/context/lsp-client.ts +283 -0
- package/src/runtime/context/semantic-navigation-contract.ts +61 -0
- package/src/runtime/context/semantic-navigation.ts +223 -0
- package/src/runtime/context/swift-navigation.ts +565 -0
- package/src/runtime/context/typescript-navigation.ts +226 -0
- package/src/runtime/control-plane/dispatch-priority.ts +28 -0
- package/src/runtime/control-plane/execution/direct-edit-work-completion.ts +48 -16
- package/src/runtime/control-plane/execution/edit-validation-coordinator.ts +86 -24
- package/src/runtime/control-plane/execution/repository-work-attribution.ts +106 -0
- package/src/runtime/control-plane/execution/retained-work-resume.ts +203 -0
- package/src/runtime/control-plane/execution/session-store.ts +18 -0
- package/src/runtime/control-plane/execution/verification-evidence.ts +58 -3
- package/src/runtime/control-plane/execution/work-completion-authority.ts +58 -0
- package/src/runtime/control-plane/execution/work-evidence-policy.ts +200 -0
- package/src/runtime/control-plane/execution/work-execution-support.ts +386 -0
- package/src/runtime/control-plane/execution/work-finalization-service.ts +1913 -0
- package/src/runtime/control-plane/execution/work-handle-authority.ts +96 -0
- package/src/runtime/control-plane/execution/work-handle-store.ts +27 -2
- package/src/runtime/control-plane/execution/work-lifecycle-audit.ts +258 -0
- package/src/runtime/control-plane/execution/work-operation-service.ts +468 -0
- package/src/runtime/control-plane/execution/work-path-scope.ts +37 -0
- package/src/runtime/control-plane/execution/work-preparation-service.ts +479 -0
- package/src/runtime/control-plane/execution/work-process-evidence.ts +129 -0
- package/src/runtime/control-plane/execution/work-remote-delivery.ts +215 -0
- package/src/runtime/control-plane/execution/work-task-receipt.ts +22 -18
- package/src/runtime/control-plane/execution/work-terminal-cleanup.ts +156 -11
- package/src/runtime/control-plane/execution/work-validation-reconciler.ts +22 -14
- package/src/runtime/control-plane/execution/work-verification-context.ts +56 -0
- package/src/runtime/control-plane/execution/work-verification-service.ts +388 -0
- package/src/runtime/control-plane/execution/work-verification-snapshot.ts +90 -9
- package/src/runtime/control-plane/facade/capability-registry.ts +129 -25
- package/src/runtime/control-plane/facade/controller-round-relay.ts +1118 -0
- package/src/runtime/control-plane/facade/controller-session-store.ts +302 -12
- package/src/runtime/control-plane/facade/goal-workloop-access.ts +17 -1
- package/src/runtime/control-plane/facade/goal-workloop.ts +928 -190
- package/src/runtime/control-plane/facade/index.ts +3 -0
- package/src/runtime/control-plane/facade/plan-contract-store.ts +275 -42
- package/src/runtime/control-plane/facade/repository-work-admission.ts +147 -0
- package/src/runtime/control-plane/facade/requirement-authority.ts +83 -0
- package/src/runtime/control-plane/facade/semantic-admission.ts +205 -0
- package/src/runtime/control-plane/facade/suggested-actions.ts +5 -1
- package/src/runtime/control-plane/facade/types.ts +90 -10
- package/src/runtime/control-plane/facade/work-contract-store.ts +157 -261
- package/src/runtime/control-plane/facade/work-state-machine.ts +250 -0
- package/src/runtime/control-plane/generated-cache-retention.ts +271 -0
- package/src/runtime/control-plane/global-scheduler/config.ts +44 -0
- package/src/runtime/control-plane/global-scheduler/dispatch-capacity.ts +80 -0
- package/src/runtime/control-plane/global-scheduler/durable-admission.ts +46 -0
- package/src/runtime/control-plane/global-scheduler/maintenance.ts +152 -0
- package/src/runtime/control-plane/global-scheduler/projection-refresh.ts +84 -0
- package/src/runtime/control-plane/global-scheduler/reconciliation.ts +3 -3
- package/src/runtime/control-plane/global-scheduler/scheduler.ts +278 -543
- package/src/runtime/control-plane/global-scheduler/source-scan.ts +59 -0
- package/src/runtime/control-plane/global-scheduler/state.ts +88 -0
- package/src/runtime/control-plane/global-scheduler/worker-attachment.ts +57 -0
- package/src/runtime/control-plane/global-scheduler/worker-exit-decision.ts +38 -0
- package/src/runtime/control-plane/global-scheduler/worker-exit-reconciler.ts +175 -0
- package/src/runtime/control-plane/global-scheduler/worker-launch.ts +111 -0
- package/src/runtime/control-plane/global-scheduler/worker-lifecycle-store.ts +64 -0
- package/src/runtime/control-plane/global-scheduler/worker-lifecycle.ts +150 -0
- package/src/runtime/control-plane/global-scheduler/worker-process.ts +119 -0
- package/src/runtime/control-plane/global-scheduler/worker-stderr.ts +60 -0
- package/src/runtime/control-plane/launcher/chatgpt-work-binding-store.ts +63 -0
- package/src/runtime/control-plane/launcher/chatgpt-work-continuation.ts +365 -117
- package/src/runtime/control-plane/launcher/launch-reservation-store.ts +89 -0
- package/src/runtime/control-plane/launcher/thin-launcher.ts +244 -11
- package/src/runtime/control-plane/persistence/controller-home-migration.ts +487 -0
- package/src/runtime/control-plane/persistence/requirement-store.ts +0 -12
- package/src/runtime/control-plane/persistence/sqlite-store.ts +33 -2
- package/src/runtime/control-plane/release-retention.ts +62 -2
- package/src/runtime/control-plane/repo-actor/registry.ts +4 -2
- package/src/runtime/control-plane/routing/route-policy.ts +24 -21
- package/src/runtime/control-plane/routing/workspace-admission.ts +70 -0
- package/src/runtime/control-plane/runtime-status-client.ts +0 -2
- package/src/runtime/diagnostics/mcp-timing.ts +13 -0
- package/src/runtime/diagnostics/mcp-transport-benchmark.ts +129 -0
- package/src/runtime/diagnostics/performance.ts +24 -1
- package/src/runtime/evidence/event-ledger.ts +1 -1
- package/src/runtime/evidence/process-check-execution.ts +18 -0
- package/src/runtime/execution/jobs/child-reference.ts +6 -2
- package/src/runtime/execution/jobs/store.ts +58 -4
- package/src/runtime/execution/jobs/types.ts +1 -1
- package/src/runtime/execution/managed-workspace.ts +203 -1
- package/src/runtime/execution/process-runtime/check-facade.ts +51 -4
- package/src/runtime/execution/process-runtime/check-result.ts +88 -0
- package/src/runtime/execution/process-runtime/check-runner-sidecar.ts +7 -1
- package/src/runtime/execution/process-runtime/check-scheduling.ts +114 -0
- package/src/runtime/execution/process-runtime/command-facade.ts +367 -40
- package/src/runtime/execution/process-runtime/gc.ts +38 -6
- package/src/runtime/execution/process-runtime/lightweight-managed.ts +1201 -0
- package/src/runtime/execution/process-runtime/persisted-check.ts +51 -5
- package/src/runtime/execution/process-runtime/resource-claims.ts +10 -6
- package/src/runtime/execution/process-runtime/runtime.ts +53 -3
- package/src/runtime/execution/process-runtime/store.ts +1 -0
- package/src/runtime/execution/process-runtime/types.ts +4 -12
- package/src/runtime/execution/thin-harness/async-process.ts +12 -2
- package/src/runtime/execution/thin-harness/search-worker.ts +54 -5
- package/src/runtime/execution/work-activity.ts +21 -0
- package/src/runtime/execution/workers/executor.ts +18 -61
- package/src/runtime/gateway/mcp/execution-tools.ts +35 -1852
- package/src/runtime/gateway/mcp/legacy-ios-tool-adapter.ts +61 -0
- package/src/runtime/gateway/mcp/process-tools.ts +83 -37
- package/src/runtime/gateway/mcp/router.ts +186 -84
- package/src/runtime/gateway/mcp/runtime-tool-definitions.ts +565 -0
- package/src/runtime/gateway/mcp/runtime-tools.ts +2378 -1947
- package/src/runtime/plugins/app-store-connect-adapter.ts +216 -47
- package/src/runtime/plugins/browser-adapter.ts +2662 -829
- package/src/runtime/plugins/browser-automation-service.ts +10 -0
- package/src/runtime/plugins/browser-handoff-host.ts +2 -25
- package/src/runtime/plugins/browser-handoff.ts +110 -5
- package/src/runtime/plugins/browser-macos-bridge.ts +701 -63
- package/src/runtime/plugins/browser-manifest-surface.ts +603 -0
- package/src/runtime/plugins/browser-native-session-service.ts +222 -0
- package/src/runtime/plugins/browser-node-bridge.ts +6 -34
- package/src/runtime/plugins/browser-provider-registry.ts +109 -0
- package/src/runtime/plugins/browser-runtime-contract.ts +154 -0
- package/src/runtime/plugins/browser-runtime.ts +193 -0
- package/src/runtime/plugins/browser-session-authority.ts +332 -0
- package/src/runtime/plugins/browser-session-store.ts +121 -0
- package/src/runtime/plugins/browser-session-types.ts +98 -0
- package/src/runtime/plugins/capability-authorization-grants.ts +354 -0
- package/src/runtime/plugins/desktop-operator-registration.ts +10 -4
- package/src/runtime/plugins/external-adapter.ts +358 -3
- package/src/runtime/plugins/external-unix-socket.ts +7 -1
- package/src/runtime/plugins/first-party-registry.ts +2 -0
- package/src/runtime/plugins/github-adapter.ts +125 -0
- package/src/runtime/plugins/ios/agent-device-provider.ts +1 -1
- package/src/runtime/plugins/ios/agent-device-typed-provider.ts +52 -9
- package/src/runtime/plugins/ios/remote-xpc-hid.ts +1171 -0
- package/src/runtime/plugins/ios-adapter.ts +106 -16
- package/src/runtime/plugins/ios-agent-device.ts +234 -64
- package/src/runtime/plugins/ios-physical-device.ts +1143 -392
- package/src/runtime/plugins/lightweight-action.ts +153 -0
- package/src/runtime/plugins/local-system-adapter.ts +123 -6
- package/src/runtime/plugins/macos-capability-broker.ts +47 -9
- package/src/runtime/plugins/plugin-action-sidecar.ts +59 -0
- package/src/runtime/plugins/plugin-management-adapter.ts +49 -8
- package/src/runtime/plugins/resend-adapter.ts +1 -1
- package/src/runtime/plugins/store.ts +330 -33
- package/src/runtime/plugins/types.ts +38 -3
- package/src/runtime/plugins/xiaohongshu-publish.ts +515 -0
- package/src/runtime/projections/git-status-sampler.ts +16 -16
- package/src/runtime/projections/materialized-view.ts +28 -10
- package/src/runtime/recovery/maintenance-executor.ts +426 -25
- package/src/runtime/root/mcp-transport.ts +180 -10
- package/src/runtime/root/package-connector-service.ts +230 -10
- package/src/runtime/root/package-runtime-release.ts +143 -17
- package/src/runtime/root/package-runtime-service.ts +461 -22
- package/src/runtime/root/release-manifest.ts +144 -18
- package/src/runtime/root/release-materialize.ts +125 -32
- package/src/runtime/root/release-store.ts +48 -11
- package/src/runtime/root/runtime.ts +23 -5
- package/src/runtime/root/service.ts +100 -29
- package/src/runtime/root/types.ts +12 -0
- package/src/runtime/safe-tooling/ios-development.ts +321 -63
- package/src/runtime/shared/trusted-executable.ts +62 -0
- package/src/runtime/shared/trusted-node-executable.ts +48 -0
- package/src/runtime/standalone-recovery/core.ts +1079 -240
- package/src/runtime/standalone-recovery/entry.ts +100 -24
- package/src/runtime/standalone-recovery/installer.ts +20 -4
- package/src/runtime/standalone-recovery/service-labels.ts +2 -0
- package/src/runtime/standalone-recovery/watchdog-heartbeat.ts +160 -0
- package/src/runtime/workflow/schedules/browser-probe.ts +81 -3
- package/src/runtime/workflow/schedules/engine.ts +364 -202
- package/src/runtime/workflow/schedules/github-issue-watch.ts +158 -0
- package/src/runtime/workflow/schedules/settlement.ts +64 -17
- package/src/runtime/workflow/schedules/store.ts +209 -19
- package/src/runtime/workflow/schedules/types.ts +2 -4
- package/src/runtime/workflow/schedules/work-continuation.ts +26 -13
- package/src/testing/test-governance.ts +8 -40
- package/assets/reference-configs/heartbeat-triage.md +0 -57
- package/assets/templates/helpers/harness-trace-grade.sh +0 -150
- package/assets/templates/helpers/heartbeat-triage.sh +0 -361
- package/assets/templates/helpers/maintenance-triage.sh +0 -167
- package/docs/architecture/current/approved-target-architecture.zh-CN.md +0 -1501
- package/docs/architecture/current/architecture-invariants.md +0 -331
- package/docs/architecture/current/automation-and-schedule-engine.md +0 -459
- package/docs/architecture/current/codegraph-context-provider.md +0 -238
- package/docs/architecture/current/control-plane-authority-inventory.md +0 -94
- package/docs/architecture/current/controller-repository-decoupling.md +0 -148
- package/docs/architecture/current/dispatch-and-agent-strategy.md +0 -373
- package/docs/architecture/current/entity-model.md +0 -612
- package/docs/architecture/current/external-plugin-governance.md +0 -396
- package/docs/architecture/current/failure-recovery.md +0 -447
- package/docs/architecture/current/governance.md +0 -213
- package/docs/architecture/current/human-interaction-plane.md +0 -87
- package/docs/architecture/current/implementation-status.md +0 -128
- package/docs/architecture/current/job-and-run-lifecycle.md +0 -413
- package/docs/architecture/current/legacy-governance.md +0 -58
- package/docs/architecture/current/local-workspace-target-grants.md +0 -70
- package/docs/architecture/current/migration-roadmap.md +0 -105
- package/docs/architecture/current/multi-repository-execution.md +0 -371
- package/docs/architecture/current/operations-runbook.md +0 -45
- package/docs/architecture/current/peer-supercontroller-runtime.md +0 -32
- package/docs/architecture/current/personal-assistant-plugin-baseline.md +0 -206
- package/docs/architecture/current/repository-access-modes.md +0 -102
- package/docs/architecture/current/route-policy.md +0 -15
- package/docs/architecture/current/route-session-concurrency.md +0 -85
- package/docs/architecture/current/runtime-architecture-simplification.md +0 -214
- package/docs/architecture/current/runtime-directory-map.md +0 -49
- package/docs/architecture/current/runtime-health-and-resource-lifecycle.md +0 -85
- package/docs/architecture/current/scheduler-and-resource-claims.md +0 -422
- package/docs/architecture/current/session-aware-execution-and-authorization.md +0 -77
- package/docs/architecture/current/system-overview.md +0 -169
- package/docs/architecture/current/target-requirements-traceability.md +0 -202
- package/docs/architecture/current/thin-harness-governance-policy.md +0 -32
- package/docs/architecture/current/thin-harness-v1.md +0 -321
- package/docs/architecture/current/verification-and-release-gates.md +0 -391
- package/docs/forge-autonomous-goal-loop.md +0 -9
- package/docs/forge-provider-configuration.md +0 -7
- package/docs/operations/chatgpt-personal-assistant-runtime.md +0 -93
- package/docs/operations/gmail-assistant-routines.md +0 -186
- package/docs/operations/personal-assistant-operator-guide.md +0 -61
- package/docs/operations/provider-configuration.md +0 -31
- package/scripts/harness-trace-grade.sh +0 -150
- package/scripts/heartbeat-triage.sh +0 -361
- package/scripts/maintenance-triage.sh +0 -167
- package/src/cli/local-bridge/frontend/src/features/settings/SettingsView.tsx +0 -2
- package/src/cli/repositories/goal-registry.ts +0 -239
- package/src/runtime/assistant/action-proposals.ts +0 -279
- package/src/runtime/assistant/intent.ts +0 -395
- package/src/runtime/assistant/model-provider.ts +0 -460
- package/src/runtime/assistant/openapi.ts +0 -223
- package/src/runtime/assistant/policy.ts +0 -89
- package/src/runtime/assistant/readiness.ts +0 -258
- package/src/runtime/assistant/routine-runtime.ts +0 -477
- package/src/runtime/assistant/schedule-binding.ts +0 -171
- package/src/runtime/assistant/standing-grants.ts +0 -375
- package/src/runtime/assistant/store.ts +0 -243
- package/src/runtime/assistant/types.ts +0 -135
- package/src/runtime/control-plane/goal-loop/config-facade.ts +0 -700
- package/src/runtime/control-plane/goal-loop/config-store.ts +0 -570
- package/src/runtime/control-plane/goal-loop/config-types.ts +0 -213
- package/src/runtime/control-plane/goal-loop/executor-router.ts +0 -109
- package/src/runtime/control-plane/goal-loop/failure-taxonomy.ts +0 -149
- package/src/runtime/control-plane/goal-loop/goal-contract-store.ts +0 -304
- package/src/runtime/control-plane/goal-loop/goal-loop-engine.ts +0 -1001
- package/src/runtime/control-plane/goal-loop/handoff-packet.ts +0 -160
- package/src/runtime/control-plane/goal-loop/index.ts +0 -13
- package/src/runtime/control-plane/goal-loop/local-tools.ts +0 -290
- package/src/runtime/control-plane/goal-loop/policy-gates.ts +0 -203
- package/src/runtime/control-plane/goal-loop/provider-dispatch.ts +0 -93
- package/src/runtime/control-plane/goal-loop/provider-registry.ts +0 -545
- package/src/runtime/control-plane/goal-loop/types.ts +0 -378
- package/src/runtime/control-plane/startup-recovery.ts +0 -198
- package/src/runtime/gateway/mcp/resource-policy.ts +0 -186
- package/src/runtime/personal-assistant/gmail-triage-manager.ts +0 -231
- package/src/runtime/personal-assistant/reporting-runtime.ts +0 -344
- package/src/runtime/personal-assistant/triage-runtime.ts +0 -748
- package/src/runtime/workflow/findings/store.ts +0 -204
- package/src/runtime/workflow/findings/types.ts +0 -58
- package/src/runtime/workflow/portfolio/engine.ts +0 -20
- package/src/runtime/workflow/portfolio/store.ts +0 -185
- package/src/runtime/workflow/portfolio/types.ts +0 -29
|
@@ -4,7 +4,14 @@ import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, re
|
|
|
4
4
|
import { homedir } from 'os';
|
|
5
5
|
import { basename, dirname, isAbsolute, join, resolve } from 'path';
|
|
6
6
|
import { observeRuntimeStatus } from '../root/status';
|
|
7
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
activeRuntimeEntrypoint,
|
|
9
|
+
activeRuntimeLaunchSpec,
|
|
10
|
+
ensureForgeRuntimeLaunchAgentContract,
|
|
11
|
+
forgeRuntimeServicePaths,
|
|
12
|
+
inspectForgeRuntimeLaunchAgentContract,
|
|
13
|
+
readForgeRuntimeServiceConfig,
|
|
14
|
+
} from '../root/service';
|
|
8
15
|
import { loadRuntimeReleaseManifest } from '../root/release-manifest';
|
|
9
16
|
import { assertRuntimeReleaseFiles, stageRuntimeReleaseFromCandidateSource, type StagedRuntimeRelease } from '../root/release-materialize';
|
|
10
17
|
import {
|
|
@@ -15,7 +22,10 @@ import {
|
|
|
15
22
|
type RuntimeReleaseAuthority,
|
|
16
23
|
} from '../root/release-store';
|
|
17
24
|
import type { RuntimeReleaseManifest } from '../root/types';
|
|
25
|
+
import { ensurePackageConnectorService, packageConnectorServicePaths, type PackageConnectorReleaseBinding } from '../root/package-connector-service';
|
|
18
26
|
import { createRecoveryHttpTransport, type RecoveryHttpTransport } from './http-transport';
|
|
27
|
+
import { observeRecoveryWatchdogHealth } from './watchdog-heartbeat';
|
|
28
|
+
import { RECOVERY_GATEWAY_LABEL, RECOVERY_WATCHDOG_LABEL } from './service-labels';
|
|
19
29
|
import {
|
|
20
30
|
migrateStoppedRepoLocalControllerHomeStorage,
|
|
21
31
|
repoLocalControllerHomeStorageNeedsMigration,
|
|
@@ -50,6 +60,12 @@ export interface PrimaryConnectorServiceConfig {
|
|
|
50
60
|
platform: 'launchd';
|
|
51
61
|
label: string;
|
|
52
62
|
plistPath?: string;
|
|
63
|
+
/** Local OAuth MCP endpoint used to distinguish Connector failure from tunnel failure. */
|
|
64
|
+
localMcpUrl?: string;
|
|
65
|
+
minimumFailures?: number;
|
|
66
|
+
minimumFailureDurationMs?: number;
|
|
67
|
+
restartCooldownMs?: number;
|
|
68
|
+
maximumRestartAttempts?: number;
|
|
53
69
|
postRestartVerifyTimeoutMs?: number;
|
|
54
70
|
}
|
|
55
71
|
|
|
@@ -59,6 +75,8 @@ export interface RecoveryConfig {
|
|
|
59
75
|
publicMcpUrl?: string;
|
|
60
76
|
recoveryPublicUrl?: string;
|
|
61
77
|
recoveryTunnelService?: PublicTunnelServiceConfig;
|
|
78
|
+
/** Optional public tunnel serving publicMcpUrl; independent from the OAuth Connector process. */
|
|
79
|
+
primaryPublicTunnelService?: PublicTunnelServiceConfig;
|
|
62
80
|
primaryRuntimeService?: PrimaryRuntimeServiceConfig;
|
|
63
81
|
primaryRuntimeSourceRoot?: string;
|
|
64
82
|
primaryConnectorService?: PrimaryConnectorServiceConfig;
|
|
@@ -95,6 +113,7 @@ export type RecoveryMutationAction =
|
|
|
95
113
|
| 'stage_and_activate_runtime_release'
|
|
96
114
|
| 'restart_primary_connector'
|
|
97
115
|
| 'restart_recovery_gateway'
|
|
116
|
+
| 'restart_recovery_watchdog'
|
|
98
117
|
| 'repair_public_tunnel';
|
|
99
118
|
|
|
100
119
|
interface RecoveryLock {
|
|
@@ -132,8 +151,8 @@ export interface VerifyStableRuntimeOptions {
|
|
|
132
151
|
export interface RecoveryWatchdogDiagnosticEvidence {
|
|
133
152
|
fingerprint: string;
|
|
134
153
|
releaseIdentity: string;
|
|
135
|
-
components: Array<'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_tunnel'>;
|
|
136
|
-
failedProbes: Array<{ name: string; component: 'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_tunnel'; detail: string; status?: number }>;
|
|
154
|
+
components: Array<'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_watchdog' | 'recovery_tunnel'>;
|
|
155
|
+
failedProbes: Array<{ name: string; component: 'runtime' | 'gateway' | 'public_mcp' | 'recovery_gateway' | 'recovery_watchdog' | 'recovery_tunnel'; detail: string; status?: number }>;
|
|
137
156
|
firstObservedAt: string;
|
|
138
157
|
lastObservedAt: string;
|
|
139
158
|
occurrences: number;
|
|
@@ -153,7 +172,7 @@ export interface RollbackResult {
|
|
|
153
172
|
}
|
|
154
173
|
|
|
155
174
|
export interface WatchdogDecision {
|
|
156
|
-
action: 'healthy' | 'degraded' | 'repair_public_tunnel' | 'restart_recovery_gateway' | 'restart_primary_runtime' | 'rollback' | 'recovery_exhausted';
|
|
175
|
+
action: 'healthy' | 'degraded' | 'repair_public_tunnel' | 'restart_recovery_gateway' | 'restart_primary_connector' | 'restart_primary_runtime' | 'rollback' | 'recovery_exhausted';
|
|
157
176
|
reason: string;
|
|
158
177
|
}
|
|
159
178
|
|
|
@@ -175,10 +194,16 @@ export interface WatchdogState {
|
|
|
175
194
|
publicTunnelFailures?: number;
|
|
176
195
|
publicTunnelFirstFailureAt?: number;
|
|
177
196
|
publicTunnelRepairFailures?: number;
|
|
197
|
+
primaryConnectorFailures?: number;
|
|
198
|
+
primaryConnectorFirstFailureAt?: number;
|
|
199
|
+
primaryConnectorRestartAttempts?: number;
|
|
200
|
+
primaryConnectorRestartFailures?: number;
|
|
201
|
+
primaryConnectorRestartLastAttemptAt?: number;
|
|
178
202
|
recoveryGatewayRestartUsed?: boolean;
|
|
179
203
|
recoveryReleaseRevision?: string;
|
|
180
204
|
lastFullVerifyAt?: number;
|
|
181
205
|
lastDecision?: WatchdogDecision['action'];
|
|
206
|
+
lastReason?: string;
|
|
182
207
|
updatedAt?: string;
|
|
183
208
|
}
|
|
184
209
|
|
|
@@ -300,6 +325,10 @@ function configuredRecoveryTunnel(config: RecoveryConfig): PublicTunnelServiceCo
|
|
|
300
325
|
return config.recoveryTunnelService;
|
|
301
326
|
}
|
|
302
327
|
|
|
328
|
+
function configuredPrimaryPublicTunnel(config: RecoveryConfig): PublicTunnelServiceConfig | undefined {
|
|
329
|
+
return config.primaryPublicTunnelService;
|
|
330
|
+
}
|
|
331
|
+
|
|
303
332
|
function configuredRecoveryPublicUrl(config: RecoveryConfig): string | undefined {
|
|
304
333
|
return config.recoveryPublicUrl;
|
|
305
334
|
}
|
|
@@ -324,6 +353,7 @@ export function loadRecoveryConfig(controllerHome: string, explicit?: string): R
|
|
|
324
353
|
...(typeof loaded.publicMcpUrl === 'string' ? { publicMcpUrl: loaded.publicMcpUrl } : {}),
|
|
325
354
|
...(typeof loaded.recoveryPublicUrl === 'string' ? { recoveryPublicUrl: loaded.recoveryPublicUrl } : {}),
|
|
326
355
|
...(loaded.recoveryTunnelService ? { recoveryTunnelService: loaded.recoveryTunnelService } : {}),
|
|
356
|
+
...(loaded.primaryPublicTunnelService ? { primaryPublicTunnelService: loaded.primaryPublicTunnelService } : {}),
|
|
327
357
|
...(loaded.primaryRuntimeService ? { primaryRuntimeService: loaded.primaryRuntimeService } : {}),
|
|
328
358
|
...(typeof loaded.primaryRuntimeSourceRoot === 'string' ? { primaryRuntimeSourceRoot: resolve(loaded.primaryRuntimeSourceRoot) } : {}),
|
|
329
359
|
...(loaded.primaryConnectorService ? { primaryConnectorService: loaded.primaryConnectorService } : {}),
|
|
@@ -423,6 +453,7 @@ function watchdogProbeComponent(name: string): RecoveryWatchdogDiagnosticEvidenc
|
|
|
423
453
|
if (name === 'runtime_status') return 'runtime';
|
|
424
454
|
if (name === 'active_gateway') return 'gateway';
|
|
425
455
|
if (name === 'recovery_gateway') return 'recovery_gateway';
|
|
456
|
+
if (name === 'recovery_watchdog') return 'recovery_watchdog';
|
|
426
457
|
if (name === 'recovery_external_http') return 'recovery_tunnel';
|
|
427
458
|
return 'public_mcp';
|
|
428
459
|
}
|
|
@@ -482,8 +513,34 @@ function recoveryLockOwnerAlive(lock: RecoveryLock): boolean {
|
|
|
482
513
|
return observed === undefined || observed === lock.processStartTime;
|
|
483
514
|
}
|
|
484
515
|
|
|
516
|
+
function recoveryLockOwnerAttributable(owner: RecoveryLock): boolean {
|
|
517
|
+
return Boolean(owner.action && owner.requestId?.trim());
|
|
518
|
+
}
|
|
519
|
+
|
|
520
|
+
function recoveryLockIdentityFailure(owner: RecoveryLock): string {
|
|
521
|
+
return `RECOVERY_OPERATION_LOCK_IDENTITY_UNCERTAIN: live mutation lock pid=${owner.pid} instance=${owner.instanceId} is missing action/request identity`;
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
function assertRecoveryLockOwnerAttributable(config: RecoveryConfig, owner: RecoveryLock): void {
|
|
525
|
+
if (recoveryLockOwnerAttributable(owner)) return;
|
|
526
|
+
audit(config, 'recovery_operation_lock_identity_uncertain', {
|
|
527
|
+
pid: owner.pid,
|
|
528
|
+
instanceId: owner.instanceId,
|
|
529
|
+
action: owner.action ?? null,
|
|
530
|
+
requestId: owner.requestId ?? null,
|
|
531
|
+
acquiredAt: owner.acquiredAt,
|
|
532
|
+
});
|
|
533
|
+
throw new Error(recoveryLockIdentityFailure(owner));
|
|
534
|
+
}
|
|
535
|
+
|
|
485
536
|
function recoveryBusyDetail(owner: RecoveryLock): string {
|
|
486
|
-
return `Recovery mutation already in progress: action=${owner.action ?? '
|
|
537
|
+
return `Recovery mutation already in progress: action=${owner.action ?? 'invalid'} request=${owner.requestId ?? 'invalid'} pid=${owner.pid} instance=${owner.instanceId}`;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
function liveRecoveryMutationLock(config: RecoveryConfig): RecoveryLock | undefined {
|
|
541
|
+
const owner = json<RecoveryLock>(lockPath(config));
|
|
542
|
+
if (!owner || !Number.isInteger(owner.pid) || owner.pid <= 0 || typeof owner.instanceId !== 'string' || !owner.instanceId.trim()) return undefined;
|
|
543
|
+
return recoveryLockOwnerAlive(owner) ? owner : undefined;
|
|
487
544
|
}
|
|
488
545
|
|
|
489
546
|
async function withLock<T>(
|
|
@@ -493,14 +550,16 @@ async function withLock<T>(
|
|
|
493
550
|
): Promise<RecoveryLockAttempt<T>> {
|
|
494
551
|
const path = lockPath(config);
|
|
495
552
|
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
553
|
+
const instanceId = randomUUID();
|
|
554
|
+
const requestId = intent.requestId?.trim() || `internal:${intent.action}:${instanceId}`;
|
|
496
555
|
const lock: RecoveryLock = {
|
|
497
556
|
schemaVersion: 1,
|
|
498
557
|
pid: process.pid,
|
|
499
|
-
instanceId
|
|
558
|
+
instanceId,
|
|
500
559
|
processStartTime: processStartTime(process.pid),
|
|
501
560
|
acquiredAt: new Date().toISOString(),
|
|
502
561
|
action: intent.action,
|
|
503
|
-
|
|
562
|
+
requestId,
|
|
504
563
|
};
|
|
505
564
|
|
|
506
565
|
for (let attempt = 0; attempt < 2; attempt += 1) {
|
|
@@ -517,13 +576,31 @@ async function withLock<T>(
|
|
|
517
576
|
if (!existing || !Number.isInteger(existing.pid) || typeof existing.instanceId !== 'string') {
|
|
518
577
|
throw new Error('RECOVERY_OPERATION_LOCK_UNCERTAIN');
|
|
519
578
|
}
|
|
520
|
-
if (recoveryLockOwnerAlive(existing))
|
|
579
|
+
if (recoveryLockOwnerAlive(existing)) {
|
|
580
|
+
assertRecoveryLockOwnerAttributable(config, existing);
|
|
581
|
+
return { acquired: false, owner: existing };
|
|
582
|
+
}
|
|
521
583
|
const latest = json<RecoveryLock>(path);
|
|
522
584
|
if (!latest || latest.instanceId !== existing.instanceId) {
|
|
523
|
-
if (attempt === 1)
|
|
585
|
+
if (attempt === 1) {
|
|
586
|
+
if (latest && recoveryLockOwnerAlive(latest)) {
|
|
587
|
+
assertRecoveryLockOwnerAttributable(config, latest);
|
|
588
|
+
return { acquired: false, owner: latest };
|
|
589
|
+
}
|
|
590
|
+
audit(config, 'recovery_operation_lock_race', {
|
|
591
|
+
existingInstanceId: existing.instanceId,
|
|
592
|
+
existingPid: existing.pid,
|
|
593
|
+
latestInstanceId: latest?.instanceId ?? null,
|
|
594
|
+
latestPid: latest?.pid ?? null,
|
|
595
|
+
});
|
|
596
|
+
throw new Error('RECOVERY_OPERATION_LOCK_RACE');
|
|
597
|
+
}
|
|
524
598
|
continue;
|
|
525
599
|
}
|
|
526
|
-
if (recoveryLockOwnerAlive(latest))
|
|
600
|
+
if (recoveryLockOwnerAlive(latest)) {
|
|
601
|
+
assertRecoveryLockOwnerAttributable(config, latest);
|
|
602
|
+
return { acquired: false, owner: latest };
|
|
603
|
+
}
|
|
527
604
|
try { writeFileSync(`${path}.stale-${Date.now()}-${existing.instanceId}`, readFileSync(path)); } catch { /* evidence best effort */ }
|
|
528
605
|
rmSync(path, { force: true });
|
|
529
606
|
continue;
|
|
@@ -541,7 +618,23 @@ async function withLock<T>(
|
|
|
541
618
|
}
|
|
542
619
|
|
|
543
620
|
export async function runtimeStatus(config: RecoveryConfig) {
|
|
544
|
-
|
|
621
|
+
const observation = observeRuntimeStatus(config.controllerHome);
|
|
622
|
+
const watchdog = loadWatchdogState(config);
|
|
623
|
+
return {
|
|
624
|
+
...observation,
|
|
625
|
+
recoveryWatchdog: {
|
|
626
|
+
lastDecision: watchdog.lastDecision,
|
|
627
|
+
lastReason: watchdog.lastReason,
|
|
628
|
+
updatedAt: watchdog.updatedAt,
|
|
629
|
+
failures: watchdog.failures,
|
|
630
|
+
rollbackUsed: watchdog.rollbackUsed,
|
|
631
|
+
runtimeRestartAttempts: watchdog.runtimeRestartAttempts ?? 0,
|
|
632
|
+
runtimeRestartFailures: watchdog.runtimeRestartFailures ?? 0,
|
|
633
|
+
primaryConnectorFailures: watchdog.primaryConnectorFailures ?? 0,
|
|
634
|
+
primaryConnectorRestartAttempts: watchdog.primaryConnectorRestartAttempts ?? 0,
|
|
635
|
+
primaryConnectorRestartFailures: watchdog.primaryConnectorRestartFailures ?? 0,
|
|
636
|
+
},
|
|
637
|
+
};
|
|
545
638
|
}
|
|
546
639
|
|
|
547
640
|
async function probe(transport: RecoveryHttpTransport, url: string, timeoutMs = 4_000): Promise<{ ok: boolean; detail: string; status?: number }> {
|
|
@@ -583,6 +676,72 @@ async function probeExternalMcp(transport: RecoveryHttpTransport, url: string):
|
|
|
583
676
|
} finally { clearTimeout(timer); }
|
|
584
677
|
}
|
|
585
678
|
|
|
679
|
+
function publicGatewayReadinessEndpoint(endpoint: string): string {
|
|
680
|
+
const url = new URL(endpoint);
|
|
681
|
+
url.pathname = '/transport-ready';
|
|
682
|
+
url.search = '';
|
|
683
|
+
url.hash = '';
|
|
684
|
+
return url.toString();
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
async function probePublicGatewayReadiness(
|
|
688
|
+
transport: RecoveryHttpTransport,
|
|
689
|
+
endpoint: string,
|
|
690
|
+
): Promise<{ ok: boolean; detail: string; status?: number; value?: unknown }> {
|
|
691
|
+
const controller = new AbortController();
|
|
692
|
+
const timer = setTimeout(() => controller.abort('RECOVERY_HTTP_TIMEOUT'), 4_000);
|
|
693
|
+
const surface = 'transport-ready';
|
|
694
|
+
try {
|
|
695
|
+
const response = await transport.request({
|
|
696
|
+
url: publicGatewayReadinessEndpoint(endpoint),
|
|
697
|
+
headers: { accept: 'application/json' },
|
|
698
|
+
timeoutMs: 4_000,
|
|
699
|
+
signal: controller.signal,
|
|
700
|
+
});
|
|
701
|
+
if (response.status === 404 || response.status === 405) {
|
|
702
|
+
// Older Connectors predate the transport-scoped capacity surface. Their
|
|
703
|
+
// whole-control-plane /ready route is intentionally not a transport
|
|
704
|
+
// liveness contract and can depend on the Runtime that Recovery is in the
|
|
705
|
+
// middle of restarting. Falling back to it creates a circular cutover
|
|
706
|
+
// dependency. Verify the actual MCP transport instead: a successful
|
|
707
|
+
// initialize response or the expected OAuth Bearer challenge proves the
|
|
708
|
+
// legacy Connector transport is reachable without weakening 5xx/timeout
|
|
709
|
+
// failure handling.
|
|
710
|
+
const legacy = await probeExternalMcp(transport, endpoint);
|
|
711
|
+
return { ...legacy, detail: `legacy-mcp ${legacy.detail}` };
|
|
712
|
+
}
|
|
713
|
+
let sessionCapacity: unknown;
|
|
714
|
+
try {
|
|
715
|
+
const payload = JSON.parse(response.body) as { sessionCapacity?: unknown };
|
|
716
|
+
sessionCapacity = payload.sessionCapacity;
|
|
717
|
+
} catch { /* malformed readiness payload remains a failed/opaque probe */ }
|
|
718
|
+
const recoveryRecommended = Boolean(
|
|
719
|
+
sessionCapacity
|
|
720
|
+
&& typeof sessionCapacity === 'object'
|
|
721
|
+
&& (sessionCapacity as { recoveryRecommended?: unknown }).recoveryRecommended === true,
|
|
722
|
+
);
|
|
723
|
+
return {
|
|
724
|
+
ok: response.ok && !recoveryRecommended,
|
|
725
|
+
detail: `${surface} HTTP ${response.status}${recoveryRecommended ? '; session capacity recommends Connector recovery' : ''}`,
|
|
726
|
+
status: response.status,
|
|
727
|
+
...(sessionCapacity === undefined ? {} : { value: sessionCapacity }),
|
|
728
|
+
};
|
|
729
|
+
} catch (error) {
|
|
730
|
+
return { ok: false, detail: `${surface} ${error instanceof Error ? error.message.slice(0, 180) : 'request failed'}` };
|
|
731
|
+
} finally {
|
|
732
|
+
clearTimeout(timer);
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
function connectorCapacityRecoveryRecommended(verified: VerifyResult): boolean {
|
|
737
|
+
const value = verified.probes.primary_connector_ready?.value;
|
|
738
|
+
return Boolean(
|
|
739
|
+
value
|
|
740
|
+
&& typeof value === 'object'
|
|
741
|
+
&& (value as { recoveryRecommended?: unknown }).recoveryRecommended === true,
|
|
742
|
+
);
|
|
743
|
+
}
|
|
744
|
+
|
|
586
745
|
function mainToken(config: RecoveryConfig): string | undefined {
|
|
587
746
|
const candidate = config.mainMcpTokenFile ?? join(config.controllerHome, 'mcp', 'mcp.tokens.json');
|
|
588
747
|
const parsed = json<{ bearerToken?: unknown }>(candidate);
|
|
@@ -753,8 +912,26 @@ export async function verifyStableRuntime(
|
|
|
753
912
|
probes.active_gateway = endpoint
|
|
754
913
|
? await probe(transport, runtimeHealthEndpoint(endpoint))
|
|
755
914
|
: { ok: false, detail: 'canonical Runtime endpoint is unavailable' };
|
|
756
|
-
if (config.publicMcpUrl)
|
|
915
|
+
if (config.publicMcpUrl) {
|
|
916
|
+
probes.external_mcp_http = await probeExternalMcp(transport, config.publicMcpUrl);
|
|
917
|
+
const connectorReadinessEndpoint = config.primaryConnectorService?.localMcpUrl?.trim() || config.publicMcpUrl;
|
|
918
|
+
probes.primary_connector_ready = await probePublicGatewayReadiness(transport, connectorReadinessEndpoint);
|
|
919
|
+
}
|
|
920
|
+
const primaryConnectorLocal = await probePrimaryConnectorLocal(config, transport);
|
|
921
|
+
if (primaryConnectorLocal) probes.primary_connector_local = primaryConnectorLocal;
|
|
757
922
|
if (config.gateway) probes.recovery_gateway = await probe(transport, `http://${config.gateway.host}:${config.gateway.port}/health`);
|
|
923
|
+
const watchdogHealth = observeRecoveryWatchdogHealth(config.controllerHome);
|
|
924
|
+
probes.recovery_watchdog = {
|
|
925
|
+
ok: watchdogHealth.ok,
|
|
926
|
+
detail: watchdogHealth.detail,
|
|
927
|
+
value: {
|
|
928
|
+
pulseAgeMs: watchdogHealth.pulseAgeMs,
|
|
929
|
+
tickAgeMs: watchdogHealth.tickAgeMs,
|
|
930
|
+
currentReleaseRevision: watchdogHealth.currentReleaseRevision,
|
|
931
|
+
watchdogReleaseRevision: watchdogHealth.runtimeIdentity?.releaseRevision,
|
|
932
|
+
watchdogPid: watchdogHealth.runtimeIdentity?.pid,
|
|
933
|
+
},
|
|
934
|
+
};
|
|
758
935
|
const recoveryPublicUrl = configuredRecoveryPublicUrl(config);
|
|
759
936
|
if (recoveryPublicUrl) probes.recovery_external_http = await probeExternalMcp(transport, recoveryPublicUrl);
|
|
760
937
|
if (options.probeMcpProtocol !== false) Object.assign(probes, await probeMcp(config, transport));
|
|
@@ -801,10 +978,8 @@ export async function verifyStableRuntime(
|
|
|
801
978
|
return result;
|
|
802
979
|
}
|
|
803
980
|
/** Explicitly records evidence only after the full independent verification passed. */
|
|
804
|
-
|
|
805
|
-
const
|
|
806
|
-
const verified = await verifyStableRuntime(config);
|
|
807
|
-
const authority = releaseAuthority(config);
|
|
981
|
+
function persistVerifiedKnownGood(config: RecoveryConfig, verified: VerifyResult): ReleaseEvidence {
|
|
982
|
+
const authority = releaseAuthority(config);
|
|
808
983
|
const active = verified.releases.active;
|
|
809
984
|
if (
|
|
810
985
|
!verified.ok
|
|
@@ -825,7 +1000,10 @@ export async function attestKnownGood(config: RecoveryConfig): Promise<ReleaseEv
|
|
|
825
1000
|
attestedAt: new Date().toISOString(),
|
|
826
1001
|
};
|
|
827
1002
|
const store = knownGood(config);
|
|
828
|
-
const releases = [
|
|
1003
|
+
const releases = [
|
|
1004
|
+
attested,
|
|
1005
|
+
...store.releases.filter((entry) => entry.path !== active.path && existsSync(entry.path)),
|
|
1006
|
+
].slice(0, 8);
|
|
829
1007
|
writeJson(statePath(config), { schemaVersion: 1, releases, updatedAt: new Date().toISOString() } satisfies KnownGoodStore);
|
|
830
1008
|
audit(config, 'known_good_attested', {
|
|
831
1009
|
revision: active.revision,
|
|
@@ -833,8 +1011,13 @@ export async function attestKnownGood(config: RecoveryConfig): Promise<ReleaseEv
|
|
|
833
1011
|
manifestSha256: active.manifestSha256,
|
|
834
1012
|
releaseAuthorityRevision: authority.revision,
|
|
835
1013
|
});
|
|
836
|
-
|
|
837
|
-
|
|
1014
|
+
return attested;
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
export async function attestKnownGood(config: RecoveryConfig): Promise<ReleaseEvidence> {
|
|
1018
|
+
const locked = await withLock(config, { action: 'attest_known_good' }, async () => (
|
|
1019
|
+
persistVerifiedKnownGood(config, await verifyStableRuntime(config))
|
|
1020
|
+
));
|
|
838
1021
|
if (!locked.acquired) throw new Error(recoveryBusyDetail(locked.owner));
|
|
839
1022
|
return locked.value;
|
|
840
1023
|
}
|
|
@@ -849,8 +1032,8 @@ function quarantine(config: RecoveryConfig, release: ReleaseEvidence | undefined
|
|
|
849
1032
|
async function rollbackPreviousLocked(config: RecoveryConfig, reason: string): Promise<RollbackResult> {
|
|
850
1033
|
const before = await verifyStableRuntime(config);
|
|
851
1034
|
const active = before.releases.active;
|
|
852
|
-
if (matchingKnownGood(config, active)) {
|
|
853
|
-
return { ok: true, noOp: true, detail: 'active whole-Runtime release is
|
|
1035
|
+
if (before.ok && matchingKnownGood(config, active)) {
|
|
1036
|
+
return { ok: true, noOp: true, detail: 'active whole-Runtime release is currently healthy and independently attested known-good', verify: before };
|
|
854
1037
|
}
|
|
855
1038
|
if (before.runtime.running) {
|
|
856
1039
|
return {
|
|
@@ -911,7 +1094,9 @@ export async function reconnectMain(config: RecoveryConfig): Promise<{ ok: boole
|
|
|
911
1094
|
// intentionally a bounded health/reconnect observation, never a rollout.
|
|
912
1095
|
const verified = await verifyStableRuntime(config);
|
|
913
1096
|
const publicProbe = verified.probes.external_mcp_http;
|
|
914
|
-
const ok = verified.probes.active_gateway?.ok === true
|
|
1097
|
+
const ok = verified.probes.active_gateway?.ok === true
|
|
1098
|
+
&& verified.probes.primary_connector_ready?.ok !== false
|
|
1099
|
+
&& (publicProbe?.ok === true || publicProbe?.status === 401);
|
|
915
1100
|
audit(config, 'reconnect_main', { ok, externalStatus: publicProbe?.status });
|
|
916
1101
|
return { ok, detail: ok ? 'canonical Runtime Gateway and primary endpoint are reachable; client session may refresh' : 'primary endpoint remains unavailable; recovery channel remains independent', verify: verified };
|
|
917
1102
|
}
|
|
@@ -1037,6 +1222,16 @@ export function decideWatchdog(input: {
|
|
|
1037
1222
|
rollbackUsed: boolean;
|
|
1038
1223
|
recoveryGatewayFailed?: boolean;
|
|
1039
1224
|
recoveryGatewayRestartUsed?: boolean;
|
|
1225
|
+
primaryConnectorConfigured?: boolean;
|
|
1226
|
+
primaryConnectorFailed?: boolean;
|
|
1227
|
+
primaryConnectorFailures?: number;
|
|
1228
|
+
primaryConnectorFirstFailureAt?: number;
|
|
1229
|
+
primaryConnectorRestartAttempts?: number;
|
|
1230
|
+
primaryConnectorMaximumRestartAttempts?: number;
|
|
1231
|
+
primaryConnectorRestartLastAttemptAt?: number;
|
|
1232
|
+
primaryConnectorRestartCooldownMs?: number;
|
|
1233
|
+
primaryConnectorMinimumFailures?: number;
|
|
1234
|
+
primaryConnectorMinimumFailureDurationMs?: number;
|
|
1040
1235
|
primaryRuntimeFailed?: boolean;
|
|
1041
1236
|
runtimeRestartAttempts?: number;
|
|
1042
1237
|
runtimeMaximumRestartAttempts?: number;
|
|
@@ -1064,6 +1259,22 @@ export function decideWatchdog(input: {
|
|
|
1064
1259
|
if (failures >= minimumFailures && sustained) return { action: 'repair_public_tunnel', reason: 'local runtime and Recovery Gateway are healthy while the dedicated Recovery public endpoint is unavailable' };
|
|
1065
1260
|
return { action: 'degraded', reason: 'Recovery tunnel failure has not yet met the bounded repair threshold' };
|
|
1066
1261
|
}
|
|
1262
|
+
if (input.primaryConnectorConfigured && input.primaryConnectorFailed) {
|
|
1263
|
+
const failures = input.primaryConnectorFailures ?? 0;
|
|
1264
|
+
const minimumFailures = Math.max(1, input.primaryConnectorMinimumFailures ?? 2);
|
|
1265
|
+
const minimumDurationMs = Math.max(0, input.primaryConnectorMinimumFailureDurationMs ?? 5_000);
|
|
1266
|
+
const sustained = input.primaryConnectorFirstFailureAt !== undefined && now - input.primaryConnectorFirstFailureAt >= minimumDurationMs;
|
|
1267
|
+
const attempts = input.primaryConnectorRestartAttempts ?? 0;
|
|
1268
|
+
const maximumAttempts = Math.max(1, input.primaryConnectorMaximumRestartAttempts ?? 3);
|
|
1269
|
+
const cooldownMs = Math.max(0, input.primaryConnectorRestartCooldownMs ?? 30_000);
|
|
1270
|
+
const cooldownElapsed = input.primaryConnectorRestartLastAttemptAt === undefined || now - input.primaryConnectorRestartLastAttemptAt >= cooldownMs;
|
|
1271
|
+
if (failures >= minimumFailures && sustained && attempts < maximumAttempts && cooldownElapsed) {
|
|
1272
|
+
return { action: 'restart_primary_connector', reason: `local Runtime is healthy but the primary Connector endpoint is unavailable; restart attempt ${attempts + 1}/${maximumAttempts}` };
|
|
1273
|
+
}
|
|
1274
|
+
return { action: 'degraded', reason: attempts >= maximumAttempts
|
|
1275
|
+
? `primary Connector restart budget exhausted (${attempts}/${maximumAttempts}); keeping the local Runtime available for independent Recovery`
|
|
1276
|
+
: 'primary Connector failure has not yet met its bounded restart threshold or cooldown' };
|
|
1277
|
+
}
|
|
1067
1278
|
if (input.failures === 0) return { action: 'healthy', reason: 'all recovery probes healthy' };
|
|
1068
1279
|
const recoveryRestartSustained = input.firstFailureAt !== undefined && now - input.firstFailureAt >= 5_000;
|
|
1069
1280
|
if (input.failures >= 2 && recoveryRestartSustained && input.recoveryGatewayFailed && !input.recoveryGatewayRestartUsed) {
|
|
@@ -1220,6 +1431,7 @@ export interface PrimaryRuntimeRecoveryDependencies {
|
|
|
1220
1431
|
runCommand?: CommandRunner;
|
|
1221
1432
|
verifyLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
|
|
1222
1433
|
runtimeRunning?: (config: RecoveryConfig) => boolean;
|
|
1434
|
+
ensureRuntimeLaunchContract?: (controllerHome: string) => void;
|
|
1223
1435
|
now?: () => number;
|
|
1224
1436
|
sleep?: (ms: number) => Promise<void>;
|
|
1225
1437
|
}
|
|
@@ -1253,11 +1465,16 @@ function primaryRuntimeLaunchdService(config: RecoveryConfig, uid: number): Laun
|
|
|
1253
1465
|
}
|
|
1254
1466
|
|
|
1255
1467
|
export interface PrimaryConnectorRecoveryDependencies {
|
|
1468
|
+
/** Durable caller identity propagated into the Recovery mutation lock and audit trail. */
|
|
1469
|
+
requestId?: string;
|
|
1256
1470
|
platform?: NodeJS.Platform;
|
|
1257
1471
|
currentUid?: () => Promise<number | undefined>;
|
|
1258
1472
|
runCommand?: CommandRunner;
|
|
1259
1473
|
verifyLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
|
|
1260
1474
|
reconnect?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string; verify: VerifyResult }>;
|
|
1475
|
+
probeConnectorLocal?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string; status?: number } | undefined>;
|
|
1476
|
+
probeConnectorOwnership?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
|
|
1477
|
+
repairConnectorBinding?: (config: RecoveryConfig) => Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string }>;
|
|
1261
1478
|
now?: () => number;
|
|
1262
1479
|
sleep?: (ms: number) => Promise<void>;
|
|
1263
1480
|
}
|
|
@@ -1272,13 +1489,98 @@ function primaryConnectorLaunchdService(config: RecoveryConfig, uid: number): La
|
|
|
1272
1489
|
return { uid, domain, target: `${domain}/${label}`, label, plistPath };
|
|
1273
1490
|
}
|
|
1274
1491
|
|
|
1492
|
+
async function probePrimaryConnectorLocal(
|
|
1493
|
+
config: RecoveryConfig,
|
|
1494
|
+
transport = createRecoveryHttpTransport(config.controllerHome),
|
|
1495
|
+
): Promise<{ ok: boolean; detail: string; status?: number } | undefined> {
|
|
1496
|
+
const localMcpUrl = config.primaryConnectorService?.localMcpUrl?.trim();
|
|
1497
|
+
if (!localMcpUrl) return undefined;
|
|
1498
|
+
return probeExternalMcp(transport, localMcpUrl);
|
|
1499
|
+
}
|
|
1500
|
+
|
|
1501
|
+
function activePackageConnectorReleaseBinding(config: RecoveryConfig): PackageConnectorReleaseBinding | undefined {
|
|
1502
|
+
const configured = config.primaryConnectorService;
|
|
1503
|
+
if (!configured || configured.platform !== 'launchd' || !configured.localMcpUrl?.trim()) return undefined;
|
|
1504
|
+
const authority = readRuntimeReleaseAuthority(config.controllerHome);
|
|
1505
|
+
if (!authority) return undefined;
|
|
1506
|
+
const releaseRoot = dirname(resolve(authority.active.manifestPath));
|
|
1507
|
+
const packageRoot = join(releaseRoot, 'package');
|
|
1508
|
+
const paths = packageConnectorServicePaths(config.controllerHome);
|
|
1509
|
+
const configuredPlist = resolve(configured.plistPath ?? paths.installedPlistPath);
|
|
1510
|
+
if (configured.label !== paths.label || configuredPlist !== resolve(paths.installedPlistPath)) return undefined;
|
|
1511
|
+
if (!existsSync(join(packageRoot, 'src', 'cli', 'index.ts')) || !existsSync(join(packageRoot, 'src', 'runtime', 'shared', 'node-ts-loader.mjs'))) return undefined;
|
|
1512
|
+
return { releaseId: authority.active.releaseId, releaseRoot, packageRoot };
|
|
1513
|
+
}
|
|
1514
|
+
|
|
1515
|
+
async function repairPrimaryConnectorBinding(config: RecoveryConfig): Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string }> {
|
|
1516
|
+
const release = activePackageConnectorReleaseBinding(config);
|
|
1517
|
+
const endpoint = config.primaryConnectorService?.localMcpUrl?.trim();
|
|
1518
|
+
if (!release || !endpoint) return { ok: true, attempted: false, noOp: true, detail: 'primary Connector does not use the canonical package-release binding contract' };
|
|
1519
|
+
try {
|
|
1520
|
+
const result = await ensurePackageConnectorService({ release, controllerHome: config.controllerHome, endpoint, platform: 'darwin' });
|
|
1521
|
+
const attempted = result.reused !== true;
|
|
1522
|
+
return {
|
|
1523
|
+
ok: true,
|
|
1524
|
+
attempted,
|
|
1525
|
+
...(attempted ? {} : { noOp: true }),
|
|
1526
|
+
detail: attempted
|
|
1527
|
+
? `primary Connector binding repaired to active immutable Runtime release ${release.releaseId}`
|
|
1528
|
+
: `primary Connector binding already matches active immutable Runtime release ${release.releaseId}`,
|
|
1529
|
+
};
|
|
1530
|
+
} catch (error) {
|
|
1531
|
+
return { ok: false, attempted: true, detail: error instanceof Error ? error.message : 'primary Connector binding repair failed' };
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
|
|
1535
|
+
async function probePrimaryConnectorOwnership(
|
|
1536
|
+
config: RecoveryConfig,
|
|
1537
|
+
service: LaunchdService,
|
|
1538
|
+
runCommand: CommandRunner = command,
|
|
1539
|
+
): Promise<{ ok: boolean; detail: string }> {
|
|
1540
|
+
const localMcpUrl = config.primaryConnectorService?.localMcpUrl?.trim();
|
|
1541
|
+
if (!localMcpUrl) return { ok: false, detail: 'primary Connector local MCP endpoint is not configured' };
|
|
1542
|
+
let port: number;
|
|
1543
|
+
try {
|
|
1544
|
+
const parsed = new URL(localMcpUrl);
|
|
1545
|
+
const rawPort = parsed.port || (parsed.protocol === 'https:' ? '443' : parsed.protocol === 'http:' ? '80' : '');
|
|
1546
|
+
port = Number(rawPort);
|
|
1547
|
+
if (!Number.isInteger(port) || port < 1 || port > 65_535) throw new Error('invalid port');
|
|
1548
|
+
} catch {
|
|
1549
|
+
return { ok: false, detail: `primary Connector local MCP endpoint has no verifiable TCP port: ${localMcpUrl}` };
|
|
1550
|
+
}
|
|
1551
|
+
const printed = await runCommand('launchctl', ['print', service.target], 5_000);
|
|
1552
|
+
const pidMatch = printed.ok ? printed.stdout.match(/\bpid\s*=\s*(\d+)/) : null;
|
|
1553
|
+
const pid = pidMatch ? Number(pidMatch[1]) : undefined;
|
|
1554
|
+
if (!pid || !Number.isInteger(pid)) {
|
|
1555
|
+
return { ok: false, detail: `configured primary Connector launchd service has no live pid: ${service.target}` };
|
|
1556
|
+
}
|
|
1557
|
+
const listening = await runCommand('lsof', ['-nP', '-a', '-p', String(pid), `-iTCP:${port}`, '-sTCP:LISTEN', '-t'], 5_000);
|
|
1558
|
+
const listenerPids = listening.stdout.split(/\s+/).map((value) => Number(value)).filter(Number.isInteger);
|
|
1559
|
+
const ok = listening.ok && listenerPids.includes(pid);
|
|
1560
|
+
return {
|
|
1561
|
+
ok,
|
|
1562
|
+
detail: ok
|
|
1563
|
+
? `configured primary Connector pid ${pid} owns TCP ${port}`
|
|
1564
|
+
: `configured primary Connector pid ${pid} does not own TCP ${port}`,
|
|
1565
|
+
};
|
|
1566
|
+
}
|
|
1567
|
+
|
|
1275
1568
|
export async function restartPrimaryConnector(
|
|
1276
1569
|
config: RecoveryConfig,
|
|
1277
1570
|
dependencies: PrimaryConnectorRecoveryDependencies = {},
|
|
1278
1571
|
): Promise<PrimaryConnectorRestartResult> {
|
|
1279
1572
|
const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
|
|
1280
1573
|
const initialLocal = await verifyLocal(config);
|
|
1281
|
-
|
|
1574
|
+
// `verifyLocalRuntime` also reports the Connector's own loopback probe.
|
|
1575
|
+
// That probe is expected to be false when this recovery action is needed,
|
|
1576
|
+
// so it must not prevent a healthy canonical Runtime from repairing the
|
|
1577
|
+
// Connector service.
|
|
1578
|
+
const canonicalRuntimeHealthy = initialLocal.runtime.ok
|
|
1579
|
+
&& initialLocal.runtime.running
|
|
1580
|
+
&& initialLocal.runtime.ready
|
|
1581
|
+
&& !initialLocal.runtime.stale
|
|
1582
|
+
&& initialLocal.probes.active_gateway?.ok === true;
|
|
1583
|
+
if (!canonicalRuntimeHealthy) {
|
|
1282
1584
|
return {
|
|
1283
1585
|
ok: false,
|
|
1284
1586
|
attempted: false,
|
|
@@ -1296,21 +1598,107 @@ export async function restartPrimaryConnector(
|
|
|
1296
1598
|
return { ok: false, attempted: false, noOp: true, detail: 'primary Connector launchd service is not configured or installed', verify: initialLocal };
|
|
1297
1599
|
}
|
|
1298
1600
|
const reconnect = dependencies.reconnect ?? reconnectMain;
|
|
1601
|
+
const probeConnectorLocal = dependencies.probeConnectorLocal ?? probePrimaryConnectorLocal;
|
|
1602
|
+
const runCommand = dependencies.runCommand ?? command;
|
|
1603
|
+
const probeConnectorOwnership = dependencies.probeConnectorOwnership
|
|
1604
|
+
?? ((candidateConfig: RecoveryConfig) => probePrimaryConnectorOwnership(candidateConfig, service, runCommand));
|
|
1299
1605
|
const now = dependencies.now ?? Date.now;
|
|
1300
1606
|
const wait = dependencies.sleep ?? sleep;
|
|
1301
1607
|
const timeoutMs = config.primaryConnectorService?.postRestartVerifyTimeoutMs ?? 30_000;
|
|
1302
|
-
const locked = await withLock(config, { action: 'restart_primary_connector' }, async () => {
|
|
1303
|
-
const
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1608
|
+
const locked = await withLock(config, { action: 'restart_primary_connector', requestId: dependencies.requestId }, async () => {
|
|
1609
|
+
const tunnelConfigured = Boolean(configuredPrimaryPublicTunnel(config));
|
|
1610
|
+
const repairConnectorBinding = dependencies.repairConnectorBinding ?? repairPrimaryConnectorBinding;
|
|
1611
|
+
const bindingRepair = await repairConnectorBinding(config);
|
|
1612
|
+
if (bindingRepair.attempted) {
|
|
1613
|
+
audit(config, bindingRepair.ok ? 'primary_connector_binding_repaired' : 'primary_connector_binding_repair_failed', { detail: bindingRepair.detail });
|
|
1307
1614
|
}
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1615
|
+
if (!bindingRepair.ok) {
|
|
1616
|
+
return { ok: false, attempted: true, detail: `primary Connector immutable release binding repair failed: ${bindingRepair.detail}`, serviceTarget: service.target, verify: initialLocal } satisfies PrimaryConnectorRestartResult;
|
|
1617
|
+
}
|
|
1618
|
+
let localConnector = await probeConnectorLocal(config);
|
|
1619
|
+
let connectorOwnership = localConnector?.ok ? await probeConnectorOwnership(config) : undefined;
|
|
1620
|
+
let observed = localConnector?.ok && connectorOwnership?.ok ? await reconnect(config) : undefined;
|
|
1621
|
+
if (observed?.ok) {
|
|
1622
|
+
return {
|
|
1623
|
+
ok: true,
|
|
1624
|
+
attempted: bindingRepair.attempted,
|
|
1625
|
+
...(bindingRepair.attempted ? {} : { noOp: true }),
|
|
1626
|
+
detail: bindingRepair.attempted
|
|
1627
|
+
? 'primary Connector immutable release binding was repaired and the public MCP endpoint recovered'
|
|
1628
|
+
: 'primary Connector and public MCP endpoint recovered before restart',
|
|
1629
|
+
serviceTarget: service.target,
|
|
1630
|
+
verify: observed.verify,
|
|
1631
|
+
} satisfies PrimaryConnectorRestartResult;
|
|
1632
|
+
}
|
|
1633
|
+
|
|
1634
|
+
// A healthy local OAuth endpoint proves the Connector is not the broken hop.
|
|
1635
|
+
// If a distinct primary tunnel is configured, skip the pointless Gateway
|
|
1636
|
+
// restart and repair the public hop directly.
|
|
1637
|
+
const restartConnector = !localConnector?.ok || !connectorOwnership?.ok || !tunnelConfigured;
|
|
1638
|
+
if (restartConnector) {
|
|
1639
|
+
const restarted = await ensureLaunchdServiceStarted(service, runCommand);
|
|
1640
|
+
if (!restarted.ok) {
|
|
1641
|
+
audit(config, 'primary_connector_restart_failed', { serviceTarget: service.target, detail: restarted.detail });
|
|
1642
|
+
return { ok: false, attempted: true, detail: restarted.detail, serviceTarget: service.target, verify: initialLocal } satisfies PrimaryConnectorRestartResult;
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
const localDeadline = now() + timeoutMs;
|
|
1646
|
+
localConnector = await probeConnectorLocal(config);
|
|
1647
|
+
while (localConnector && !localConnector.ok && now() < localDeadline) {
|
|
1648
|
+
await wait(1_000);
|
|
1649
|
+
localConnector = await probeConnectorLocal(config);
|
|
1650
|
+
}
|
|
1651
|
+
connectorOwnership = localConnector?.ok ? await probeConnectorOwnership(config) : undefined;
|
|
1652
|
+
if (localConnector && (!localConnector.ok || !connectorOwnership?.ok)) {
|
|
1653
|
+
const detail = localConnector.ok
|
|
1654
|
+
? (connectorOwnership?.detail ?? 'configured primary Connector listener ownership could not be verified')
|
|
1655
|
+
: localConnector.detail;
|
|
1656
|
+
audit(config, 'primary_connector_restart_local_unverified', { serviceTarget: service.target, detail });
|
|
1657
|
+
return {
|
|
1658
|
+
ok: false,
|
|
1659
|
+
attempted: true,
|
|
1660
|
+
detail: localConnector.ok
|
|
1661
|
+
? 'primary Connector restarted but the configured launchd service does not own its local MCP listener'
|
|
1662
|
+
: 'primary Connector restarted but its local MCP endpoint did not recover before timeout',
|
|
1663
|
+
serviceTarget: service.target,
|
|
1664
|
+
verify: initialLocal,
|
|
1665
|
+
} satisfies PrimaryConnectorRestartResult;
|
|
1666
|
+
}
|
|
1667
|
+
}
|
|
1668
|
+
|
|
1669
|
+
observed ??= await reconnect(config);
|
|
1670
|
+
if (!observed.ok && localConnector?.ok && connectorOwnership?.ok && configuredPrimaryPublicTunnel(config)) {
|
|
1671
|
+
const tunnel = primaryPublicTunnelService(config, service.uid);
|
|
1672
|
+
if (tunnel) {
|
|
1673
|
+
const tunnelRestarted = await ensureLaunchdServiceStarted(tunnel, dependencies.runCommand ?? command);
|
|
1674
|
+
if (!tunnelRestarted.ok) {
|
|
1675
|
+
audit(config, 'primary_public_tunnel_restart_failed', { serviceTarget: tunnel.target, detail: tunnelRestarted.detail });
|
|
1676
|
+
return {
|
|
1677
|
+
ok: false,
|
|
1678
|
+
attempted: true,
|
|
1679
|
+
detail: `primary Connector is locally healthy but public tunnel restart failed: ${tunnelRestarted.detail}`,
|
|
1680
|
+
serviceTarget: service.target,
|
|
1681
|
+
verify: observed.verify,
|
|
1682
|
+
} satisfies PrimaryConnectorRestartResult;
|
|
1683
|
+
}
|
|
1684
|
+
const tunnelDeadline = now() + (configuredPrimaryPublicTunnel(config)?.postRestartVerifyTimeoutMs ?? 20_000);
|
|
1685
|
+
while (!observed.ok && now() < tunnelDeadline) {
|
|
1686
|
+
await wait(1_000);
|
|
1687
|
+
observed = await reconnect(config);
|
|
1688
|
+
}
|
|
1689
|
+
audit(config, observed.ok ? 'primary_public_tunnel_restart_succeeded' : 'primary_public_tunnel_restart_unverified', {
|
|
1690
|
+
serviceTarget: tunnel.target,
|
|
1691
|
+
detail: observed.detail,
|
|
1692
|
+
});
|
|
1693
|
+
}
|
|
1694
|
+
} else {
|
|
1695
|
+
const deadline = now() + timeoutMs;
|
|
1696
|
+
while (!observed.ok && now() < deadline) {
|
|
1697
|
+
await wait(1_000);
|
|
1698
|
+
observed = await reconnect(config);
|
|
1699
|
+
}
|
|
1313
1700
|
}
|
|
1701
|
+
|
|
1314
1702
|
audit(config, observed.ok ? 'primary_connector_restart_succeeded' : 'primary_connector_restart_unverified', {
|
|
1315
1703
|
serviceTarget: service.target,
|
|
1316
1704
|
detail: observed.detail,
|
|
@@ -1318,7 +1706,9 @@ export async function restartPrimaryConnector(
|
|
|
1318
1706
|
return {
|
|
1319
1707
|
ok: observed.ok,
|
|
1320
1708
|
attempted: true,
|
|
1321
|
-
detail: observed.ok
|
|
1709
|
+
detail: observed.ok
|
|
1710
|
+
? 'primary Connector recovery completed and the public MCP endpoint is reachable'
|
|
1711
|
+
: 'primary Connector recovery completed but the public MCP endpoint did not recover before timeout',
|
|
1322
1712
|
serviceTarget: service.target,
|
|
1323
1713
|
verify: observed.verify,
|
|
1324
1714
|
} satisfies PrimaryConnectorRestartResult;
|
|
@@ -1362,6 +1752,44 @@ async function waitForLaunchdServiceUnloaded(input: {
|
|
|
1362
1752
|
return !printed.ok;
|
|
1363
1753
|
}
|
|
1364
1754
|
|
|
1755
|
+
interface PrimaryRuntimePortObservation {
|
|
1756
|
+
port: number;
|
|
1757
|
+
pids: number[];
|
|
1758
|
+
uncertain?: string;
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1761
|
+
interface PrimaryRuntimePortCleanupResult {
|
|
1762
|
+
released: boolean;
|
|
1763
|
+
cleaned: boolean;
|
|
1764
|
+
detail: string;
|
|
1765
|
+
pid?: number;
|
|
1766
|
+
signal?: 'SIGTERM' | 'SIGKILL';
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
async function observePrimaryRuntimePort(config: RecoveryConfig, runCommand: CommandRunner): Promise<PrimaryRuntimePortObservation> {
|
|
1770
|
+
const paths = forgeRuntimeServicePaths(config.controllerHome);
|
|
1771
|
+
let port: number;
|
|
1772
|
+
try {
|
|
1773
|
+
port = readForgeRuntimeServiceConfig(paths.configPath).port;
|
|
1774
|
+
} catch {
|
|
1775
|
+
return { port: -1, pids: [], uncertain: 'primary Runtime service port could not be read' };
|
|
1776
|
+
}
|
|
1777
|
+
const result = await runCommand('lsof', ['-nP', '-Fp', `-iTCP:${port}`, '-sTCP:LISTEN'], 3_000);
|
|
1778
|
+
if (result.status === 1 && result.stderr.trim().length === 0) return { port, pids: [] };
|
|
1779
|
+
if (!result.ok) {
|
|
1780
|
+
return { port, pids: [], uncertain: `primary Runtime listener observation failed: ${result.stderr || result.stdout || result.status}` };
|
|
1781
|
+
}
|
|
1782
|
+
const pids = [...new Set(result.stdout.split(/\r?\n/)
|
|
1783
|
+
.map((line) => /^p(\d+)$/.exec(line.trim())?.[1])
|
|
1784
|
+
.filter((value): value is string => Boolean(value))
|
|
1785
|
+
.map(Number)
|
|
1786
|
+
.filter((value) => Number.isInteger(value) && value > 0))];
|
|
1787
|
+
if (result.stdout.trim().length > 0 && pids.length === 0) {
|
|
1788
|
+
return { port, pids: [], uncertain: 'primary Runtime listener identity was not machine-readable' };
|
|
1789
|
+
}
|
|
1790
|
+
return { port, pids };
|
|
1791
|
+
}
|
|
1792
|
+
|
|
1365
1793
|
async function waitForPrimaryRuntimePortReleased(input: {
|
|
1366
1794
|
config: RecoveryConfig;
|
|
1367
1795
|
timeoutMs: number;
|
|
@@ -1369,25 +1797,173 @@ async function waitForPrimaryRuntimePortReleased(input: {
|
|
|
1369
1797
|
wait: (ms: number) => Promise<void>;
|
|
1370
1798
|
runCommand: CommandRunner;
|
|
1371
1799
|
}): Promise<boolean> {
|
|
1372
|
-
const paths = forgeRuntimeServicePaths(input.config.controllerHome);
|
|
1373
|
-
let port: number;
|
|
1374
|
-
try {
|
|
1375
|
-
port = readForgeRuntimeServiceConfig(paths.configPath).port;
|
|
1376
|
-
} catch {
|
|
1377
|
-
return false;
|
|
1378
|
-
}
|
|
1379
|
-
const isListening = async (): Promise<boolean> => {
|
|
1380
|
-
const result = await input.runCommand('lsof', ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN'], 3_000);
|
|
1381
|
-
if (result.ok) return result.stdout.trim().length > 0;
|
|
1382
|
-
if (result.status === 1 && result.stderr.trim().length === 0) return false;
|
|
1383
|
-
return true;
|
|
1384
|
-
};
|
|
1385
1800
|
const deadline = input.now() + input.timeoutMs;
|
|
1386
1801
|
while (input.now() < deadline) {
|
|
1387
|
-
|
|
1802
|
+
const observation = await observePrimaryRuntimePort(input.config, input.runCommand);
|
|
1803
|
+
if (!observation.uncertain && observation.pids.length === 0) return true;
|
|
1388
1804
|
await input.wait(250);
|
|
1389
1805
|
}
|
|
1390
|
-
|
|
1806
|
+
const observation = await observePrimaryRuntimePort(input.config, input.runCommand);
|
|
1807
|
+
return !observation.uncertain && observation.pids.length === 0;
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
function commandLineHasArgument(commandLine: string, flag: string, value: string): boolean {
|
|
1811
|
+
const candidates = [
|
|
1812
|
+
`${flag} ${value}`,
|
|
1813
|
+
`${flag} "${value}"`,
|
|
1814
|
+
`${flag} '${value}'`,
|
|
1815
|
+
];
|
|
1816
|
+
return candidates.some((candidate) => {
|
|
1817
|
+
const index = commandLine.indexOf(candidate);
|
|
1818
|
+
if (index < 0) return false;
|
|
1819
|
+
const before = index === 0 ? '' : commandLine[index - 1];
|
|
1820
|
+
const afterIndex = index + candidate.length;
|
|
1821
|
+
const after = afterIndex >= commandLine.length ? '' : commandLine[afterIndex];
|
|
1822
|
+
return (!before || /\s/.test(before)) && (!after || /\s/.test(after));
|
|
1823
|
+
});
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
async function verifyPrimaryRuntimeListenerIdentity(input: {
|
|
1827
|
+
config: RecoveryConfig;
|
|
1828
|
+
pid: number;
|
|
1829
|
+
uid: number;
|
|
1830
|
+
port: number;
|
|
1831
|
+
runCommand: CommandRunner;
|
|
1832
|
+
}): Promise<{ ok: boolean; detail: string }> {
|
|
1833
|
+
const paths = forgeRuntimeServicePaths(input.config.controllerHome);
|
|
1834
|
+
const physicalEntrypoint = activeRuntimeEntrypoint(input.config.controllerHome);
|
|
1835
|
+
const launchSpec = activeRuntimeLaunchSpec(input.config.controllerHome);
|
|
1836
|
+
if (!physicalEntrypoint || !launchSpec) return { ok: false, detail: 'active Runtime release identity is unavailable' };
|
|
1837
|
+
const manifestIndex = launchSpec.args.indexOf('--release-manifest');
|
|
1838
|
+
const manifestPath = manifestIndex >= 0 ? launchSpec.args[manifestIndex + 1] : undefined;
|
|
1839
|
+
if (!manifestPath) return { ok: false, detail: 'active Runtime release manifest identity is unavailable' };
|
|
1840
|
+
|
|
1841
|
+
const result = await input.runCommand('ps', ['-ww', '-p', String(input.pid), '-o', 'uid=', '-o', 'command='], 3_000);
|
|
1842
|
+
if (!result.ok) return { ok: false, detail: `listener process identity could not be read: ${result.stderr || result.stdout || result.status}` };
|
|
1843
|
+
const line = result.stdout.trim().split(/\r?\n/).find(Boolean) ?? '';
|
|
1844
|
+
const match = /^\s*(\d+)\s+(.+)$/.exec(line);
|
|
1845
|
+
if (!match) return { ok: false, detail: 'listener process identity was not machine-readable' };
|
|
1846
|
+
const observedUid = Number(match[1]);
|
|
1847
|
+
const commandLine = match[2]!.trim();
|
|
1848
|
+
if (observedUid !== input.uid) return { ok: false, detail: `listener uid ${observedUid} does not match Recovery uid ${input.uid}` };
|
|
1849
|
+
const executableMatches = [resolve(paths.activeEntrypointPath), resolve(physicalEntrypoint)]
|
|
1850
|
+
.some((candidate) => commandLine === candidate || commandLine.startsWith(`${candidate} `));
|
|
1851
|
+
if (!executableMatches) return { ok: false, detail: 'listener executable is not the active Forge Runtime entrypoint' };
|
|
1852
|
+
if (!commandLineHasArgument(commandLine, '--controller-home', resolve(input.config.controllerHome))) {
|
|
1853
|
+
return { ok: false, detail: 'listener controller-home identity does not match' };
|
|
1854
|
+
}
|
|
1855
|
+
if (!commandLineHasArgument(commandLine, '--port', String(input.port))) {
|
|
1856
|
+
return { ok: false, detail: 'listener port identity does not match' };
|
|
1857
|
+
}
|
|
1858
|
+
if (!commandLineHasArgument(commandLine, '--release-manifest', resolve(manifestPath))) {
|
|
1859
|
+
return { ok: false, detail: 'listener release-manifest identity does not match' };
|
|
1860
|
+
}
|
|
1861
|
+
return { ok: true, detail: 'listener matches the current Forge Runtime release and controller identity' };
|
|
1862
|
+
}
|
|
1863
|
+
|
|
1864
|
+
async function cleanupVerifiedStalePrimaryRuntimeListener(input: {
|
|
1865
|
+
config: RecoveryConfig;
|
|
1866
|
+
uid: number;
|
|
1867
|
+
now: () => number;
|
|
1868
|
+
wait: (ms: number) => Promise<void>;
|
|
1869
|
+
runCommand: CommandRunner;
|
|
1870
|
+
}): Promise<PrimaryRuntimePortCleanupResult> {
|
|
1871
|
+
const initial = await observePrimaryRuntimePort(input.config, input.runCommand);
|
|
1872
|
+
if (initial.uncertain) return { released: false, cleaned: false, detail: initial.uncertain };
|
|
1873
|
+
if (initial.pids.length === 0) return { released: true, cleaned: false, detail: 'primary Runtime port is already released' };
|
|
1874
|
+
if (initial.pids.length !== 1) return { released: false, cleaned: false, detail: `primary Runtime port has ${initial.pids.length} listeners; cleanup requires exactly one` };
|
|
1875
|
+
const pid = initial.pids[0]!;
|
|
1876
|
+
const identity = await verifyPrimaryRuntimeListenerIdentity({ config: input.config, pid, uid: input.uid, port: initial.port, runCommand: input.runCommand });
|
|
1877
|
+
if (!identity.ok) return { released: false, cleaned: false, detail: identity.detail, pid };
|
|
1878
|
+
|
|
1879
|
+
const term = await input.runCommand('kill', ['-TERM', String(pid)], 3_000);
|
|
1880
|
+
const releasedAfterTerm = await waitForPrimaryRuntimePortReleased({ ...input, timeoutMs: 5_000 });
|
|
1881
|
+
if (releasedAfterTerm) return { released: true, cleaned: true, detail: 'verified stale Forge Runtime listener released after SIGTERM', pid, signal: 'SIGTERM' };
|
|
1882
|
+
if (!term.ok) return { released: false, cleaned: false, detail: `verified listener SIGTERM failed: ${term.stderr || term.stdout || term.status}`, pid };
|
|
1883
|
+
|
|
1884
|
+
const beforeKill = await observePrimaryRuntimePort(input.config, input.runCommand);
|
|
1885
|
+
if (beforeKill.uncertain || beforeKill.pids.length !== 1 || beforeKill.pids[0] !== pid) {
|
|
1886
|
+
return { released: false, cleaned: false, detail: 'listener identity changed after SIGTERM; refusing SIGKILL', pid };
|
|
1887
|
+
}
|
|
1888
|
+
const identityBeforeKill = await verifyPrimaryRuntimeListenerIdentity({ config: input.config, pid, uid: input.uid, port: beforeKill.port, runCommand: input.runCommand });
|
|
1889
|
+
if (!identityBeforeKill.ok) return { released: false, cleaned: false, detail: `listener identity changed after SIGTERM: ${identityBeforeKill.detail}`, pid };
|
|
1890
|
+
const killed = await input.runCommand('kill', ['-KILL', String(pid)], 3_000);
|
|
1891
|
+
if (!killed.ok) return { released: false, cleaned: false, detail: `verified listener SIGKILL failed: ${killed.stderr || killed.stdout || killed.status}`, pid };
|
|
1892
|
+
const releasedAfterKill = await waitForPrimaryRuntimePortReleased({ ...input, timeoutMs: 5_000 });
|
|
1893
|
+
return releasedAfterKill
|
|
1894
|
+
? { released: true, cleaned: true, detail: 'verified stale Forge Runtime listener released after SIGKILL', pid, signal: 'SIGKILL' }
|
|
1895
|
+
: { released: false, cleaned: false, detail: 'verified stale Forge Runtime listener remained on the port after SIGKILL', pid };
|
|
1896
|
+
}
|
|
1897
|
+
|
|
1898
|
+
interface PrimaryRuntimeStoppedTransition {
|
|
1899
|
+
ok: boolean;
|
|
1900
|
+
detail: string;
|
|
1901
|
+
staleListenerCleanup?: PrimaryRuntimePortCleanupResult;
|
|
1902
|
+
}
|
|
1903
|
+
|
|
1904
|
+
/**
|
|
1905
|
+
* Stop the one canonical Runtime service and prove that every execution owner
|
|
1906
|
+
* boundary is gone before release authority may change. This is the shared
|
|
1907
|
+
* transition primitive for activation and recovery; callers must not publish
|
|
1908
|
+
* or roll back a release after a partial stop.
|
|
1909
|
+
*/
|
|
1910
|
+
async function stopPrimaryRuntimeForReleaseTransition(input: {
|
|
1911
|
+
config: RecoveryConfig;
|
|
1912
|
+
service: LaunchdService;
|
|
1913
|
+
now: () => number;
|
|
1914
|
+
wait: (ms: number) => Promise<void>;
|
|
1915
|
+
runCommand: CommandRunner;
|
|
1916
|
+
runtimeRunning: (config: RecoveryConfig) => boolean;
|
|
1917
|
+
}): Promise<PrimaryRuntimeStoppedTransition> {
|
|
1918
|
+
const stopped = await input.runCommand('launchctl', ['bootout', input.service.target], 15_000);
|
|
1919
|
+
const stoppedCleanly = stopped.ok || /not found|no such process|could not find service|service is not loaded/i.test(`${stopped.stderr}\n${stopped.stdout}`);
|
|
1920
|
+
if (!stoppedCleanly) {
|
|
1921
|
+
return { ok: false, detail: `primary Runtime bootout failed: ${stopped.stderr || stopped.stdout || stopped.status}` };
|
|
1922
|
+
}
|
|
1923
|
+
const runtimeStopped = await waitForPrimaryRuntimeState({
|
|
1924
|
+
config: input.config,
|
|
1925
|
+
expectedRunning: false,
|
|
1926
|
+
timeoutMs: 20_000,
|
|
1927
|
+
now: input.now,
|
|
1928
|
+
wait: input.wait,
|
|
1929
|
+
runtimeRunning: input.runtimeRunning,
|
|
1930
|
+
});
|
|
1931
|
+
if (!runtimeStopped) return { ok: false, detail: 'primary Runtime owner remained live after bounded launchd bootout' };
|
|
1932
|
+
const serviceUnloaded = await waitForLaunchdServiceUnloaded({
|
|
1933
|
+
service: input.service,
|
|
1934
|
+
timeoutMs: 20_000,
|
|
1935
|
+
now: input.now,
|
|
1936
|
+
wait: input.wait,
|
|
1937
|
+
runCommand: input.runCommand,
|
|
1938
|
+
});
|
|
1939
|
+
if (!serviceUnloaded) return { ok: false, detail: 'primary Runtime launchd service remained loaded after bounded bootout' };
|
|
1940
|
+
|
|
1941
|
+
let portReleased = await waitForPrimaryRuntimePortReleased({
|
|
1942
|
+
config: input.config,
|
|
1943
|
+
timeoutMs: 20_000,
|
|
1944
|
+
now: input.now,
|
|
1945
|
+
wait: input.wait,
|
|
1946
|
+
runCommand: input.runCommand,
|
|
1947
|
+
});
|
|
1948
|
+
let staleListenerCleanup: PrimaryRuntimePortCleanupResult | undefined;
|
|
1949
|
+
if (!portReleased) {
|
|
1950
|
+
staleListenerCleanup = await cleanupVerifiedStalePrimaryRuntimeListener({
|
|
1951
|
+
config: input.config,
|
|
1952
|
+
uid: input.service.uid,
|
|
1953
|
+
now: input.now,
|
|
1954
|
+
wait: input.wait,
|
|
1955
|
+
runCommand: input.runCommand,
|
|
1956
|
+
});
|
|
1957
|
+
portReleased = staleListenerCleanup.released;
|
|
1958
|
+
}
|
|
1959
|
+
if (!portReleased) {
|
|
1960
|
+
return {
|
|
1961
|
+
ok: false,
|
|
1962
|
+
detail: `primary Runtime TCP port remained occupied after bounded launchd bootout: ${staleListenerCleanup?.detail ?? 'listener did not release'}`,
|
|
1963
|
+
staleListenerCleanup,
|
|
1964
|
+
};
|
|
1965
|
+
}
|
|
1966
|
+
return { ok: true, detail: 'primary Runtime service, owner, and TCP listener are stopped', staleListenerCleanup };
|
|
1391
1967
|
}
|
|
1392
1968
|
|
|
1393
1969
|
async function verifyPrimaryRuntimeAfterStart(input: {
|
|
@@ -1406,6 +1982,73 @@ async function verifyPrimaryRuntimeAfterStart(input: {
|
|
|
1406
1982
|
return observed;
|
|
1407
1983
|
}
|
|
1408
1984
|
|
|
1985
|
+
interface PrimaryRuntimeRebindStartResult {
|
|
1986
|
+
ok: boolean;
|
|
1987
|
+
detail: string;
|
|
1988
|
+
verify: VerifyResult;
|
|
1989
|
+
}
|
|
1990
|
+
|
|
1991
|
+
/**
|
|
1992
|
+
* Complete the only safe restart sequence after release authority changes:
|
|
1993
|
+
* rebind the release-pinned launchd contract, start the sole Runtime service,
|
|
1994
|
+
* and require whole-Runtime verification. A started-but-unverified Runtime is
|
|
1995
|
+
* stopped again so callers never leave a restart loop behind.
|
|
1996
|
+
*/
|
|
1997
|
+
async function rebindStartAndVerifyPrimaryRuntime(input: {
|
|
1998
|
+
config: RecoveryConfig;
|
|
1999
|
+
service: LaunchdService;
|
|
2000
|
+
runCommand: CommandRunner;
|
|
2001
|
+
now: () => number;
|
|
2002
|
+
wait: (ms: number) => Promise<void>;
|
|
2003
|
+
verifyLocal: (config: RecoveryConfig) => Promise<VerifyResult>;
|
|
2004
|
+
ensureRuntimeLaunchContract?: (controllerHome: string) => void;
|
|
2005
|
+
beforeStart?: () => void;
|
|
2006
|
+
contractFailureContext?: string;
|
|
2007
|
+
timeoutMs: number;
|
|
2008
|
+
successDetail: string;
|
|
2009
|
+
}): Promise<PrimaryRuntimeRebindStartResult> {
|
|
2010
|
+
try {
|
|
2011
|
+
const ensureRuntimeLaunchContract = input.ensureRuntimeLaunchContract
|
|
2012
|
+
?? ((controllerHome: string) => { ensureForgeRuntimeLaunchAgentContract({ controllerHome, installUserLaunchAgent: true }); });
|
|
2013
|
+
ensureRuntimeLaunchContract(input.config.controllerHome);
|
|
2014
|
+
} catch (error) {
|
|
2015
|
+
return {
|
|
2016
|
+
ok: false,
|
|
2017
|
+
detail: `primary Runtime launchd contract rebuild failed ${input.contractFailureContext ?? 'after release transition'}: ${error instanceof Error ? error.message : String(error)}`,
|
|
2018
|
+
verify: await input.verifyLocal(input.config),
|
|
2019
|
+
};
|
|
2020
|
+
}
|
|
2021
|
+
|
|
2022
|
+
try {
|
|
2023
|
+
input.beforeStart?.();
|
|
2024
|
+
} catch (error) {
|
|
2025
|
+
return {
|
|
2026
|
+
ok: false,
|
|
2027
|
+
detail: `release transition pre-start preparation failed: ${error instanceof Error ? error.message : String(error)}`,
|
|
2028
|
+
verify: await input.verifyLocal(input.config),
|
|
2029
|
+
};
|
|
2030
|
+
}
|
|
2031
|
+
|
|
2032
|
+
const started = await ensureLaunchdServiceStarted(input.service, input.runCommand);
|
|
2033
|
+
if (!started.ok) {
|
|
2034
|
+
return { ok: false, detail: started.detail, verify: await input.verifyLocal(input.config) };
|
|
2035
|
+
}
|
|
2036
|
+
const verify = await verifyPrimaryRuntimeAfterStart({
|
|
2037
|
+
config: input.config,
|
|
2038
|
+
timeoutMs: input.timeoutMs,
|
|
2039
|
+
now: input.now,
|
|
2040
|
+
wait: input.wait,
|
|
2041
|
+
verifyLocal: input.verifyLocal,
|
|
2042
|
+
});
|
|
2043
|
+
if (verify.ok) return { ok: true, detail: input.successDetail, verify };
|
|
2044
|
+
await input.runCommand('launchctl', ['bootout', input.service.target], 15_000);
|
|
2045
|
+
return {
|
|
2046
|
+
ok: false,
|
|
2047
|
+
detail: 'release transition started a Runtime that failed whole-Runtime verification; service was stopped to prevent a restart loop',
|
|
2048
|
+
verify,
|
|
2049
|
+
};
|
|
2050
|
+
}
|
|
2051
|
+
|
|
1409
2052
|
export async function restartPrimaryRuntime(
|
|
1410
2053
|
config: RecoveryConfig,
|
|
1411
2054
|
dependencies: PrimaryRuntimeRecoveryDependencies = {},
|
|
@@ -1469,31 +2112,21 @@ export async function recoverPrimaryRuntime(
|
|
|
1469
2112
|
const before = await verifyLocal(config);
|
|
1470
2113
|
if (before.ok) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime recovered before rollback', serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRecoveryResult;
|
|
1471
2114
|
|
|
1472
|
-
const stopped = await
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
2115
|
+
const stopped = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
|
|
2116
|
+
if (!stopped.ok) {
|
|
2117
|
+
const action = stopped.detail.startsWith('primary Runtime bootout failed')
|
|
2118
|
+
? 'primary_runtime_recovery_stop_failed'
|
|
2119
|
+
: 'primary_runtime_recovery_stop_unverified';
|
|
2120
|
+
audit(config, action, { serviceTarget: service.target, detail: stopped.detail, pid: stopped.staleListenerCleanup?.pid });
|
|
2121
|
+
return { ok: false, attempted: true, detail: stopped.detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
|
|
1478
2122
|
}
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
if (!serviceUnloaded) {
|
|
1487
|
-
const detail = 'primary Runtime launchd service remained loaded after bounded bootout';
|
|
1488
|
-
audit(config, 'primary_runtime_recovery_stop_unverified', { serviceTarget: service.target, detail });
|
|
1489
|
-
return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
|
|
1490
|
-
}
|
|
1491
|
-
|
|
1492
|
-
const portReleased = await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand });
|
|
1493
|
-
if (!portReleased) {
|
|
1494
|
-
const detail = 'primary Runtime TCP port remained occupied after bounded launchd bootout';
|
|
1495
|
-
audit(config, 'primary_runtime_recovery_stop_unverified', { serviceTarget: service.target, detail });
|
|
1496
|
-
return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
|
|
2123
|
+
if (stopped.staleListenerCleanup?.cleaned) {
|
|
2124
|
+
audit(config, 'primary_runtime_recovery_stale_listener_cleaned', {
|
|
2125
|
+
serviceTarget: service.target,
|
|
2126
|
+
pid: stopped.staleListenerCleanup.pid,
|
|
2127
|
+
signal: stopped.staleListenerCleanup.signal,
|
|
2128
|
+
detail: stopped.staleListenerCleanup.detail,
|
|
2129
|
+
});
|
|
1497
2130
|
}
|
|
1498
2131
|
|
|
1499
2132
|
const rollback = await rollbackPreviousLocked(config, reason);
|
|
@@ -1502,26 +2135,30 @@ export async function recoverPrimaryRuntime(
|
|
|
1502
2135
|
return { ok: false, attempted: true, detail: rollback.detail, serviceTarget: service.target, rollback, verify: rollback.verify ?? await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
|
|
1503
2136
|
}
|
|
1504
2137
|
|
|
1505
|
-
const
|
|
1506
|
-
if (!started.ok) {
|
|
1507
|
-
audit(config, 'primary_runtime_recovery_start_failed', { serviceTarget: service.target, detail: started.detail, rollbackOperationId: rollback.operationId });
|
|
1508
|
-
return { ok: false, attempted: true, detail: started.detail, serviceTarget: service.target, rollback, verify: await verifyLocal(config) } satisfies PrimaryRuntimeRecoveryResult;
|
|
1509
|
-
}
|
|
1510
|
-
const after = await verifyPrimaryRuntimeAfterStart({
|
|
2138
|
+
const restarted = await rebindStartAndVerifyPrimaryRuntime({
|
|
1511
2139
|
config,
|
|
1512
|
-
|
|
2140
|
+
service,
|
|
2141
|
+
runCommand,
|
|
1513
2142
|
now,
|
|
1514
2143
|
wait,
|
|
1515
2144
|
verifyLocal,
|
|
2145
|
+
ensureRuntimeLaunchContract: dependencies.ensureRuntimeLaunchContract,
|
|
2146
|
+
contractFailureContext: 'after rollback',
|
|
2147
|
+
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 45_000,
|
|
2148
|
+
successDetail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified',
|
|
1516
2149
|
});
|
|
1517
|
-
if (
|
|
1518
|
-
audit(config, 'primary_runtime_recovery_succeeded', { serviceTarget: service.target, rollbackOperationId: rollback.operationId, restoredRelease:
|
|
1519
|
-
return { ok: true, attempted: true, detail:
|
|
2150
|
+
if (restarted.ok) {
|
|
2151
|
+
audit(config, 'primary_runtime_recovery_succeeded', { serviceTarget: service.target, rollbackOperationId: rollback.operationId, restoredRelease: restarted.verify.releases.active?.revision });
|
|
2152
|
+
return { ok: true, attempted: true, detail: restarted.detail, serviceTarget: service.target, rollback, verify: restarted.verify } satisfies PrimaryRuntimeRecoveryResult;
|
|
1520
2153
|
}
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
2154
|
+
const contractFailure = restarted.detail.startsWith('primary Runtime launchd contract rebuild failed');
|
|
2155
|
+
audit(config, contractFailure ? 'primary_runtime_recovery_launchd_contract_failed' : 'primary_runtime_recovery_unverified', {
|
|
2156
|
+
serviceTarget: service.target,
|
|
2157
|
+
rollbackOperationId: rollback.operationId,
|
|
2158
|
+
detail: restarted.detail,
|
|
2159
|
+
reasonCodes: restarted.verify.runtime.reasonCodes,
|
|
2160
|
+
});
|
|
2161
|
+
return { ok: false, attempted: true, detail: restarted.detail, serviceTarget: service.target, rollback, verify: restarted.verify } satisfies PrimaryRuntimeRecoveryResult;
|
|
1525
2162
|
});
|
|
1526
2163
|
if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), serviceTarget: service.target, verify: initial };
|
|
1527
2164
|
return locked.value;
|
|
@@ -1668,7 +2305,25 @@ export async function activateRuntimeRelease(
|
|
|
1668
2305
|
} satisfies RuntimeReleaseActivationResult;
|
|
1669
2306
|
}
|
|
1670
2307
|
if (current) {
|
|
1671
|
-
|
|
2308
|
+
const behaviorEquivalent = runtimeBehaviorEquivalent(current.active.manifestPath, candidate.manifestPath);
|
|
2309
|
+
let serviceContractMatches = false;
|
|
2310
|
+
if (behaviorEquivalent && !storageMigrationRequired) {
|
|
2311
|
+
try {
|
|
2312
|
+
serviceContractMatches = inspectForgeRuntimeLaunchAgentContract({
|
|
2313
|
+
controllerHome: config.controllerHome,
|
|
2314
|
+
inspectUserLaunchAgent: true,
|
|
2315
|
+
}).matches;
|
|
2316
|
+
} catch (error) {
|
|
2317
|
+
audit(config, 'runtime_release_activation_service_contract_inspection_failed', {
|
|
2318
|
+
operationId,
|
|
2319
|
+
requestId: lockRequestId,
|
|
2320
|
+
activeRevision: current.active.releaseId,
|
|
2321
|
+
candidateRevision: candidate.manifest.releaseId,
|
|
2322
|
+
detail: error instanceof Error ? error.message : String(error),
|
|
2323
|
+
});
|
|
2324
|
+
}
|
|
2325
|
+
}
|
|
2326
|
+
if (behaviorEquivalent && !storageMigrationRequired && serviceContractMatches) {
|
|
1672
2327
|
const verify = await verifyLocal(config);
|
|
1673
2328
|
audit(config, 'runtime_release_activation_behavior_identical', {
|
|
1674
2329
|
operationId,
|
|
@@ -1682,8 +2337,8 @@ export async function activateRuntimeRelease(
|
|
|
1682
2337
|
attempted: false,
|
|
1683
2338
|
noOp: true,
|
|
1684
2339
|
detail: verify.ok
|
|
1685
|
-
? 'candidate Runtime behavior
|
|
1686
|
-
: 'candidate Runtime behavior
|
|
2340
|
+
? 'candidate Runtime behavior and launchd service contract are identical to the active release; restart skipped'
|
|
2341
|
+
: 'candidate Runtime behavior and launchd service contract are identical to the active release, but the active Runtime is unhealthy',
|
|
1687
2342
|
operationId,
|
|
1688
2343
|
verify,
|
|
1689
2344
|
} satisfies RuntimeReleaseActivationResult;
|
|
@@ -1701,30 +2356,22 @@ export async function activateRuntimeRelease(
|
|
|
1701
2356
|
return { ok: false, attempted: false, noOp: true, detail, operationId } satisfies RuntimeReleaseActivationResult;
|
|
1702
2357
|
}
|
|
1703
2358
|
const before = await verifyLocal(config);
|
|
1704
|
-
const stopped = await
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
const runtimeStopped = await waitForPrimaryRuntimeState({ config, expectedRunning: false, timeoutMs: 20_000, now, wait, runtimeRunning });
|
|
1712
|
-
if (!runtimeStopped) {
|
|
1713
|
-
const detail = 'primary Runtime owner remained live after bounded launchd bootout';
|
|
1714
|
-
audit(config, 'runtime_release_activation_stop_unverified', { serviceTarget: service.target, operationId });
|
|
1715
|
-
return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
|
|
2359
|
+
const stopped = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
|
|
2360
|
+
if (!stopped.ok) {
|
|
2361
|
+
const action = stopped.detail.startsWith('primary Runtime bootout failed')
|
|
2362
|
+
? 'runtime_release_activation_stop_failed'
|
|
2363
|
+
: 'runtime_release_activation_stop_unverified';
|
|
2364
|
+
audit(config, action, { serviceTarget: service.target, operationId, detail: stopped.detail, pid: stopped.staleListenerCleanup?.pid });
|
|
2365
|
+
return { ok: false, attempted: true, detail: stopped.detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
|
|
1716
2366
|
}
|
|
1717
|
-
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
const detail = 'primary Runtime TCP port remained occupied after bounded launchd bootout';
|
|
1726
|
-
audit(config, 'runtime_release_activation_stop_unverified', { serviceTarget: service.target, operationId, detail });
|
|
1727
|
-
return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
|
|
2367
|
+
if (stopped.staleListenerCleanup?.cleaned) {
|
|
2368
|
+
audit(config, 'runtime_release_activation_stale_listener_cleaned', {
|
|
2369
|
+
serviceTarget: service.target,
|
|
2370
|
+
operationId,
|
|
2371
|
+
pid: stopped.staleListenerCleanup.pid,
|
|
2372
|
+
signal: stopped.staleListenerCleanup.signal,
|
|
2373
|
+
detail: stopped.staleListenerCleanup.detail,
|
|
2374
|
+
});
|
|
1728
2375
|
}
|
|
1729
2376
|
let committed: RuntimeReleaseAuthority;
|
|
1730
2377
|
try {
|
|
@@ -1739,73 +2386,59 @@ export async function activateRuntimeRelease(
|
|
|
1739
2386
|
audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId });
|
|
1740
2387
|
return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
|
|
1741
2388
|
}
|
|
1742
|
-
try {
|
|
1743
|
-
ensureForgeRuntimeLaunchAgentContract({ controllerHome: config.controllerHome, installUserLaunchAgent: true });
|
|
1744
|
-
} catch (error) {
|
|
1745
|
-
const detail = `runtime launchd stable-release contract update failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
1746
|
-
audit(config, 'runtime_release_activation_launchd_contract_failed', { serviceTarget: service.target, operationId, detail });
|
|
1747
|
-
return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
|
|
1748
|
-
}
|
|
1749
2389
|
let storageMigration: ControllerHomeStorageMigration | undefined;
|
|
1750
|
-
|
|
2390
|
+
const activated = await rebindStartAndVerifyPrimaryRuntime({
|
|
2391
|
+
config,
|
|
2392
|
+
service,
|
|
2393
|
+
runCommand,
|
|
2394
|
+
now,
|
|
2395
|
+
wait,
|
|
2396
|
+
verifyLocal,
|
|
2397
|
+
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
|
|
2398
|
+
successDetail: 'requested Runtime release started and passed whole-Runtime verification',
|
|
2399
|
+
beforeStart: () => {
|
|
2400
|
+
storageMigration = migrateStoppedRepoLocalControllerHomeStorage(config.controllerHome, platform);
|
|
2401
|
+
if (storageMigration.migrated) {
|
|
2402
|
+
audit(config, 'runtime_controller_home_noindex_migrated', {
|
|
2403
|
+
serviceTarget: service.target,
|
|
2404
|
+
operationId,
|
|
2405
|
+
logicalHome: storageMigration.logicalHome,
|
|
2406
|
+
physicalHome: storageMigration.physicalHome,
|
|
2407
|
+
});
|
|
2408
|
+
}
|
|
2409
|
+
},
|
|
2410
|
+
});
|
|
2411
|
+
let after = activated.verify;
|
|
1751
2412
|
let activationFailureDetail: string | undefined;
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
audit(config, 'runtime_controller_home_noindex_migrated', {
|
|
1756
|
-
serviceTarget: service.target,
|
|
1757
|
-
operationId,
|
|
1758
|
-
logicalHome: storageMigration.logicalHome,
|
|
1759
|
-
physicalHome: storageMigration.physicalHome,
|
|
1760
|
-
});
|
|
1761
|
-
}
|
|
1762
|
-
} catch (error) {
|
|
1763
|
-
activationFailureDetail = `Controller Home .noindex migration failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
1764
|
-
audit(config, 'runtime_controller_home_noindex_migration_failed', { serviceTarget: service.target, operationId, detail: activationFailureDetail });
|
|
2413
|
+
if (activated.ok && after.releases.active?.revision === candidate.manifest.releaseId) {
|
|
2414
|
+
audit(config, 'runtime_release_activation_succeeded', { serviceTarget: service.target, operationId, requestId: lockRequestId, activeRevision: after.releases.active?.revision, expectedAuthorityRevision: guard.expectedAuthorityRevision, expectedActiveReleaseId: guard.expectedActiveReleaseId, controllerHomeStorageMigrated: storageMigration?.migrated === true });
|
|
2415
|
+
return { ok: true, attempted: true, detail: storageMigration?.migrated ? 'requested Runtime release activated, Controller Home migrated to .noindex storage, and whole-Runtime verification passed' : 'requested Runtime release activated and passed whole-Runtime verification', serviceTarget: service.target, operationId, verify: after } satisfies RuntimeReleaseActivationResult;
|
|
1765
2416
|
}
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
2417
|
+
if (!activated.ok) {
|
|
2418
|
+
activationFailureDetail = activated.detail;
|
|
2419
|
+
const auditAction = activated.detail.startsWith('primary Runtime launchd contract rebuild failed')
|
|
2420
|
+
? 'runtime_release_activation_launchd_contract_failed'
|
|
2421
|
+
: activated.detail.startsWith('release transition pre-start preparation failed')
|
|
2422
|
+
? 'runtime_controller_home_noindex_migration_failed'
|
|
2423
|
+
: activated.detail.includes('start')
|
|
2424
|
+
? 'runtime_release_activation_start_failed'
|
|
2425
|
+
: 'runtime_release_activation_unverified';
|
|
2426
|
+
audit(config, auditAction, { serviceTarget: service.target, operationId, detail: activated.detail });
|
|
1769
2427
|
} else {
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
activationFailureDetail = started.detail;
|
|
1773
|
-
audit(config, 'runtime_release_activation_start_failed', { serviceTarget: service.target, operationId, detail: started.detail });
|
|
1774
|
-
after = await verifyLocal(config);
|
|
1775
|
-
} else {
|
|
1776
|
-
after = await verifyPrimaryRuntimeAfterStart({
|
|
1777
|
-
config,
|
|
1778
|
-
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
|
|
1779
|
-
now,
|
|
1780
|
-
wait,
|
|
1781
|
-
verifyLocal,
|
|
1782
|
-
});
|
|
1783
|
-
if (after.ok && after.releases.active?.revision === candidate.manifest.releaseId) {
|
|
1784
|
-
audit(config, 'runtime_release_activation_succeeded', { serviceTarget: service.target, operationId, requestId: lockRequestId, activeRevision: after.releases.active?.revision, expectedAuthorityRevision: guard.expectedAuthorityRevision, expectedActiveReleaseId: guard.expectedActiveReleaseId, controllerHomeStorageMigrated: storageMigration?.migrated === true });
|
|
1785
|
-
return { ok: true, attempted: true, detail: storageMigration?.migrated ? 'requested Runtime release activated, Controller Home migrated to .noindex storage, and whole-Runtime verification passed' : 'requested Runtime release activated and passed whole-Runtime verification', serviceTarget: service.target, operationId, verify: after } satisfies RuntimeReleaseActivationResult;
|
|
1786
|
-
}
|
|
1787
|
-
}
|
|
2428
|
+
activationFailureDetail = `activated Runtime release identity mismatch: expected ${candidate.manifest.releaseId}, observed ${after.releases.active?.revision ?? 'none'}`;
|
|
2429
|
+
audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId, detail: activationFailureDetail });
|
|
1788
2430
|
}
|
|
1789
2431
|
|
|
1790
2432
|
// Activation failed: stop, restore the previous whole release and its SQLite
|
|
1791
2433
|
// backup, restart the one service, and require verification again.
|
|
1792
|
-
await runCommand('launchctl', ['bootout', service.target], 15_000);
|
|
1793
2434
|
let rollback: RollbackResult;
|
|
1794
|
-
const
|
|
1795
|
-
|
|
1796
|
-
? await waitForLaunchdServiceUnloaded({ service, timeoutMs: 20_000, now, wait, runCommand })
|
|
1797
|
-
: false;
|
|
1798
|
-
const rollbackPortReleased = rollbackRuntimeStopped && rollbackServiceUnloaded
|
|
1799
|
-
? await waitForPrimaryRuntimePortReleased({ config, timeoutMs: 20_000, now, wait, runCommand })
|
|
1800
|
-
: false;
|
|
1801
|
-
if (!rollbackRuntimeStopped || !rollbackServiceUnloaded || !rollbackPortReleased) {
|
|
2435
|
+
const rollbackStop = await stopPrimaryRuntimeForReleaseTransition({ config, service, now, wait, runCommand, runtimeRunning });
|
|
2436
|
+
if (!rollbackStop.ok) {
|
|
1802
2437
|
rollback = {
|
|
1803
2438
|
ok: false,
|
|
1804
|
-
detail:
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
? 'candidate Runtime launchd service remained loaded after bounded rollback bootout'
|
|
1808
|
-
: 'candidate Runtime TCP port remained occupied after bounded rollback bootout',
|
|
2439
|
+
detail: rollbackStop.detail
|
|
2440
|
+
.replace('primary Runtime', 'candidate Runtime')
|
|
2441
|
+
.replace('bounded launchd bootout', 'bounded rollback bootout'),
|
|
1809
2442
|
};
|
|
1810
2443
|
} else {
|
|
1811
2444
|
try {
|
|
@@ -1821,7 +2454,6 @@ export async function activateRuntimeRelease(
|
|
|
1821
2454
|
physicalHome: storageMigration.physicalHome,
|
|
1822
2455
|
});
|
|
1823
2456
|
}
|
|
1824
|
-
ensureForgeRuntimeLaunchAgentContract({ controllerHome: config.controllerHome, installUserLaunchAgent: true });
|
|
1825
2457
|
const previousRevision = previousActive?.releaseId ?? '';
|
|
1826
2458
|
const previousIdentity = previousActive?.artifactIdentity;
|
|
1827
2459
|
if (
|
|
@@ -1830,24 +2462,18 @@ export async function activateRuntimeRelease(
|
|
|
1830
2462
|
) {
|
|
1831
2463
|
throw new Error('RECOVERY_RUNTIME_RELEASE_ROLLBACK_AUTHORITY_MISMATCH');
|
|
1832
2464
|
}
|
|
1833
|
-
const restarted = await
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
rollback = { ok: true, operationId: rollbackOperationId, detail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified', verify: rolledBack };
|
|
1846
|
-
} else {
|
|
1847
|
-
await runCommand('launchctl', ['bootout', service.target], 15_000);
|
|
1848
|
-
rollback = { ok: false, operationId: rollbackOperationId, detail: 'previous whole-Runtime release started but failed verification; service was stopped to prevent a restart loop', verify: rolledBack };
|
|
1849
|
-
}
|
|
1850
|
-
}
|
|
2465
|
+
const restarted = await rebindStartAndVerifyPrimaryRuntime({
|
|
2466
|
+
config,
|
|
2467
|
+
service,
|
|
2468
|
+
runCommand,
|
|
2469
|
+
now,
|
|
2470
|
+
wait,
|
|
2471
|
+
verifyLocal,
|
|
2472
|
+
contractFailureContext: 'after rollback',
|
|
2473
|
+
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
|
|
2474
|
+
successDetail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified',
|
|
2475
|
+
});
|
|
2476
|
+
rollback = { ok: restarted.ok, operationId: rollbackOperationId, detail: restarted.detail, verify: restarted.verify };
|
|
1851
2477
|
} catch (error) {
|
|
1852
2478
|
rollback = { ok: false, detail: error instanceof Error ? error.message : 'previous whole-Runtime release rollback failed' };
|
|
1853
2479
|
}
|
|
@@ -1896,15 +2522,26 @@ export async function stageAndActivateConfiguredRuntimeRelease(
|
|
|
1896
2522
|
// Capture the base before staging. Staging can take long enough for another
|
|
1897
2523
|
// activation to win; the later publish must prove this base is still current.
|
|
1898
2524
|
const expectedAuthority = releaseAuthority(config);
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
2525
|
+
const stagedAttempt = await withLock(config, {
|
|
2526
|
+
action: 'stage_and_activate_runtime_release',
|
|
2527
|
+
...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
|
|
2528
|
+
}, async () => {
|
|
2529
|
+
try {
|
|
2530
|
+
const staged = (dependencies.stage ?? stageRuntimeReleaseFromCandidateSource)({ controllerHome: config.controllerHome, sourceRoot });
|
|
2531
|
+
assertRuntimeReleaseFiles(staged);
|
|
2532
|
+
return { ok: true as const, staged };
|
|
2533
|
+
} catch (error) {
|
|
2534
|
+
return { ok: false as const, detail: error instanceof Error ? error.message : 'Runtime release staging failed' };
|
|
2535
|
+
}
|
|
2536
|
+
});
|
|
2537
|
+
if (!stagedAttempt.acquired) {
|
|
2538
|
+
return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(stagedAttempt.owner) };
|
|
1907
2539
|
}
|
|
2540
|
+
if (!stagedAttempt.value.ok) {
|
|
2541
|
+
audit(config, 'runtime_release_stage_failed', { sourceRoot, detail: stagedAttempt.value.detail });
|
|
2542
|
+
return { ok: false, attempted: false, noOp: true, detail: stagedAttempt.value.detail };
|
|
2543
|
+
}
|
|
2544
|
+
const staged: StagedRuntimeRelease = stagedAttempt.value.staged;
|
|
1908
2545
|
const activationGuard: RuntimeReleaseActivationGuard = {
|
|
1909
2546
|
...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
|
|
1910
2547
|
expectedAuthorityRevision: expectedAuthority?.revision ?? null,
|
|
@@ -1933,8 +2570,7 @@ export async function stageAndActivateConfiguredRuntimeRelease(
|
|
|
1933
2570
|
};
|
|
1934
2571
|
}
|
|
1935
2572
|
|
|
1936
|
-
function
|
|
1937
|
-
const configured = configuredRecoveryTunnel(config);
|
|
2573
|
+
function tunnelLaunchdService(configured: PublicTunnelServiceConfig | undefined, uid: number): LaunchdService | undefined {
|
|
1938
2574
|
if (!configured || configured.platform !== 'launchd') return undefined;
|
|
1939
2575
|
if (!/^com\.[A-Za-z0-9._-]{1,180}$/.test(configured.label)) return undefined;
|
|
1940
2576
|
const plistPath = configured.plistPath;
|
|
@@ -1948,6 +2584,14 @@ function recoveryTunnelService(config: RecoveryConfig, uid: number): LaunchdServ
|
|
|
1948
2584
|
};
|
|
1949
2585
|
}
|
|
1950
2586
|
|
|
2587
|
+
function recoveryTunnelService(config: RecoveryConfig, uid: number): LaunchdService | undefined {
|
|
2588
|
+
return tunnelLaunchdService(configuredRecoveryTunnel(config), uid);
|
|
2589
|
+
}
|
|
2590
|
+
|
|
2591
|
+
function primaryPublicTunnelService(config: RecoveryConfig, uid: number): LaunchdService | undefined {
|
|
2592
|
+
return tunnelLaunchdService(configuredPrimaryPublicTunnel(config), uid);
|
|
2593
|
+
}
|
|
2594
|
+
|
|
1951
2595
|
function tunnelRepairAllowed(config: RecoveryConfig, now: number): boolean {
|
|
1952
2596
|
const cooldownMs = configuredRecoveryTunnel(config)?.cooldownMs ?? 60_000;
|
|
1953
2597
|
const prior = json<{ lastAttemptAt?: unknown }>(publicTunnelRepairStatePath(config));
|
|
@@ -2015,17 +2659,32 @@ export interface RecoveryGatewayRestartResult {
|
|
|
2015
2659
|
serviceTarget?: string;
|
|
2016
2660
|
}
|
|
2017
2661
|
|
|
2018
|
-
export interface
|
|
2662
|
+
export interface RecoveryWatchdogRestartResult {
|
|
2663
|
+
ok: boolean;
|
|
2664
|
+
attempted: boolean;
|
|
2665
|
+
noOp?: boolean;
|
|
2666
|
+
detail: string;
|
|
2667
|
+
serviceTarget?: string;
|
|
2668
|
+
}
|
|
2669
|
+
|
|
2670
|
+
interface RecoveryRoleRestartDependencies {
|
|
2019
2671
|
platform?: NodeJS.Platform;
|
|
2020
2672
|
currentUid?: () => Promise<number | undefined>;
|
|
2021
2673
|
runCommand?: CommandRunner;
|
|
2022
|
-
probeGateway?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
|
|
2023
2674
|
now?: () => number;
|
|
2024
2675
|
sleep?: (ms: number) => Promise<void>;
|
|
2025
2676
|
}
|
|
2026
2677
|
|
|
2027
|
-
|
|
2028
|
-
|
|
2678
|
+
export interface RecoveryGatewayRestartDependencies extends RecoveryRoleRestartDependencies {
|
|
2679
|
+
probeGateway?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
|
|
2680
|
+
}
|
|
2681
|
+
|
|
2682
|
+
export interface RecoveryWatchdogRestartDependencies extends RecoveryRoleRestartDependencies {
|
|
2683
|
+
probeWatchdog?: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
|
|
2684
|
+
}
|
|
2685
|
+
|
|
2686
|
+
function recoveryRoleLaunchdService(config: RecoveryConfig, uid: number, role: 'gateway' | 'watchdog'): LaunchdService {
|
|
2687
|
+
const label = role === 'gateway' ? RECOVERY_GATEWAY_LABEL : RECOVERY_WATCHDOG_LABEL;
|
|
2029
2688
|
const generated = join(recoveryRoot(config), 'launchd', `${label}.plist`);
|
|
2030
2689
|
const installed = join(homedir(), 'Library', 'LaunchAgents', `${label}.plist`);
|
|
2031
2690
|
return {
|
|
@@ -2042,46 +2701,81 @@ async function probeRecoveryGateway(config: RecoveryConfig): Promise<{ ok: boole
|
|
|
2042
2701
|
return probe(createRecoveryHttpTransport(config.controllerHome), `http://${config.gateway.host}:${config.gateway.port}/health`);
|
|
2043
2702
|
}
|
|
2044
2703
|
|
|
2045
|
-
|
|
2046
|
-
config
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2704
|
+
async function probeRecoveryWatchdog(config: RecoveryConfig): Promise<{ ok: boolean; detail: string }> {
|
|
2705
|
+
const health = observeRecoveryWatchdogHealth(config.controllerHome);
|
|
2706
|
+
return { ok: health.ok, detail: health.detail };
|
|
2707
|
+
}
|
|
2708
|
+
|
|
2709
|
+
async function restartRecoveryRole(input: {
|
|
2710
|
+
config: RecoveryConfig;
|
|
2711
|
+
role: 'gateway' | 'watchdog';
|
|
2712
|
+
action: 'restart_recovery_gateway' | 'restart_recovery_watchdog';
|
|
2713
|
+
check: (config: RecoveryConfig) => Promise<{ ok: boolean; detail: string }>;
|
|
2714
|
+
dependencies: RecoveryRoleRestartDependencies;
|
|
2715
|
+
}): Promise<RecoveryGatewayRestartResult> {
|
|
2716
|
+
const display = input.role === 'gateway' ? 'Recovery Gateway' : 'Recovery Watchdog';
|
|
2717
|
+
if ((input.dependencies.platform ?? process.platform) !== 'darwin') return { ok: false, attempted: false, noOp: true, detail: `${display} launchd restart is only supported on macOS` };
|
|
2718
|
+
const initial = await input.check(input.config);
|
|
2719
|
+
if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: `${display} is already healthy` };
|
|
2720
|
+
const uid = await (input.dependencies.currentUid ?? currentUid)();
|
|
2721
|
+
if (uid === undefined) return { ok: false, attempted: false, noOp: true, detail: `${display} launchd UID is unavailable` };
|
|
2722
|
+
const service = recoveryRoleLaunchdService(input.config, uid, input.role);
|
|
2723
|
+
if (!existsSync(service.plistPath)) return { ok: false, attempted: false, noOp: true, detail: `${display} launchd plist is missing: ${service.plistPath}`, serviceTarget: service.target };
|
|
2724
|
+
const eventPrefix = input.role === 'gateway' ? 'recovery_gateway' : 'recovery_watchdog';
|
|
2725
|
+
const locked = await withLock(input.config, { action: input.action }, async () => {
|
|
2726
|
+
const before = await input.check(input.config);
|
|
2727
|
+
if (before.ok) return { ok: true, attempted: false, noOp: true, detail: `${display} recovered before restart`, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
|
|
2728
|
+
const started = await ensureLaunchdServiceStarted(service, input.dependencies.runCommand ?? command);
|
|
2062
2729
|
if (!started.ok) {
|
|
2063
|
-
audit(config,
|
|
2730
|
+
audit(input.config, `${eventPrefix}_restart_failed`, { serviceTarget: service.target, detail: started.detail });
|
|
2064
2731
|
return { ok: false, attempted: true, detail: started.detail, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
|
|
2065
2732
|
}
|
|
2066
|
-
const now = dependencies.now ?? Date.now;
|
|
2067
|
-
const wait = dependencies.sleep ?? sleep;
|
|
2733
|
+
const now = input.dependencies.now ?? Date.now;
|
|
2734
|
+
const wait = input.dependencies.sleep ?? sleep;
|
|
2068
2735
|
const deadline = now() + 20_000;
|
|
2069
2736
|
let observed = before;
|
|
2070
2737
|
while (now() < deadline) {
|
|
2071
2738
|
await wait(1_000);
|
|
2072
|
-
observed = await check(config);
|
|
2739
|
+
observed = await input.check(input.config);
|
|
2073
2740
|
if (observed.ok) {
|
|
2074
|
-
audit(config,
|
|
2075
|
-
return { ok: true, attempted: true, detail:
|
|
2741
|
+
audit(input.config, `${eventPrefix}_restart_succeeded`, { serviceTarget: service.target });
|
|
2742
|
+
return { ok: true, attempted: true, detail: `${display} restarted and passed local health verification`, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
|
|
2076
2743
|
}
|
|
2077
2744
|
}
|
|
2078
|
-
audit(config,
|
|
2079
|
-
return { ok: false, attempted: true, detail:
|
|
2745
|
+
audit(input.config, `${eventPrefix}_restart_unverified`, { serviceTarget: service.target, detail: observed.detail });
|
|
2746
|
+
return { ok: false, attempted: true, detail: `${display} restarted but did not pass local health verification before timeout`, serviceTarget: service.target } satisfies RecoveryGatewayRestartResult;
|
|
2080
2747
|
});
|
|
2081
2748
|
if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), serviceTarget: service.target };
|
|
2082
2749
|
return locked.value;
|
|
2083
2750
|
}
|
|
2084
2751
|
|
|
2752
|
+
export async function restartRecoveryGateway(
|
|
2753
|
+
config: RecoveryConfig,
|
|
2754
|
+
dependencies: RecoveryGatewayRestartDependencies = {},
|
|
2755
|
+
): Promise<RecoveryGatewayRestartResult> {
|
|
2756
|
+
if (!config.gateway) return { ok: false, attempted: false, noOp: true, detail: 'Recovery Gateway is not configured' };
|
|
2757
|
+
return restartRecoveryRole({
|
|
2758
|
+
config,
|
|
2759
|
+
role: 'gateway',
|
|
2760
|
+
action: 'restart_recovery_gateway',
|
|
2761
|
+
check: dependencies.probeGateway ?? probeRecoveryGateway,
|
|
2762
|
+
dependencies,
|
|
2763
|
+
});
|
|
2764
|
+
}
|
|
2765
|
+
|
|
2766
|
+
export async function restartRecoveryWatchdog(
|
|
2767
|
+
config: RecoveryConfig,
|
|
2768
|
+
dependencies: RecoveryWatchdogRestartDependencies = {},
|
|
2769
|
+
): Promise<RecoveryWatchdogRestartResult> {
|
|
2770
|
+
return restartRecoveryRole({
|
|
2771
|
+
config,
|
|
2772
|
+
role: 'watchdog',
|
|
2773
|
+
action: 'restart_recovery_watchdog',
|
|
2774
|
+
check: dependencies.probeWatchdog ?? probeRecoveryWatchdog,
|
|
2775
|
+
dependencies,
|
|
2776
|
+
});
|
|
2777
|
+
}
|
|
2778
|
+
|
|
2085
2779
|
function pidAlive(pid: number | undefined): boolean {
|
|
2086
2780
|
if (typeof pid !== 'number' || !Number.isInteger(pid) || pid <= 0) return false;
|
|
2087
2781
|
try { process.kill(pid, 0); return true; } catch { return false; }
|
|
@@ -2101,7 +2795,7 @@ export function gatewayToken(config: RecoveryConfig): string | undefined {
|
|
|
2101
2795
|
export function initializeStandaloneRecovery(
|
|
2102
2796
|
controllerHome: string,
|
|
2103
2797
|
port = 8787,
|
|
2104
|
-
extensions: Partial<Pick<RecoveryConfig, 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
|
|
2798
|
+
extensions: Partial<Pick<RecoveryConfig, 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryPublicTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
|
|
2105
2799
|
): RecoveryConfig {
|
|
2106
2800
|
const root = resolve(controllerHome);
|
|
2107
2801
|
const tokenPath = join(root, 'recovery', 'config', 'gateway-token.json');
|
|
@@ -2128,6 +2822,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
2128
2822
|
rollback?: RollbackResult;
|
|
2129
2823
|
publicTunnelRepair?: PublicTunnelRepairResult;
|
|
2130
2824
|
recoveryGatewayRestart?: RecoveryGatewayRestartResult;
|
|
2825
|
+
primaryConnectorRestart?: PrimaryConnectorRestartResult;
|
|
2131
2826
|
primaryRuntimeRestart?: PrimaryRuntimeRestartResult;
|
|
2132
2827
|
primaryRuntimeRecovery?: PrimaryRuntimeRecoveryResult;
|
|
2133
2828
|
}> {
|
|
@@ -2160,6 +2855,35 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
2160
2855
|
fullVerificationPerformed = true;
|
|
2161
2856
|
}
|
|
2162
2857
|
const lastFullVerifyAt = fullVerificationPerformed ? now : scopedPrior.lastFullVerifyAt;
|
|
2858
|
+
const activeMutation = liveRecoveryMutationLock(config);
|
|
2859
|
+
if (activeMutation) {
|
|
2860
|
+
const attributable = recoveryLockOwnerAttributable(activeMutation);
|
|
2861
|
+
if (!attributable) {
|
|
2862
|
+
audit(config, 'recovery_operation_lock_identity_uncertain', {
|
|
2863
|
+
pid: activeMutation.pid,
|
|
2864
|
+
instanceId: activeMutation.instanceId,
|
|
2865
|
+
action: activeMutation.action ?? null,
|
|
2866
|
+
requestId: activeMutation.requestId ?? null,
|
|
2867
|
+
acquiredAt: activeMutation.acquiredAt,
|
|
2868
|
+
});
|
|
2869
|
+
}
|
|
2870
|
+
const reason = attributable
|
|
2871
|
+
? `Recovery mutation in progress: action=${activeMutation.action} request=${activeMutation.requestId}; watchdog repair escalation suppressed`
|
|
2872
|
+
: 'Recovery mutation lock identity is uncertain; watchdog repair escalation suppressed';
|
|
2873
|
+
const state: WatchdogState = {
|
|
2874
|
+
...scopedPrior,
|
|
2875
|
+
failures: 0,
|
|
2876
|
+
firstFailureAt: undefined,
|
|
2877
|
+
publicTunnelFailures: 0,
|
|
2878
|
+
publicTunnelFirstFailureAt: undefined,
|
|
2879
|
+
primaryConnectorFailures: 0,
|
|
2880
|
+
primaryConnectorFirstFailureAt: undefined,
|
|
2881
|
+
lastFullVerifyAt,
|
|
2882
|
+
lastDecision: 'degraded',
|
|
2883
|
+
lastReason: reason,
|
|
2884
|
+
};
|
|
2885
|
+
return { state, decision: { action: 'degraded', reason }, verify: verified };
|
|
2886
|
+
}
|
|
2163
2887
|
if (runtimeStartupGrace && !localVerify.ok) {
|
|
2164
2888
|
const state: WatchdogState = {
|
|
2165
2889
|
...scopedPrior,
|
|
@@ -2168,6 +2892,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
2168
2892
|
runtimeHealthySince: undefined,
|
|
2169
2893
|
lastFullVerifyAt,
|
|
2170
2894
|
lastDecision: 'degraded',
|
|
2895
|
+
lastReason: 'canonical Runtime is within startup grace; restart escalation suppressed',
|
|
2171
2896
|
};
|
|
2172
2897
|
return {
|
|
2173
2898
|
state,
|
|
@@ -2177,8 +2902,46 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
2177
2902
|
}
|
|
2178
2903
|
const recoveryHealthy = verified.probes.recovery_gateway?.ok !== false
|
|
2179
2904
|
&& verified.probes.recovery_external_http?.ok !== false;
|
|
2180
|
-
const primaryRuntimeHealthy =
|
|
2181
|
-
|
|
2905
|
+
const primaryRuntimeHealthy = localVerify.runtime.ok
|
|
2906
|
+
&& localVerify.runtime.running
|
|
2907
|
+
&& localVerify.runtime.ready
|
|
2908
|
+
&& !localVerify.runtime.stale
|
|
2909
|
+
&& localVerify.probes.active_gateway?.ok === true;
|
|
2910
|
+
const primaryConnectorConfigured = Boolean(config.primaryConnectorService);
|
|
2911
|
+
const primaryConnectorLocalFailed = verified.probes.primary_connector_local?.ok === false;
|
|
2912
|
+
const primaryConnectorCapacityFailed = connectorCapacityRecoveryRecommended(verified);
|
|
2913
|
+
const primaryPublicTransportManaged = Boolean(configuredPrimaryPublicTunnel(config));
|
|
2914
|
+
const primaryPublicTransportFailed = Boolean(
|
|
2915
|
+
config.publicMcpUrl
|
|
2916
|
+
&& primaryRuntimeHealthy
|
|
2917
|
+
&& !primaryConnectorCapacityFailed
|
|
2918
|
+
&& (
|
|
2919
|
+
verified.probes.external_mcp_http?.ok === false
|
|
2920
|
+
|| verified.probes.primary_connector_ready?.ok === false
|
|
2921
|
+
),
|
|
2922
|
+
);
|
|
2923
|
+
const primaryConnectorFailed = Boolean(
|
|
2924
|
+
primaryConnectorConfigured
|
|
2925
|
+
&& primaryRuntimeHealthy
|
|
2926
|
+
&& (
|
|
2927
|
+
primaryConnectorLocalFailed
|
|
2928
|
+
|| primaryConnectorCapacityFailed
|
|
2929
|
+
|| (primaryPublicTransportManaged && primaryPublicTransportFailed)
|
|
2930
|
+
),
|
|
2931
|
+
);
|
|
2932
|
+
if (primaryRuntimeHealthy && !primaryConnectorFailed && !primaryPublicTransportFailed && recoveryHealthy) {
|
|
2933
|
+
if (fullVerificationPerformed && verified.ok && !matchingKnownGood(config, verified.releases.active)) {
|
|
2934
|
+
try {
|
|
2935
|
+
const attestation = await withLock(config, { action: 'attest_known_good' }, async () => persistVerifiedKnownGood(config, verified));
|
|
2936
|
+
if (!attestation.acquired) {
|
|
2937
|
+
audit(config, 'known_good_attestation_deferred', { reason: recoveryBusyDetail(attestation.owner) });
|
|
2938
|
+
}
|
|
2939
|
+
} catch (error) {
|
|
2940
|
+
audit(config, 'known_good_attestation_deferred', {
|
|
2941
|
+
reason: error instanceof Error ? error.message : 'watchdog known-good attestation failed',
|
|
2942
|
+
});
|
|
2943
|
+
}
|
|
2944
|
+
}
|
|
2182
2945
|
const stable = recordWatchdogRuntimeHealthy(
|
|
2183
2946
|
scopeWatchdogStateToRuntimeRelease(scopedPrior, verified.releases.active),
|
|
2184
2947
|
now,
|
|
@@ -2193,12 +2956,38 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
2193
2956
|
publicTunnelFailures: 0,
|
|
2194
2957
|
publicTunnelFirstFailureAt: undefined,
|
|
2195
2958
|
publicTunnelRepairFailures: 0,
|
|
2959
|
+
primaryConnectorFailures: 0,
|
|
2960
|
+
primaryConnectorFirstFailureAt: undefined,
|
|
2961
|
+
primaryConnectorRestartAttempts: 0,
|
|
2962
|
+
primaryConnectorRestartFailures: 0,
|
|
2963
|
+
primaryConnectorRestartLastAttemptAt: undefined,
|
|
2196
2964
|
recoveryGatewayRestartUsed: false,
|
|
2197
2965
|
lastFullVerifyAt,
|
|
2198
2966
|
lastDecision: 'healthy',
|
|
2967
|
+
lastReason: 'primary runtime and standalone Recovery verification passed',
|
|
2199
2968
|
};
|
|
2200
2969
|
return { state, decision: { action: 'healthy', reason: 'primary runtime and standalone Recovery verification passed' }, verify: verified };
|
|
2201
2970
|
}
|
|
2971
|
+
if (primaryRuntimeHealthy && !primaryConnectorFailed && primaryPublicTransportFailed && recoveryHealthy) {
|
|
2972
|
+
const reason = primaryPublicTransportManaged
|
|
2973
|
+
? 'primary public transport is unavailable but no safely repairable primary Connector path was established'
|
|
2974
|
+
: 'primary public transport is unavailable while the local Connector is healthy, but no managed primary public tunnel is configured; external transport repair is required';
|
|
2975
|
+
const state: WatchdogState = {
|
|
2976
|
+
...recordWatchdogRuntimeHealthy(
|
|
2977
|
+
scopeWatchdogStateToRuntimeRelease(scopedPrior, verified.releases.active),
|
|
2978
|
+
now,
|
|
2979
|
+
watchdogRuntimeRestartBudgetStableMs(config),
|
|
2980
|
+
),
|
|
2981
|
+
failures: scopedPrior.failures + 1,
|
|
2982
|
+
firstFailureAt: scopedPrior.firstFailureAt ?? now,
|
|
2983
|
+
primaryConnectorFailures: 0,
|
|
2984
|
+
primaryConnectorFirstFailureAt: undefined,
|
|
2985
|
+
lastFullVerifyAt,
|
|
2986
|
+
lastDecision: 'degraded',
|
|
2987
|
+
lastReason: reason,
|
|
2988
|
+
};
|
|
2989
|
+
return { state, decision: { action: 'degraded', reason }, verify: verified };
|
|
2990
|
+
}
|
|
2202
2991
|
const publicTunnelFailed = isExternalTunnelFailure(config, verified, localVerify);
|
|
2203
2992
|
const evidenceClasses = Object.entries(localVerify.probes)
|
|
2204
2993
|
.filter(([name, value]) => !name.startsWith('recovery_') && !value.ok)
|
|
@@ -2221,16 +3010,31 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
2221
3010
|
firstFailureAt: budgetPrior.firstFailureAt ?? now,
|
|
2222
3011
|
publicTunnelFailures: (budgetPrior.publicTunnelFailures ?? 0) + 1,
|
|
2223
3012
|
publicTunnelFirstFailureAt: budgetPrior.publicTunnelFirstFailureAt ?? now,
|
|
3013
|
+
primaryConnectorFailures: 0,
|
|
3014
|
+
primaryConnectorFirstFailureAt: undefined,
|
|
2224
3015
|
}
|
|
2225
|
-
:
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
2233
|
-
|
|
3016
|
+
: primaryConnectorFailed
|
|
3017
|
+
? {
|
|
3018
|
+
...budgetPrior,
|
|
3019
|
+
lastFullVerifyAt,
|
|
3020
|
+
failures: budgetPrior.failures + 1,
|
|
3021
|
+
firstFailureAt: budgetPrior.firstFailureAt ?? now,
|
|
3022
|
+
publicTunnelFailures: 0,
|
|
3023
|
+
publicTunnelFirstFailureAt: undefined,
|
|
3024
|
+
primaryConnectorFailures: (budgetPrior.primaryConnectorFailures ?? 0) + 1,
|
|
3025
|
+
primaryConnectorFirstFailureAt: budgetPrior.primaryConnectorFirstFailureAt ?? now,
|
|
3026
|
+
}
|
|
3027
|
+
: {
|
|
3028
|
+
...budgetPrior,
|
|
3029
|
+
lastFullVerifyAt,
|
|
3030
|
+
failures: budgetPrior.failures + 1,
|
|
3031
|
+
firstFailureAt: budgetPrior.firstFailureAt ?? now,
|
|
3032
|
+
publicTunnelFailures: 0,
|
|
3033
|
+
publicTunnelFirstFailureAt: undefined,
|
|
3034
|
+
primaryConnectorFailures: 0,
|
|
3035
|
+
primaryConnectorFirstFailureAt: undefined,
|
|
3036
|
+
};
|
|
3037
|
+
if (!primaryRuntimeHealthy) state.runtimeHealthySince = undefined;
|
|
2234
3038
|
const primaryConfig = configuredPrimaryRuntimeService(config);
|
|
2235
3039
|
const decision = decideWatchdog({
|
|
2236
3040
|
...state,
|
|
@@ -2238,7 +3042,7 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
2238
3042
|
evidenceClasses,
|
|
2239
3043
|
activeKnownGood,
|
|
2240
3044
|
previousKnownGood,
|
|
2241
|
-
primaryRuntimeFailed: !
|
|
3045
|
+
primaryRuntimeFailed: !primaryRuntimeHealthy,
|
|
2242
3046
|
runtimeMaximumRestartAttempts: primaryConfig.maximumRestartAttempts,
|
|
2243
3047
|
runtimeRestartCooldownMs: primaryConfig.restartCooldownMs,
|
|
2244
3048
|
runtimeMinimumFailures: primaryConfig.minimumFailures,
|
|
@@ -2246,12 +3050,23 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
2246
3050
|
runtimeRecoveryLastAttemptAt: state.runtimeRecoveryLastAttemptAt,
|
|
2247
3051
|
runtimeRecoveryCooldownMs: primaryConfig.recoveryCooldownMs,
|
|
2248
3052
|
recoveryGatewayFailed: verified.probes.recovery_gateway?.ok === false,
|
|
3053
|
+
primaryConnectorConfigured,
|
|
3054
|
+
primaryConnectorFailed,
|
|
3055
|
+
primaryConnectorFailures: state.primaryConnectorFailures,
|
|
3056
|
+
primaryConnectorFirstFailureAt: state.primaryConnectorFirstFailureAt,
|
|
3057
|
+
primaryConnectorRestartAttempts: state.primaryConnectorRestartAttempts,
|
|
3058
|
+
primaryConnectorMaximumRestartAttempts: config.primaryConnectorService?.maximumRestartAttempts,
|
|
3059
|
+
primaryConnectorRestartLastAttemptAt: state.primaryConnectorRestartLastAttemptAt,
|
|
3060
|
+
primaryConnectorRestartCooldownMs: config.primaryConnectorService?.restartCooldownMs,
|
|
3061
|
+
primaryConnectorMinimumFailures: config.primaryConnectorService?.minimumFailures,
|
|
3062
|
+
primaryConnectorMinimumFailureDurationMs: config.primaryConnectorService?.minimumFailureDurationMs,
|
|
2249
3063
|
publicTunnelConfigured: Boolean(configuredRecoveryTunnel(config)),
|
|
2250
3064
|
publicTunnelFailed,
|
|
2251
3065
|
publicTunnelMinimumFailures: configuredRecoveryTunnel(config)?.minimumFailures,
|
|
2252
3066
|
publicTunnelMinimumFailureDurationMs: configuredRecoveryTunnel(config)?.minimumFailureDurationMs,
|
|
2253
3067
|
});
|
|
2254
3068
|
state.lastDecision = decision.action;
|
|
3069
|
+
state.lastReason = decision.reason;
|
|
2255
3070
|
if (decision.action === 'repair_public_tunnel') {
|
|
2256
3071
|
const publicTunnelRepair = await repairPublicTunnel(config);
|
|
2257
3072
|
const nextState = publicTunnelRepair.ok
|
|
@@ -2263,6 +3078,30 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
2263
3078
|
const recoveryGatewayRestart = await restartRecoveryGateway(config);
|
|
2264
3079
|
return { state: { ...state, recoveryGatewayRestartUsed: true }, decision, verify: verified, recoveryGatewayRestart };
|
|
2265
3080
|
}
|
|
3081
|
+
if (decision.action === 'restart_primary_connector') {
|
|
3082
|
+
const primaryConnectorRestart = await restartPrimaryConnector(config, {
|
|
3083
|
+
requestId: `watchdog:restart_primary_connector:${now}`,
|
|
3084
|
+
});
|
|
3085
|
+
const attempts = (state.primaryConnectorRestartAttempts ?? 0) + (primaryConnectorRestart.attempted ? 1 : 0);
|
|
3086
|
+
const nextState: WatchdogState = primaryConnectorRestart.ok
|
|
3087
|
+
? {
|
|
3088
|
+
...state,
|
|
3089
|
+
failures: 0,
|
|
3090
|
+
firstFailureAt: undefined,
|
|
3091
|
+
primaryConnectorFailures: 0,
|
|
3092
|
+
primaryConnectorFirstFailureAt: undefined,
|
|
3093
|
+
primaryConnectorRestartAttempts: 0,
|
|
3094
|
+
primaryConnectorRestartFailures: 0,
|
|
3095
|
+
primaryConnectorRestartLastAttemptAt: now,
|
|
3096
|
+
}
|
|
3097
|
+
: {
|
|
3098
|
+
...state,
|
|
3099
|
+
primaryConnectorRestartAttempts: attempts,
|
|
3100
|
+
primaryConnectorRestartFailures: (state.primaryConnectorRestartFailures ?? 0) + (primaryConnectorRestart.attempted ? 1 : 0),
|
|
3101
|
+
primaryConnectorRestartLastAttemptAt: now,
|
|
3102
|
+
};
|
|
3103
|
+
return { state: nextState, decision, verify: verified, primaryConnectorRestart };
|
|
3104
|
+
}
|
|
2266
3105
|
if (decision.action === 'restart_primary_runtime') {
|
|
2267
3106
|
const primaryRuntimeRestart = await restartPrimaryRuntime(config);
|
|
2268
3107
|
const attempts = (state.runtimeRestartAttempts ?? 0) + (primaryRuntimeRestart.attempted ? 1 : 0);
|