@moretea-labs/forge 1.7.2 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +58 -62
- package/CHANGELOG.md +26 -0
- package/CLAUDE.md +42 -62
- package/README.md +2 -2
- package/SKILL.md +0 -1
- package/THIRD_PARTY_NOTICES.md +5 -2
- package/adapters/browser/index.ts +2 -0
- package/adapters/chatgpt/browser-delivery-host.ts +71 -0
- package/adapters/chatgpt/browser-delivery-runtime.ts +783 -0
- package/adapters/chatgpt/controller-binding-store.ts +55 -0
- package/adapters/chatgpt/controller-host.ts +119 -0
- package/adapters/chatgpt/controller-round-host.ts +78 -0
- package/adapters/chatgpt/controller-round-settlement-store.ts +45 -0
- package/adapters/chatgpt/provider-delivery.ts +113 -0
- package/adapters/chatgpt/work-conversation-binding-store.ts +192 -0
- package/adapters/chatgpt/wsl-bridge-delivery-host.ts +63 -0
- package/adapters/computer/desktop-operator-contract.ts +7 -0
- package/adapters/computer/desktop-operator-discovery.ts +201 -0
- package/adapters/computer/desktop-operator-negotiation.ts +174 -0
- package/adapters/computer/desktop-operator-provider.ts +312 -0
- package/adapters/computer/index.ts +1 -0
- package/adapters/computer/interaction-target-authority.ts +944 -0
- package/adapters/controller-process/binding-store.ts +34 -0
- package/adapters/controller-process/controller-host.ts +26 -0
- package/adapters/mcp/access-mode.ts +132 -0
- package/adapters/mcp/audit.ts +32 -0
- package/adapters/mcp/auth.ts +407 -0
- package/adapters/mcp/controller-round-compatibility.ts +184 -0
- package/adapters/mcp/frozen-client-semantic-compatibility.ts +292 -0
- package/adapters/mcp/instructions.ts +36 -0
- package/adapters/mcp/legacy-context.ts +18 -0
- package/adapters/mcp/multi-repository.ts +437 -0
- package/adapters/mcp/oauth.ts +391 -0
- package/adapters/mcp/paths.ts +140 -0
- package/adapters/mcp/policy.ts +232 -0
- package/adapters/mcp/proxy-env.ts +62 -0
- package/adapters/mcp/readiness-projection.ts +24 -0
- package/adapters/mcp/redaction.ts +21 -0
- package/adapters/mcp/repo.ts +24 -0
- package/adapters/mcp/runtime-gateway/artifact-adapter.ts +52 -0
- package/adapters/mcp/runtime-gateway/context-adapter.ts +829 -0
- package/adapters/mcp/runtime-gateway/controller-authority-adapter.ts +191 -0
- package/adapters/mcp/runtime-gateway/engineering-preconditions.ts +147 -0
- package/adapters/mcp/runtime-gateway/execution-control-adapter.ts +28 -0
- package/adapters/mcp/runtime-gateway/execution-tools.ts +207 -0
- package/adapters/mcp/runtime-gateway/filesystem-adapter.ts +38 -0
- package/adapters/mcp/runtime-gateway/gateway-execution-adapter.ts +411 -0
- package/adapters/mcp/runtime-gateway/legacy-ios-tool-adapter.ts +105 -0
- package/adapters/mcp/runtime-gateway/maintenance-adapter.ts +44 -0
- package/adapters/mcp/runtime-gateway/model-adapter.ts +63 -0
- package/adapters/mcp/runtime-gateway/persisted-check-process.ts +3 -0
- package/adapters/mcp/runtime-gateway/plugin-adapter.ts +308 -0
- package/adapters/mcp/runtime-gateway/process-tools.ts +289 -0
- package/adapters/mcp/runtime-gateway/protected-computer-adapter.ts +76 -0
- package/adapters/mcp/runtime-gateway/recovery-adapter.ts +268 -0
- package/adapters/mcp/runtime-gateway/recovery-client-adapter.ts +70 -0
- package/adapters/mcp/runtime-gateway/recovery-tool-contract.ts +99 -0
- package/adapters/mcp/runtime-gateway/repository-compat-adapter.ts +110 -0
- package/adapters/mcp/runtime-gateway/result-adapter.ts +108 -0
- package/adapters/mcp/runtime-gateway/router.ts +728 -0
- package/adapters/mcp/runtime-gateway/runtime-observation-adapter.ts +1251 -0
- package/adapters/mcp/runtime-gateway/runtime-tool-definitions.ts +634 -0
- package/adapters/mcp/runtime-gateway/runtime-tool-shared.ts +27 -0
- package/adapters/mcp/runtime-gateway/runtime-tools.ts +68 -0
- package/adapters/mcp/runtime-gateway/scheduler-adapter.ts +186 -0
- package/adapters/mcp/runtime-gateway/shared-adapter.ts +25 -0
- package/adapters/mcp/runtime-gateway/status-inbox-adapter.ts +870 -0
- package/adapters/mcp/runtime-gateway/work-adapter.ts +3186 -0
- package/adapters/mcp/runtime-gateway/work-compat-adapter.ts +502 -0
- package/adapters/mcp/runtime-gateway/work-validation-reconciler.ts +1 -0
- package/adapters/mcp/server.ts +1048 -0
- package/adapters/mcp/setup.ts +791 -0
- package/adapters/mcp/tool-capability-cache.ts +87 -0
- package/adapters/mcp/tool-mapping/access-tools.ts +387 -0
- package/adapters/mcp/tool-mapping/legacy-tool-service.ts +5265 -0
- package/adapters/mcp/tool-mapping/post-finalize-work-attribution.ts +150 -0
- package/adapters/mcp/tool-mapping/repository-tools.ts +1691 -0
- package/adapters/mcp/tool-mapping/tools.ts +9 -0
- package/adapters/mcp/tool-mapping/umbrella-tools.ts +117 -0
- package/adapters/mcp/toolset-names.ts +74 -0
- package/adapters/mcp/toolset-selection.ts +41 -0
- package/adapters/mcp/toolset.ts +336 -0
- package/adapters/mcp/transports/http.ts +1456 -0
- package/adapters/mcp/transports/session-registry.ts +513 -0
- package/adapters/mcp/transports/stdio.ts +8 -0
- package/adapters/mcp/tunnels/openai-secure-tunnel.ts +139 -0
- package/adapters/mcp/types.ts +41 -0
- package/assets/AGENTS.md +2 -16
- package/assets/CLAUDE.md +2 -16
- package/assets/hooks/AGENTS.md +3 -14
- package/assets/hooks/CLAUDE.md +3 -14
- package/assets/hooks/hook-input.sh +9 -2
- package/assets/hooks/lib/workflow-state.sh +28 -10
- package/assets/hooks/prompt-guard.sh +63 -37
- package/assets/hooks/security-sentinel.sh +2 -1
- package/assets/hooks/session-start-context.sh +7 -5
- package/assets/hooks/stop-orchestrator.sh +13 -3
- package/assets/initializer-question-pack.v4.json +11 -7
- package/assets/partials/05-workflow.partial.md +18 -73
- package/assets/partials/07-footer.partial.md +1 -1
- package/assets/partials-agents/02-operating-mode.partial.md +1 -2
- package/assets/partials-agents/04-task-protocol.partial.md +14 -59
- package/assets/partials-agents/06-quality-safety.partial.md +11 -16
- package/assets/partials-agents/08-deep-docs.partial.md +1 -2
- package/assets/plan-map.json +11 -11
- package/assets/plugin-registry.v1.json +9 -3
- package/assets/plugin-trust-policy.v1.json +6 -0
- package/assets/recovery/ForgeRecovery.ps1 +100 -0
- package/assets/recovery/forge-wsl-rescue +271 -0
- package/assets/reference-configs/development-protocol.md +11 -15
- package/assets/reference-configs/document-generation.md +1 -1
- package/assets/reference-configs/global-working-rules.md +9 -1
- package/assets/reference-configs/harness-overview.md +22 -112
- package/assets/reference-configs/hook-operations.md +1 -1
- package/assets/reference-configs/sprint-contracts.md +2 -2
- package/assets/skill-version.json +2 -2
- package/assets/templates/helpers/architecture-event.ts +3 -34
- package/assets/templates/helpers/architecture-queue.sh +61 -10
- package/assets/templates/helpers/archive-workflow.sh +90 -180
- package/assets/templates/helpers/capability-config.ts +8 -11
- package/assets/templates/helpers/capability-resolver.ts +37 -19
- package/assets/templates/helpers/capture-plan.sh +2 -2
- package/assets/templates/helpers/check-architecture-sync.sh +1 -9
- package/assets/templates/helpers/check-task-sync.sh +82 -11
- package/assets/templates/helpers/check-task-workflow.sh +1 -3
- package/assets/templates/helpers/context-contract-sync.sh +1 -12
- package/assets/templates/helpers/contract-worktree.sh +26 -24
- package/assets/templates/helpers/ensure-task-workflow.sh +6 -8
- package/assets/templates/helpers/inspect-project-state.ts +1 -1
- package/assets/templates/helpers/migrate-project-template.sh +1 -1
- package/assets/templates/helpers/migrate-workflow-docs.ts +53 -239
- package/assets/templates/helpers/new-plan.sh +2 -2
- package/assets/templates/helpers/plan-to-todo.sh +3 -31
- package/assets/templates/helpers/sprint-backlog.sh +1 -1
- package/assets/templates/helpers/switch-plan.sh +1 -1
- package/assets/templates/helpers/workflow-contract.ts +1 -0
- package/assets/workflow-contract.v1.json +63 -171
- package/assets/workflows/xiaohongshu/xiaohongshu.collect-metrics.v1.draft.json +118 -0
- package/assets/workflows/xiaohongshu/xiaohongshu.publish-image-note.v1.draft.json +316 -0
- package/assets/workflows/xiaohongshu/xiaohongshu.publish-long-text.v1.draft.json +324 -0
- package/docs/architecture/CURRENT.md +148 -13
- package/docs/architecture/EVOLUTION.md +13 -0
- package/docs/architecture/decisions/20260801-controller-home-sqlite-state.md +2 -1
- package/docs/architecture/decisions/20260831-independent-windows-wsl-host-rescue.md +59 -0
- package/docs/architecture/decisions/20260904-ios-device-migration-baseline.md +120 -0
- package/docs/architecture/decisions/20260904-repository-runtime-data-boundary.md +42 -0
- package/docs/architecture/decisions/20260907-assistant-learning-loop.md +35 -0
- package/docs/architecture/decisions/20260907-controller-round-transition-policy.md +296 -0
- package/docs/architecture/decisions/20260907-maintenance-diagnosis-bounded-read.md +39 -0
- package/docs/architecture/domains/verification.md +7 -0
- package/docs/architecture/domains/workflow-engine.md +1 -1
- package/docs/architecture/index.md +3 -3
- package/docs/architecture/modules/controller-runtime/transport-lifecycle.md +19 -8
- package/docs/architecture/requests/2026-09-04-computer-plugin-workflow-taxonomy.md +124 -0
- package/docs/forge-chatgpt-mcp-setup.md +5 -3
- package/docs/forge-plugin-management.md +20 -4
- package/docs/forge-runtime-self-healing-loop.md +2 -0
- package/docs/operations/benchmark-cross-version-freeze.md +38 -0
- package/docs/operations/controller-browser-plugin.md +3 -1
- package/docs/operations/homebrew.md +2 -2
- package/docs/operations/platform-support.md +16 -8
- package/docs/operations/platform-support.zh-CN.md +2 -2
- package/docs/operations/standalone-disaster-recovery.md +26 -1
- package/docs/operations/uu-remote-rescue.md +41 -0
- package/docs/operations/windows-wsl-host-rescue.md +133 -0
- package/docs/public-usage-guide.md +1 -1
- package/docs/public-usage-guide.zh-CN.md +1 -1
- package/docs/tutorials/01-install-and-start.md +1 -1
- package/docs/tutorials/01-install-and-start.zh-CN.md +1 -1
- package/docs/tutorials/02-connect-chatgpt.md +4 -4
- package/docs/tutorials/02-connect-chatgpt.zh-CN.md +4 -4
- package/docs/wiki/Implementation.md +1 -1
- package/package.json +23 -14
- package/packages/kernel/controller/api/index.ts +2 -0
- package/packages/kernel/controller/application/controller-round-service.ts +29 -0
- package/packages/kernel/controller/application/controller-service.ts +30 -0
- package/packages/kernel/controller/domain/controller-round-transition-policy.ts +308 -0
- package/packages/kernel/controller/domain/controller-round.ts +107 -0
- package/packages/kernel/controller/domain/execution-quality.ts +133 -0
- package/packages/kernel/controller/domain/types.ts +56 -0
- package/packages/kernel/controller/index.ts +1 -0
- package/packages/kernel/controller/infrastructure/controller-binding-store.ts +84 -0
- package/packages/kernel/controller/infrastructure/controller-round-store.ts +1384 -0
- package/packages/kernel/controller/infrastructure/controller-session-store.ts +578 -0
- package/packages/kernel/controller/ports/controller-host.ts +16 -0
- package/packages/kernel/goal/api/index.ts +1 -0
- package/packages/kernel/goal/domain/types.ts +40 -0
- package/packages/kernel/goal/index.ts +1 -0
- package/packages/kernel/identity/api/index.ts +5 -0
- package/packages/kernel/identity/application/identity-service.ts +67 -0
- package/packages/kernel/identity/domain/scope.ts +63 -0
- package/packages/kernel/identity/domain/semantic-sync.ts +81 -0
- package/packages/kernel/identity/domain/types.ts +44 -0
- package/packages/kernel/identity/domain/workspace.ts +56 -0
- package/packages/kernel/identity/index.ts +1 -0
- package/packages/kernel/identity/infrastructure/identity-store.ts +75 -0
- package/packages/kernel/memory/api/index.ts +5 -0
- package/packages/kernel/memory/application/experience-service.ts +118 -0
- package/packages/kernel/memory/application/outcome-service.ts +77 -0
- package/packages/kernel/memory/domain/experience.ts +103 -0
- package/packages/kernel/memory/domain/operational-prior.ts +92 -0
- package/packages/kernel/memory/index.ts +1 -0
- package/packages/kernel/progression/api/index.ts +2 -0
- package/packages/kernel/progression/application/projector.ts +211 -0
- package/packages/kernel/progression/domain/types.ts +116 -0
- package/packages/kernel/progression/index.ts +1 -0
- package/packages/kernel/scheduler/api/index.ts +6 -0
- package/packages/kernel/scheduler/application/continuation-service.ts +212 -0
- package/packages/kernel/scheduler/application/eligibility.ts +252 -0
- package/packages/kernel/scheduler/application/schedule-service.ts +24 -0
- package/packages/kernel/scheduler/application/settlement.ts +197 -0
- package/packages/kernel/scheduler/domain/continuation.ts +38 -0
- package/packages/kernel/scheduler/domain/schedule.ts +149 -0
- package/packages/kernel/scheduler/index.ts +1 -0
- package/packages/kernel/scheduler/infrastructure/continuation-dispatch-store.ts +38 -0
- package/packages/kernel/scheduler/infrastructure/schedule-store.ts +785 -0
- package/packages/kernel/work/api/index.ts +15 -0
- package/packages/kernel/work/application/work-service.ts +148 -0
- package/packages/kernel/work/domain/admission-policy.ts +34 -0
- package/packages/kernel/work/domain/check-receipt.ts +43 -0
- package/packages/kernel/work/domain/context-closure-receipt.ts +32 -0
- package/packages/kernel/work/domain/engineering-contracts.ts +157 -0
- package/packages/kernel/work/domain/engineering-design.ts +281 -0
- package/packages/kernel/work/domain/engineering-profile.ts +242 -0
- package/packages/kernel/work/domain/execution-concurrency.ts +254 -0
- package/packages/kernel/work/domain/execution-snapshot.ts +33 -0
- package/packages/kernel/work/domain/implementation-review.ts +510 -0
- package/packages/kernel/work/domain/project-engineering-contract.ts +202 -0
- package/packages/kernel/work/domain/repository-completion-receipt.ts +73 -0
- package/packages/kernel/work/domain/state-machine.ts +259 -0
- package/packages/kernel/work/domain/types.ts +528 -0
- package/packages/kernel/work/index.ts +1 -0
- package/packages/kernel/work/infrastructure/work-contract-store.ts +1876 -0
- package/packages/kernel/work/ports/work-contract-store.ts +18 -0
- package/packages/plugin-runtime/browser/index.ts +3 -0
- package/packages/plugin-runtime/browser/provider-registry.ts +110 -0
- package/packages/plugin-runtime/browser/runtime-contract.ts +150 -0
- package/packages/plugin-runtime/browser/session-authority.ts +61 -0
- package/packages/plugin-runtime/computer/index.ts +6 -0
- package/packages/plugin-runtime/computer/provider-error.ts +26 -0
- package/packages/plugin-runtime/computer/provider-registration.ts +52 -0
- package/packages/plugin-runtime/computer/provider-registry.ts +63 -0
- package/packages/plugin-runtime/computer/provider.ts +19 -0
- package/packages/plugin-runtime/computer/target-authority.ts +190 -0
- package/packages/plugin-runtime/computer/target-persistence.ts +54 -0
- package/packages/plugin-runtime/external/index.ts +1 -0
- package/packages/plugin-runtime/external/unix-jsonl-transport.ts +457 -0
- package/packages/plugin-runtime/index.ts +3 -0
- package/packages/protocols/browser/index.ts +1 -0
- package/packages/protocols/browser/session.ts +98 -0
- package/packages/protocols/computer/contract.ts +199 -0
- package/packages/protocols/computer/index.ts +1 -0
- package/packages/protocols/failure/contract.ts +158 -0
- package/packages/protocols/failure/index.ts +1 -0
- package/packages/protocols/handoff/index.ts +1 -0
- package/packages/protocols/handoff/status.ts +22 -0
- package/packages/protocols/mcp/execution-context.ts +11 -0
- package/packages/protocols/mcp/index.ts +2 -0
- package/packages/protocols/mcp/tool-contract.ts +42 -0
- package/packages/workflow-runtime/api/index.ts +2 -0
- package/packages/workflow-runtime/application/interpreter.ts +342 -0
- package/packages/workflow-runtime/domain/workflow-asset.ts +223 -0
- package/packages/workflow-runtime/index.ts +1 -0
- package/scripts/assemble-template.ts +1 -1
- package/scripts/benchmark-computer-hotpath.ts +255 -0
- package/scripts/benchmark-controller-round2.ts +54 -2
- package/scripts/benchmark-ios-agent-device.ts +3 -6
- package/scripts/benchmark-mcp-transport.ts +3 -4
- package/scripts/benchmark-operational-memory-activation.ts +235 -0
- package/scripts/benchmark-operational-shadow.ts +28 -0
- package/scripts/bootstrap-runtime-maintenance-recovery.sh +4 -1
- package/scripts/canary-chatgpt-provider-delivery.ts +97 -0
- package/scripts/canary-controller-semantic-wait.ts +195 -0
- package/scripts/canary-global-hook.sh +2 -2
- package/scripts/check-mcp-compatibility.ts +173 -7
- package/scripts/check-platform-support.mjs +28 -2
- package/scripts/check-public-docs.sh +1 -1
- package/scripts/check-release-readiness.sh +1 -1
- package/scripts/check-repository-hygiene.mjs +92 -0
- package/scripts/check-runtime-architecture.mjs +1206 -95
- package/scripts/check-source-duplication.mjs +9 -1
- package/scripts/check-tarball-install-smoke.sh +98 -15
- package/scripts/controller-ngrok-rotation.sh +5 -4
- package/scripts/create-project-dirs.sh +10 -74
- package/scripts/forge-local-recovery-helper.mjs +204 -0
- package/scripts/forge-uu-remote-rescue-helper.d.mts +31 -0
- package/scripts/forge-uu-remote-rescue-helper.mjs +409 -0
- package/scripts/forge-windows-host-recovery-helper.mjs +136 -0
- package/scripts/hook-shim.sh +2 -2
- package/scripts/init-project.sh +12 -70
- package/scripts/install-desktop-operator-registration.ts +4 -2
- package/scripts/install-independent-windows-wsl-recovery.ts +125 -0
- package/scripts/install-standalone-recovery.ts +122 -10
- package/scripts/install-uu-remote-rescue-registration.ts +74 -0
- package/scripts/install-windows-host-recovery-registration.ts +22 -0
- package/scripts/lib/controller-home-env.sh +6 -3
- package/scripts/lib/project-init-lib.sh +39 -582
- package/scripts/migrate-chatgpt-automation-schedules.ts +2 -2
- package/scripts/migrate-windows-wsl-controller-home.ts +306 -0
- package/scripts/open-source-audit-allowlist.txt +12 -1
- package/scripts/portability-bindings.mjs +70 -0
- package/scripts/run-bounded-command.mjs +55 -0
- package/scripts/run-bun-test-file.ts +22 -6
- package/scripts/run-governed-gate.ts +4 -1
- package/scripts/smoke-mcp-http-runtime.ts +1 -1
- package/scripts/smoke-runtime-recovery.ts +34 -1
- package/scripts/smoke-schedule-engine.ts +1 -1
- package/scripts/stage-runtime-release.ts +6 -1
- package/scripts/test-governance.ts +51 -2
- package/src/cli/agent-jobs/job-manager.ts +10 -1
- package/src/cli/chatgpt-browser/bind-server.ts +2 -1
- package/src/cli/chatgpt-browser/binding.ts +20 -10
- package/src/cli/chatgpt-browser/bridge-extension.ts +75 -0
- package/src/cli/chatgpt-browser/bridge-provider.ts +376 -11
- package/src/cli/chatgpt-browser/engine.ts +10 -11
- package/src/cli/chatgpt-browser/native-provider.ts +24 -83
- package/src/cli/chatgpt-browser/oracle-provider.ts +5 -3
- package/src/cli/chatgpt-browser/types.ts +1 -0
- package/src/cli/commands/bootstrap-control.ts +334 -0
- package/src/cli/commands/brain-assistant.ts +38 -0
- package/src/cli/commands/brain.ts +193 -40
- package/src/cli/commands/chatgpt.ts +127 -0
- package/src/cli/commands/computer.ts +437 -0
- package/src/cli/commands/doctor.ts +18 -1
- package/src/cli/commands/init-hook.ts +71 -61
- package/src/cli/commands/init.ts +3 -1
- package/src/cli/commands/mcp.ts +16 -10
- package/src/cli/commands/plugin.ts +38 -12
- package/src/cli/commands/recovery.ts +493 -80
- package/src/cli/commands/runtime.ts +9 -4
- package/src/cli/commands/setup-profile.ts +89 -39
- package/src/cli/commands/status.ts +2 -1
- package/src/cli/controller/check-runner.ts +487 -98
- package/src/cli/controller/context/types.ts +7 -1
- package/src/cli/controller/context-pack.ts +23 -3
- package/src/cli/controller/issue-store.ts +1 -1
- package/src/cli/controller/systemd-user.ts +110 -0
- package/src/cli/controller/types.ts +10 -70
- package/src/cli/controller/worklog.ts +67 -13
- package/src/cli/editing/edit-operation-contract.ts +92 -0
- package/src/cli/editing/edit-session.ts +289 -13
- package/src/cli/github/plugin.ts +17 -12
- package/src/cli/hook/route-registry.ts +3 -5
- package/src/cli/hook/runtime.ts +49 -3
- package/src/cli/index.ts +3 -0
- package/src/cli/local-bridge/connector-freshness.ts +2 -2
- package/src/cli/local-bridge/console-automations.ts +2 -2
- package/src/cli/local-bridge/facade-api.ts +41 -14
- package/src/cli/local-bridge/job-store.ts +1 -1
- package/src/cli/local-bridge/server.ts +24 -27
- package/src/cli/mcp/access-mode.ts +2 -132
- package/src/cli/mcp/access-tools.ts +2 -387
- package/src/cli/mcp/audit.ts +2 -32
- package/src/cli/mcp/auth.ts +2 -404
- package/src/cli/mcp/instructions.ts +2 -34
- package/src/cli/mcp/legacy-context.ts +2 -16
- package/src/cli/mcp/legacy-tool-service.ts +2 -5308
- package/src/cli/mcp/multi-repository.ts +2 -436
- package/src/cli/mcp/oauth.ts +2 -375
- package/src/cli/mcp/openai-secure-tunnel.ts +2 -0
- package/src/cli/mcp/paths.ts +2 -140
- package/src/cli/mcp/policy.ts +2 -232
- package/src/cli/mcp/post-finalize-work-attribution.ts +2 -150
- package/src/cli/mcp/proxy-env.ts +2 -62
- package/src/cli/mcp/readiness-projection.ts +2 -24
- package/src/cli/mcp/redaction.ts +2 -21
- package/src/cli/mcp/repo.ts +2 -24
- package/src/cli/mcp/repository-tools.ts +2 -1613
- package/src/cli/mcp/server.ts +2 -951
- package/src/cli/mcp/setup.ts +2 -737
- package/src/cli/mcp/tool-capability-cache.ts +2 -87
- package/src/cli/mcp/tool-contract.ts +2 -14
- package/src/cli/mcp/tools.ts +2 -9
- package/src/cli/mcp/toolset-names.ts +2 -70
- package/src/cli/mcp/toolset-selection.ts +2 -41
- package/src/cli/mcp/toolset.ts +2 -336
- package/src/cli/mcp/transports/AGENTS.md +1 -10
- package/src/cli/mcp/transports/CLAUDE.md +1 -10
- package/src/cli/mcp/transports/http.ts +2 -1414
- package/src/cli/mcp/transports/session-registry.ts +2 -490
- package/src/cli/mcp/transports/stdio.ts +2 -8
- package/src/cli/mcp/types.ts +2 -41
- package/src/cli/mcp/umbrella-tools.ts +2 -117
- package/src/cli/repositories/command-executor.ts +64 -18
- package/src/cli/repositories/command-scope.ts +117 -1
- package/src/cli/repositories/controller-home.ts +119 -1
- package/src/cli/repositories/identity.ts +13 -0
- package/src/cli/repositories/registry.ts +68 -49
- package/src/cli/repositories/repository-snapshot.ts +53 -7
- package/src/cli/repositories/runtime-storage.ts +27 -2
- package/src/cli/repositories/safe-patch.ts +34 -38
- package/src/cli/repositories/selected-path-actions.ts +22 -2
- package/src/cli/repositories/structured-git.ts +105 -25
- package/src/cli/runtime/helper-runner.ts +7 -0
- package/src/cli/runtime-invocation.ts +8 -1
- package/src/cli/tools/codegraph.ts +44 -25
- package/src/core/adoption/manifest-templates.ts +1 -1
- package/src/core/adoption/plan.ts +3 -26
- package/src/core/adoption/workflow-contract-plan.ts +17 -9
- package/src/effects/process-runner.ts +8 -14
- package/src/runtime/context/assistant-context.ts +154 -0
- package/src/runtime/context/assistant-work-context.ts +147 -0
- package/src/runtime/context/codegraph-cache-boundary.ts +255 -0
- package/src/runtime/context/codegraph-read-provider.ts +55 -22
- package/src/runtime/context/codegraph-sidecar.cjs +8 -1
- package/src/runtime/context/context-closure.ts +268 -0
- package/src/runtime/context/context-plane.ts +211 -0
- package/src/runtime/context/project-engineering-contract.ts +60 -0
- package/src/runtime/control-plane/bootstrap/capability-provider.ts +74 -0
- package/src/runtime/control-plane/bootstrap/control-api.ts +57 -0
- package/src/runtime/control-plane/bootstrap/index.ts +5 -0
- package/src/runtime/control-plane/bootstrap/state-machine.ts +141 -0
- package/src/runtime/control-plane/bootstrap/store.ts +21 -0
- package/src/runtime/control-plane/bootstrap/types.ts +87 -0
- package/src/runtime/control-plane/cleanup-artifact-retention.ts +328 -0
- package/src/runtime/control-plane/codegraph-cache-retention.ts +164 -0
- package/src/runtime/control-plane/concurrency/work-execution-concurrency.ts +320 -0
- package/src/runtime/control-plane/execution/content-equivalent-commit-authority.ts +169 -0
- package/src/runtime/control-plane/execution/controller-authority-recovery.ts +350 -0
- package/src/runtime/control-plane/execution/direct-canonical-work-reconciliation.ts +85 -0
- package/src/runtime/control-plane/execution/direct-edit-work-completion.ts +369 -99
- package/src/runtime/control-plane/execution/edit-validation-coordinator.ts +103 -15
- package/src/runtime/control-plane/execution/implementation-review-content.ts +95 -0
- package/src/runtime/control-plane/execution/repository-work-attribution.ts +2 -2
- package/src/runtime/control-plane/execution/retained-work-resume.ts +336 -6
- package/src/runtime/control-plane/execution/validation.ts +28 -3
- package/src/runtime/control-plane/execution/work-authority-reconciler.ts +116 -0
- package/src/runtime/control-plane/execution/work-completion-authority.ts +44 -2
- package/src/runtime/control-plane/execution/work-evidence-policy.ts +34 -3
- package/src/runtime/control-plane/execution/work-execution-support.ts +42 -22
- package/src/runtime/control-plane/execution/work-finalization-service.ts +1103 -166
- package/src/runtime/control-plane/execution/work-handle-authority.ts +219 -22
- package/src/runtime/control-plane/execution/work-handle-store.ts +77 -0
- package/src/runtime/control-plane/execution/work-head-settlement.ts +75 -0
- package/src/runtime/control-plane/execution/work-lifecycle-audit.ts +145 -23
- package/src/runtime/control-plane/execution/work-operation-service.ts +49 -13
- package/src/runtime/control-plane/execution/work-preparation-service.ts +31 -15
- package/src/runtime/control-plane/execution/work-remote-delivery.ts +1 -1
- package/src/runtime/control-plane/execution/work-task-receipt.ts +20 -1
- package/src/runtime/control-plane/execution/work-terminal-cleanup.ts +407 -76
- package/src/runtime/control-plane/execution/work-validation-reconciler.ts +88 -18
- package/src/runtime/control-plane/execution/work-verification-context.ts +1 -1
- package/src/runtime/control-plane/execution/work-verification-service.ts +258 -4
- package/src/runtime/control-plane/execution/work-verification-snapshot.ts +200 -15
- package/src/runtime/control-plane/facade/capability-registry.ts +21 -10
- package/src/runtime/control-plane/facade/controller-round-relay.ts +3 -1118
- package/src/runtime/control-plane/facade/controller-session-store.ts +2 -488
- package/src/runtime/control-plane/facade/goal-workloop-access.ts +6 -3
- package/src/runtime/control-plane/facade/goal-workloop.ts +972 -137
- package/src/runtime/control-plane/facade/handoff-inbox-application.ts +158 -0
- package/src/runtime/control-plane/facade/handoff-inbox-store.ts +10 -0
- package/src/runtime/control-plane/facade/index.ts +4 -2
- package/src/runtime/control-plane/facade/operation-digest.ts +22 -58
- package/src/runtime/control-plane/facade/plan-contract-store.ts +1004 -38
- package/src/runtime/control-plane/facade/repository-work-admission.ts +19 -1
- package/src/runtime/control-plane/facade/requirement-authority.ts +218 -0
- package/src/runtime/control-plane/facade/requirement-board.ts +6 -1
- package/src/runtime/control-plane/facade/rh-work-operation-contract.ts +48 -0
- package/src/runtime/control-plane/facade/semantic-admission.ts +8 -16
- package/src/runtime/control-plane/facade/suggested-actions.ts +4 -1
- package/src/runtime/control-plane/facade/types.ts +189 -532
- package/src/runtime/control-plane/facade/work-admission-policy.ts +84 -35
- package/src/runtime/control-plane/facade/work-contract-store.ts +2 -1043
- package/src/runtime/control-plane/facade/work-implementation-review.ts +2 -0
- package/src/runtime/control-plane/facade/work-state-machine.ts +2 -250
- package/src/runtime/control-plane/generated-cache-retention.ts +239 -18
- package/src/runtime/control-plane/global-scheduler/durable-admission.ts +12 -1
- package/src/runtime/control-plane/global-scheduler/maintenance.ts +76 -13
- package/src/runtime/control-plane/global-scheduler/periodic-cleanup-entry.ts +55 -0
- package/src/runtime/control-plane/global-scheduler/periodic-cleanup-process.ts +98 -0
- package/src/runtime/control-plane/global-scheduler/reconciliation.ts +1 -1
- package/src/runtime/control-plane/global-scheduler/scheduler-worker-entry.ts +10 -0
- package/src/runtime/control-plane/global-scheduler/scheduler.ts +173 -81
- package/src/runtime/control-plane/global-scheduler/worker-launch.ts +18 -6
- package/src/runtime/control-plane/launcher/chatgpt-round-continuation.ts +397 -0
- package/src/runtime/control-plane/launcher/chatgpt-work-binding-store.ts +2 -177
- package/src/runtime/control-plane/launcher/chatgpt-work-continuation.ts +301 -560
- package/src/runtime/control-plane/launcher/provider-mcp-bootstrap.ts +20 -4
- package/src/runtime/control-plane/launcher/thin-launcher.ts +50 -4
- package/src/runtime/control-plane/lifecycle-retention-metrics.ts +60 -0
- package/src/runtime/control-plane/persistence/experience-store.ts +169 -0
- package/src/runtime/control-plane/persistence/operational-prior-store.ts +348 -0
- package/src/runtime/control-plane/persistence/requirement-store.ts +30 -22
- package/src/runtime/control-plane/persistence/sqlite-store.ts +545 -76
- package/src/runtime/control-plane/persistence/workflow-content-store.ts +98 -0
- package/src/runtime/control-plane/persistence/workflow-registry-store.ts +186 -0
- package/src/runtime/control-plane/persistence/workflow-run-store.ts +123 -0
- package/src/runtime/control-plane/release-retention.ts +56 -16
- package/src/runtime/control-plane/repository-namespace-retention.ts +208 -0
- package/src/runtime/control-plane/routing/route-policy.ts +19 -0
- package/src/runtime/control-plane/runtime-cleanup.ts +574 -37
- package/src/runtime/control-plane/runtime-generation.ts +6 -0
- package/src/runtime/control-plane/workspace/semantic-sync-service.ts +465 -0
- package/src/runtime/control-plane/workspace/workspace-store.ts +80 -0
- package/src/runtime/diagnostics/incident-repair.ts +406 -0
- package/src/runtime/diagnostics/mcp-timing.ts +125 -6
- package/src/runtime/diagnostics/performance.ts +6 -14
- package/src/runtime/diagnostics/read-only-tool.ts +1 -1
- package/src/runtime/evidence/artifact-store.ts +65 -2
- package/src/runtime/evidence/operational-shadow.ts +38 -0
- package/src/runtime/evidence/process-check-receipt.ts +5 -43
- package/src/runtime/evidence/sensitive-output.ts +4 -4
- package/src/runtime/execution/jobs/receipt-recovery.ts +1 -1
- package/src/runtime/execution/jobs/store.ts +121 -2
- package/src/runtime/execution/managed-workspace.ts +55 -5
- package/src/runtime/execution/process-runtime/canary.ts +19 -5
- package/src/runtime/execution/process-runtime/check-classification.ts +48 -0
- package/src/runtime/execution/process-runtime/check-facade.ts +23 -9
- package/src/runtime/execution/process-runtime/check-receipt.ts +1 -1
- package/src/runtime/execution/process-runtime/check-result-retention.ts +132 -0
- package/src/runtime/execution/process-runtime/check-result.ts +73 -2
- package/src/runtime/execution/process-runtime/check-runner-sidecar.ts +42 -3
- package/src/runtime/execution/process-runtime/check-storage.ts +187 -0
- package/src/runtime/execution/process-runtime/command-facade.ts +38 -87
- package/src/runtime/execution/process-runtime/index.ts +1 -0
- package/src/runtime/execution/process-runtime/interactive-admission.ts +2 -1
- package/src/runtime/execution/process-runtime/lightweight-managed.ts +32 -1
- package/src/runtime/execution/process-runtime/persisted-check.ts +128 -21
- package/src/runtime/execution/process-runtime/resource-claims.ts +60 -3
- package/src/runtime/execution/process-runtime/runtime.ts +132 -15
- package/src/runtime/execution/process-runtime/types.ts +22 -0
- package/src/runtime/execution/workers/executor.ts +16 -13
- package/src/runtime/execution/workers/worker-entry.ts +1 -1
- package/src/runtime/gateway/mcp/execution-tools.ts +2 -207
- package/src/runtime/gateway/mcp/legacy-ios-tool-adapter.ts +2 -61
- package/src/runtime/gateway/mcp/persisted-check-process.ts +2 -3
- package/src/runtime/gateway/mcp/process-tools.ts +2 -289
- package/src/runtime/gateway/mcp/router.ts +2 -1125
- package/src/runtime/gateway/mcp/runtime-tool-definitions.ts +2 -565
- package/src/runtime/gateway/mcp/runtime-tools.ts +2 -6655
- package/src/runtime/gateway/mcp/work-validation-reconciler.ts +2 -1
- package/src/runtime/maintenance/cleanup.ts +38 -24
- package/src/runtime/platform/browser-product-discovery.ts +174 -0
- package/src/runtime/platform/computer-platform.ts +14 -0
- package/src/runtime/platform/executable-discovery.ts +54 -0
- package/src/runtime/platform/fetch-http-server.ts +90 -0
- package/src/runtime/platform/service-manager.ts +138 -0
- package/src/runtime/platform/windows-host-discovery.ts +118 -0
- package/src/runtime/platform/windows-recovery-deployment.ts +67 -0
- package/src/runtime/plugins/action-application.ts +135 -0
- package/src/runtime/plugins/app-store-connect-adapter.ts +9 -21
- package/src/runtime/plugins/browser-adapter.ts +140 -100
- package/src/runtime/plugins/browser-automation-service.ts +15 -36
- package/src/runtime/plugins/browser-macos-bridge.ts +69 -21
- package/src/runtime/plugins/browser-manifest-surface.ts +30 -6
- package/src/runtime/plugins/browser-native-session-service.ts +1 -1
- package/src/runtime/plugins/browser-provider-registry.ts +2 -109
- package/src/runtime/plugins/browser-registration.ts +16 -0
- package/src/runtime/plugins/browser-runtime-contract.ts +2 -154
- package/src/runtime/plugins/browser-runtime.ts +41 -9
- package/src/runtime/plugins/browser-session-authority.ts +101 -291
- package/src/runtime/plugins/browser-session-legacy-migration.ts +203 -0
- package/src/runtime/plugins/browser-session-store.ts +229 -36
- package/src/runtime/plugins/browser-session-types.ts +2 -98
- package/src/runtime/plugins/capability-authorization-grants.ts +17 -0
- package/src/runtime/plugins/catalog-trust.ts +82 -0
- package/src/runtime/plugins/computer-registration.ts +839 -0
- package/src/runtime/plugins/config-store.ts +119 -0
- package/src/runtime/plugins/desktop-operator-external-policy.ts +932 -0
- package/src/runtime/plugins/desktop-operator-registration.ts +111 -8
- package/src/runtime/plugins/errors.ts +8 -1
- package/src/runtime/plugins/external-adapter.ts +45 -322
- package/src/runtime/plugins/external-provider-policy.ts +48 -0
- package/src/runtime/plugins/external-registration.ts +9 -1
- package/src/runtime/plugins/external-unix-socket-probe.cjs +9 -3
- package/src/runtime/plugins/external-unix-socket.ts +64 -193
- package/src/runtime/plugins/first-party-external-registration.ts +163 -0
- package/src/runtime/plugins/first-party-registry.ts +3 -1
- package/src/runtime/plugins/github-adapter.ts +1 -2
- package/src/runtime/plugins/gmail-adapter.ts +6 -6
- package/src/runtime/plugins/google-calendar-adapter.ts +6 -6
- package/src/runtime/plugins/google-shared.ts +31 -46
- package/src/runtime/plugins/google-tasks-adapter.ts +6 -6
- package/src/runtime/plugins/ios-adapter.ts +1 -2
- package/src/runtime/plugins/local-system-adapter.ts +180 -0
- package/src/runtime/plugins/macos-capability-broker.ts +15 -105
- package/src/runtime/plugins/resend-adapter.ts +15 -24
- package/src/runtime/plugins/store.ts +493 -51
- package/src/runtime/plugins/types.ts +8 -1
- package/src/runtime/plugins/uu-remote-rescue-registration.ts +155 -0
- package/src/runtime/plugins/windows-host-recovery-install.ts +68 -0
- package/src/runtime/plugins/windows-host-recovery-registration.ts +85 -0
- package/src/runtime/plugins/xiaohongshu-publish.ts +86 -430
- package/src/runtime/projections/controller-context.ts +12 -6
- package/src/runtime/projections/git-status-sampler.ts +73 -22
- package/src/runtime/projections/materialized-view.ts +14 -8
- package/src/runtime/recovery/capability-action-application.ts +228 -0
- package/src/runtime/recovery/index.ts +3 -0
- package/src/runtime/recovery/lifecycle-inventory.ts +338 -0
- package/src/runtime/recovery/local-jobs-repair.ts +14 -14
- package/src/runtime/recovery/maintenance-executor.ts +358 -76
- package/src/runtime/recovery/quarantine-retention.ts +256 -0
- package/src/runtime/release/identity.ts +6 -0
- package/src/runtime/resources/leases/store.ts +1 -1
- package/src/runtime/root/active-execution-power-assertion.ts +139 -0
- package/src/runtime/root/assistant-learning-loop.ts +45 -0
- package/src/runtime/root/browser-session-composition.ts +59 -0
- package/src/runtime/root/computer-composition.ts +191 -0
- package/src/runtime/root/computer-target-composition.ts +16 -0
- package/src/runtime/root/computer-target-persistence.ts +39 -0
- package/src/runtime/root/controller-round-composition.ts +107 -0
- package/src/runtime/root/controller-services.ts +2 -0
- package/src/runtime/root/entry.ts +4 -3
- package/src/runtime/root/gateway-adapter.ts +3 -3
- package/src/runtime/root/local-bridge.ts +5 -2
- package/src/runtime/root/mcp-transport.ts +5 -5
- package/src/runtime/root/package-connector-service.ts +108 -56
- package/src/runtime/root/package-runtime-release.ts +85 -18
- package/src/runtime/root/package-runtime-service.ts +60 -97
- package/src/runtime/root/release-execution-canary.ts +71 -0
- package/src/runtime/root/release-loader.ts +68 -0
- package/src/runtime/root/release-manifest.ts +106 -2
- package/src/runtime/root/release-materialize.ts +103 -4
- package/src/runtime/root/release-store.ts +31 -2
- package/src/runtime/root/runtime.ts +240 -25
- package/src/runtime/root/scheduled-controller-composition.ts +51 -0
- package/src/runtime/root/scheduler.ts +12 -4
- package/src/runtime/root/service-runner.ts +1 -1
- package/src/runtime/root/service.ts +41 -16
- package/src/runtime/root/status.ts +1 -0
- package/src/runtime/root/types.ts +24 -1
- package/src/runtime/shared/local-bridge-surface.ts +1 -1
- package/src/runtime/shared/process-environment.ts +19 -4
- package/src/runtime/shared/storage-capacity.ts +68 -0
- package/src/runtime/standalone-recovery/controller-home-migration.ts +1116 -0
- package/src/runtime/standalone-recovery/core.ts +984 -230
- package/src/runtime/standalone-recovery/entry.ts +182 -51
- package/src/runtime/standalone-recovery/independent-host-rescue.ts +172 -0
- package/src/runtime/standalone-recovery/installer.ts +212 -104
- package/src/runtime/standalone-recovery/mcp-server.ts +243 -0
- package/src/runtime/standalone-recovery/mutation-identity-contract.ts +21 -0
- package/src/runtime/standalone-recovery/operation-lock.ts +160 -0
- package/src/runtime/standalone-recovery/performance.ts +135 -0
- package/src/runtime/standalone-recovery/release.ts +5 -0
- package/src/runtime/watchdog/workflow-watchdog.ts +1 -1
- package/src/runtime/workflow/schedules/chatgpt-automation-migration.ts +1 -1
- package/src/runtime/workflow/schedules/engine.ts +191 -343
- package/src/runtime/workflow/schedules/pending-activation.ts +1 -1
- package/src/runtime/workflow/schedules/settlement.ts +2 -191
- package/src/runtime/workflow/schedules/store.ts +2 -607
- package/src/runtime/workflow/schedules/types.ts +2 -127
- package/src/runtime/workflow/schedules/work-continuation.ts +180 -15
- package/src/runtime/workflows/first-party/xiaohongshu.ts +122 -0
- package/src/runtime/workflows/runtime.ts +486 -0
- package/src/testing/test-governance.ts +90 -29
- package/assets/templates/helpers/archive-architecture-request.sh +0 -244
- package/docs/architecture/history/README.md +0 -7
- package/docs/architecture/history/chatgpt-handoff-facade.md +0 -397
- package/docs/architecture/history/global-hook-runtime.md +0 -166
- package/docs/architecture/history/ios-semantic-automation-provider-v2.md +0 -170
- package/docs/architecture/requests/archive/2026/20260528-154740-ai-hooks-ai-hooks-hook-input-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-154816-assets-hooks-assets-hooks-hook-input-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-161851-ai-harness-workflow-contract-json-ai-harness-workflow-contract-json.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-161853-scripts-check-task-workflow-sh-scripts-check-task-workflow-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-161911-assets-workflow-contract-v1-json-assets-workflow-contract-v1-json.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-222416-ai-hooks-ai-hooks-hook-input-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-222444-ai-hooks-ai-hooks-pre-edit-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-222457-ai-hooks-ai-hooks-pre-edit-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-222648-ai-hooks-ai-hooks-pre-edit-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-222705-ai-hooks-ai-hooks-worktree-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-222726-ai-hooks-ai-hooks-prompt-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-222727-ai-hooks-ai-hooks-prompt-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-224736-ai-hooks-ai-hooks-hook-input-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-233259-ai-hooks-ai-hooks-prompt-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-233335-assets-hooks-assets-hooks-prompt-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-233456-ai-hooks-ai-hooks-prompt-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-233457-assets-hooks-assets-hooks-prompt-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-233637-ai-hooks-ai-hooks-prompt-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260528-233638-assets-hooks-assets-hooks-prompt-guard-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260529-000322-root-package-json.md +0 -59
- package/docs/architecture/requests/archive/2026/20260529-004238-ai-hooks-ai-hooks-lib-workflow-state-sh.md +0 -59
- package/docs/architecture/requests/archive/2026/20260529-020654-ai-harness-policy-json-ai-harness-policy-json.md +0 -63
- package/docs/architecture/requests/archive/2026/20260529-094236-ai-hooks-ai-hooks-prompt-guard-sh.md +0 -60
- package/docs/architecture/requests/archive/2026/20260529-094323-assets-hooks-assets-hooks-prompt-guard-sh.md +0 -60
- package/docs/architecture/requests/archive/2026/20260529-094331-ai-hooks-ai-hooks-prompt-guard-sh.md +0 -60
- package/docs/architecture/requests/archive/2026/20260529-094339-assets-hooks-assets-hooks-prompt-guard-sh.md +0 -60
- package/docs/architecture/requests/archive/2026/20260529-094430-assets-hooks-assets-hooks-session-start-context-sh.md +0 -60
- package/docs/architecture/requests/archive/2026/20260529-094446-ai-hooks-ai-hooks-session-start-context-sh.md +0 -60
- package/docs/architecture/requests/archive/2026/20260613-002040-runtime-harness-hook-adapters.md +0 -68
- package/docs/architecture/requests/archive/2026/root.md +0 -68
- package/docs/architecture/requests/archive/2026/runtime-harness-hook-adapters.md +0 -76
- package/docs/architecture/requests/archive/2026/verification-evals-checks.md +0 -68
- package/docs/architecture/requests/archive/2026/workflow-engine-contract-assets.md +0 -69
- package/scripts/archive-architecture-request.sh +0 -4
- package/scripts/run-skill-evals.ts +0 -1022
|
@@ -1,8 +1,11 @@
|
|
|
1
|
+
import { assertRuntimePerformanceEvidence, measureRuntimePerformance, samePerformanceIdentity, type RuntimePerformanceDependencies, type RuntimePerformanceEvidence, type RuntimePerformanceIdentity } from './performance';
|
|
1
2
|
import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'crypto';
|
|
2
3
|
import { spawn, spawnSync } from 'child_process';
|
|
3
4
|
import { chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, rmSync, writeFileSync } from 'fs';
|
|
4
|
-
import { homedir } from 'os';
|
|
5
|
-
import { basename, dirname, isAbsolute, join, resolve } from 'path';
|
|
5
|
+
import { homedir, hostname } from 'os';
|
|
6
|
+
import { basename, delimiter, dirname, isAbsolute, join, resolve } from 'path';
|
|
7
|
+
import { assertStorageHeadroom } from '../shared/storage-capacity';
|
|
8
|
+
import { resolveBunExecutable } from '../shared/process-environment';
|
|
6
9
|
import { observeRuntimeStatus } from '../root/status';
|
|
7
10
|
import {
|
|
8
11
|
activeRuntimeEntrypoint,
|
|
@@ -12,8 +15,19 @@ import {
|
|
|
12
15
|
inspectForgeRuntimeLaunchAgentContract,
|
|
13
16
|
readForgeRuntimeServiceConfig,
|
|
14
17
|
} from '../root/service';
|
|
18
|
+
import { systemdUserUnitName, systemdUserUnitPath } from '../../cli/controller/systemd-user';
|
|
19
|
+
import {
|
|
20
|
+
openAiSecureTunnelConnectArgs,
|
|
21
|
+
openAiSecureTunnelStatusArgs,
|
|
22
|
+
parseOpenAiSecureTunnelRuntimeStatus,
|
|
23
|
+
type OpenAiSecureTunnelRuntimeObservation,
|
|
24
|
+
} from '../../../adapters/mcp/tunnels/openai-secure-tunnel';
|
|
25
|
+
import {
|
|
26
|
+
renderPackageRuntimeSystemdUserService,
|
|
27
|
+
writePackageRuntimeSystemdUserService,
|
|
28
|
+
} from '../root/package-runtime-service';
|
|
15
29
|
import { loadRuntimeReleaseManifest } from '../root/release-manifest';
|
|
16
|
-
import { assertRuntimeReleaseFiles, stageRuntimeReleaseFromCandidateSource, type StagedRuntimeRelease } from '../root/release-materialize';
|
|
30
|
+
import { assertRuntimeReleaseExecutionCanaries, assertRuntimeReleaseFiles, stageRuntimeReleaseFromCandidateSource, type RuntimeReleaseExecutionCanaryDependencies, type StagedRuntimeRelease } from '../root/release-materialize';
|
|
17
31
|
import {
|
|
18
32
|
publishRuntimeRelease,
|
|
19
33
|
readRuntimeReleaseAuthority,
|
|
@@ -22,7 +36,7 @@ import {
|
|
|
22
36
|
type RuntimeReleaseAuthority,
|
|
23
37
|
} from '../root/release-store';
|
|
24
38
|
import type { RuntimeReleaseManifest } from '../root/types';
|
|
25
|
-
import { ensurePackageConnectorService, packageConnectorServicePaths, type PackageConnectorReleaseBinding } from '../root/package-connector-service';
|
|
39
|
+
import { ensurePackageConnectorService, packageConnectorAuthMode, packageConnectorServicePaths, type PackageConnectorReleaseBinding } from '../root/package-connector-service';
|
|
26
40
|
import { createRecoveryHttpTransport, type RecoveryHttpTransport } from './http-transport';
|
|
27
41
|
import { observeRecoveryWatchdogHealth } from './watchdog-heartbeat';
|
|
28
42
|
import { RECOVERY_GATEWAY_LABEL, RECOVERY_WATCHDOG_LABEL } from './service-labels';
|
|
@@ -32,20 +46,51 @@ import {
|
|
|
32
46
|
rollbackStoppedRepoLocalControllerHomeStorage,
|
|
33
47
|
type ControllerHomeStorageMigration,
|
|
34
48
|
} from '../../cli/repositories/controller-home';
|
|
49
|
+
import { readCurrentRecoveryRelease } from './release';
|
|
50
|
+
import { recoveryOperationLockPath } from './operation-lock';
|
|
51
|
+
import { RECOVERY_MUTATION_IDENTITY_FIELDS, type RecoveryMutationIdentityArguments } from './mutation-identity-contract';
|
|
35
52
|
|
|
36
53
|
/** Standalone recovery reads only canonical Runtime observation and whole-release authority. */
|
|
37
|
-
|
|
38
|
-
platform: 'launchd';
|
|
39
|
-
label: string;
|
|
40
|
-
plistPath?: string;
|
|
54
|
+
interface PublicTunnelRecoveryPolicy {
|
|
41
55
|
minimumFailures?: number;
|
|
42
56
|
minimumFailureDurationMs?: number;
|
|
43
57
|
cooldownMs?: number;
|
|
44
58
|
postRestartVerifyTimeoutMs?: number;
|
|
45
59
|
}
|
|
46
60
|
|
|
47
|
-
export interface
|
|
61
|
+
export interface LaunchdPublicTunnelServiceConfig extends PublicTunnelRecoveryPolicy {
|
|
48
62
|
platform: 'launchd';
|
|
63
|
+
label: string;
|
|
64
|
+
plistPath?: string;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface OpenAiSecureTunnelServiceConfig extends PublicTunnelRecoveryPolicy {
|
|
68
|
+
platform: 'openai-secure-tunnel';
|
|
69
|
+
/** Stable tunnel-client runtime identity. Recovery must not repoint another alias. */
|
|
70
|
+
alias: string;
|
|
71
|
+
/** Non-secret OpenAI tunnel identity. */
|
|
72
|
+
tunnelId: string;
|
|
73
|
+
/** Loopback MCP endpoint owned by the local service behind this tunnel. */
|
|
74
|
+
mcpServerUrl: string;
|
|
75
|
+
/** Secret reference only (env:NAME or file:/absolute/path). Recovery never reads the secret. */
|
|
76
|
+
runtimeApiKeyRef?: string;
|
|
77
|
+
profile?: string;
|
|
78
|
+
profileDir?: string;
|
|
79
|
+
adminProfile?: string;
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export type PublicTunnelServiceConfig = LaunchdPublicTunnelServiceConfig | OpenAiSecureTunnelServiceConfig;
|
|
83
|
+
|
|
84
|
+
export interface SystemdUserRecoveryTunnelServiceConfig extends PublicTunnelRecoveryPolicy {
|
|
85
|
+
platform: 'systemd-user';
|
|
86
|
+
/** Exact systemd --user unit owned by the dedicated Recovery public tunnel. */
|
|
87
|
+
unitName: string;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export type RecoveryTunnelServiceConfig = PublicTunnelServiceConfig | SystemdUserRecoveryTunnelServiceConfig;
|
|
91
|
+
|
|
92
|
+
export interface PrimaryRuntimeServiceConfig {
|
|
93
|
+
platform: 'launchd' | 'systemd-user';
|
|
49
94
|
minimumFailures?: number;
|
|
50
95
|
minimumFailureDurationMs?: number;
|
|
51
96
|
restartCooldownMs?: number;
|
|
@@ -56,10 +101,7 @@ export interface PrimaryRuntimeServiceConfig {
|
|
|
56
101
|
postRestartVerifyTimeoutMs?: number;
|
|
57
102
|
}
|
|
58
103
|
|
|
59
|
-
|
|
60
|
-
platform: 'launchd';
|
|
61
|
-
label: string;
|
|
62
|
-
plistPath?: string;
|
|
104
|
+
interface PrimaryConnectorRecoveryPolicy {
|
|
63
105
|
/** Local OAuth MCP endpoint used to distinguish Connector failure from tunnel failure. */
|
|
64
106
|
localMcpUrl?: string;
|
|
65
107
|
minimumFailures?: number;
|
|
@@ -69,16 +111,32 @@ export interface PrimaryConnectorServiceConfig {
|
|
|
69
111
|
postRestartVerifyTimeoutMs?: number;
|
|
70
112
|
}
|
|
71
113
|
|
|
114
|
+
export interface LaunchdPrimaryConnectorServiceConfig extends PrimaryConnectorRecoveryPolicy {
|
|
115
|
+
platform: 'launchd';
|
|
116
|
+
label: string;
|
|
117
|
+
plistPath?: string;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export interface SystemdPrimaryConnectorServiceConfig extends PrimaryConnectorRecoveryPolicy {
|
|
121
|
+
platform: 'systemd-user';
|
|
122
|
+
/** Optional assertion. When present it must equal the package Connector label derived from Controller Home. */
|
|
123
|
+
label?: string;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
export type PrimaryConnectorServiceConfig = LaunchdPrimaryConnectorServiceConfig | SystemdPrimaryConnectorServiceConfig;
|
|
127
|
+
|
|
72
128
|
export interface RecoveryConfig {
|
|
73
129
|
schemaVersion: 1;
|
|
74
130
|
controllerHome: string;
|
|
75
131
|
publicMcpUrl?: string;
|
|
76
132
|
recoveryPublicUrl?: string;
|
|
77
|
-
recoveryTunnelService?:
|
|
133
|
+
recoveryTunnelService?: RecoveryTunnelServiceConfig;
|
|
78
134
|
/** Optional public tunnel serving publicMcpUrl; independent from the OAuth Connector process. */
|
|
79
135
|
primaryPublicTunnelService?: PublicTunnelServiceConfig;
|
|
80
136
|
primaryRuntimeService?: PrimaryRuntimeServiceConfig;
|
|
81
137
|
primaryRuntimeSourceRoot?: string;
|
|
138
|
+
/** Repository Registry authority paired with primaryRuntimeSourceRoot. */
|
|
139
|
+
primaryRuntimeSourceRepositoryId?: string;
|
|
82
140
|
primaryConnectorService?: PrimaryConnectorServiceConfig;
|
|
83
141
|
mainMcpTokenFile?: string;
|
|
84
142
|
expectedToolFingerprint?: string;
|
|
@@ -86,6 +144,75 @@ export interface RecoveryConfig {
|
|
|
86
144
|
gateway?: { host: string; port: number; bearerTokenFile: string };
|
|
87
145
|
}
|
|
88
146
|
|
|
147
|
+
export interface RecoveryMachineIdentity {
|
|
148
|
+
schemaVersion: 1;
|
|
149
|
+
host: string;
|
|
150
|
+
platform: NodeJS.Platform;
|
|
151
|
+
controllerHome: string;
|
|
152
|
+
recovery: {
|
|
153
|
+
releaseRevision?: string;
|
|
154
|
+
manifestSha256?: string;
|
|
155
|
+
};
|
|
156
|
+
targetRuntime: {
|
|
157
|
+
id: string;
|
|
158
|
+
servicePlatform: 'launchd' | 'systemd-user';
|
|
159
|
+
serviceLabel: string;
|
|
160
|
+
activeReleaseId?: string;
|
|
161
|
+
authorityRevision?: number;
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function recoveryMachineIdentity(
|
|
166
|
+
config: RecoveryConfig,
|
|
167
|
+
dependencies: { host?: string; platform?: NodeJS.Platform } = {},
|
|
168
|
+
): RecoveryMachineIdentity {
|
|
169
|
+
const controllerHome = resolve(config.controllerHome);
|
|
170
|
+
const platform = dependencies.platform ?? process.platform;
|
|
171
|
+
const host = dependencies.host?.trim() || hostname();
|
|
172
|
+
const recoveryRelease = readCurrentRecoveryRelease(controllerHome);
|
|
173
|
+
const runtimeAuthority = readRuntimeReleaseAuthority(controllerHome);
|
|
174
|
+
const servicePlatform = (config.primaryRuntimeService ?? defaultPrimaryRuntimeServiceConfig(platform)).platform;
|
|
175
|
+
const serviceLabel = forgeRuntimeServicePaths(controllerHome).label;
|
|
176
|
+
const activeReleaseId = runtimeAuthority?.active.releaseId;
|
|
177
|
+
return {
|
|
178
|
+
schemaVersion: 1,
|
|
179
|
+
host,
|
|
180
|
+
platform,
|
|
181
|
+
controllerHome,
|
|
182
|
+
recovery: {
|
|
183
|
+
...(recoveryRelease ? { releaseRevision: recoveryRelease.releaseRevision, manifestSha256: recoveryRelease.manifestSha256 } : {}),
|
|
184
|
+
},
|
|
185
|
+
targetRuntime: {
|
|
186
|
+
id: `${servicePlatform}:${serviceLabel}:${activeReleaseId ?? 'none'}`,
|
|
187
|
+
servicePlatform,
|
|
188
|
+
serviceLabel,
|
|
189
|
+
...(activeReleaseId ? { activeReleaseId } : {}),
|
|
190
|
+
...(runtimeAuthority ? { authorityRevision: runtimeAuthority.revision } : {}),
|
|
191
|
+
},
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
export function recoveryMutationIdentityExpectations(identity: Pick<RecoveryMachineIdentity, 'host' | 'platform' | 'controllerHome' | 'recovery' | 'targetRuntime'>): RecoveryMutationIdentityArguments {
|
|
196
|
+
return {
|
|
197
|
+
expected_host: identity.host,
|
|
198
|
+
expected_platform: identity.platform,
|
|
199
|
+
expected_controller_home: identity.controllerHome,
|
|
200
|
+
expected_recovery_release: identity.recovery.releaseRevision ?? 'none',
|
|
201
|
+
expected_target_runtime: identity.targetRuntime.id,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export function assertRecoveryMutationIdentity(config: RecoveryConfig, args: Record<string, unknown>): RecoveryMachineIdentity {
|
|
206
|
+
const identity = recoveryMachineIdentity(config);
|
|
207
|
+
const expected = recoveryMutationIdentityExpectations(identity);
|
|
208
|
+
for (const field of RECOVERY_MUTATION_IDENTITY_FIELDS) {
|
|
209
|
+
const supplied = typeof args[field] === 'string' ? args[field].trim() : '';
|
|
210
|
+
if (!supplied) throw new Error(`RECOVERY_TARGET_IDENTITY_REQUIRED:${field}`);
|
|
211
|
+
if (supplied !== expected[field]) throw new Error(`RECOVERY_TARGET_IDENTITY_MISMATCH:${field}`);
|
|
212
|
+
}
|
|
213
|
+
return identity;
|
|
214
|
+
}
|
|
215
|
+
|
|
89
216
|
interface ReleaseEvidence {
|
|
90
217
|
path: string;
|
|
91
218
|
revision: string;
|
|
@@ -96,6 +223,7 @@ interface ReleaseEvidence {
|
|
|
96
223
|
releaseAuthorityRevision?: number;
|
|
97
224
|
releaseFencingTokenSha256?: string;
|
|
98
225
|
attestedAt?: string;
|
|
226
|
+
performance?: RuntimePerformanceEvidence;
|
|
99
227
|
}
|
|
100
228
|
|
|
101
229
|
interface KnownGoodStore {
|
|
@@ -143,7 +271,7 @@ export interface VerifyResult {
|
|
|
143
271
|
probes: Record<string, { ok: boolean; detail: string; status?: number; value?: unknown }>;
|
|
144
272
|
}
|
|
145
273
|
|
|
146
|
-
export interface VerifyStableRuntimeOptions {
|
|
274
|
+
export interface VerifyStableRuntimeOptions extends RuntimeReleaseExecutionCanaryDependencies {
|
|
147
275
|
probeMcpProtocol?: boolean;
|
|
148
276
|
}
|
|
149
277
|
|
|
@@ -281,10 +409,14 @@ function normalizeRecoveryReadOnlyTool(
|
|
|
281
409
|
return input;
|
|
282
410
|
}
|
|
283
411
|
|
|
412
|
+
export function defaultPrimaryRuntimeServiceConfig(platform: NodeJS.Platform = process.platform): PrimaryRuntimeServiceConfig {
|
|
413
|
+
return { platform: platform === 'linux' ? 'systemd-user' : 'launchd' };
|
|
414
|
+
}
|
|
415
|
+
|
|
284
416
|
const DEFAULT_CONFIG: Omit<RecoveryConfig, 'controllerHome'> = {
|
|
285
417
|
schemaVersion: 1,
|
|
286
418
|
readOnlyTool: { name: STABLE_RECOVERY_READ_ONLY_TOOL.name, arguments: { ...STABLE_RECOVERY_READ_ONLY_TOOL.arguments } },
|
|
287
|
-
primaryRuntimeService:
|
|
419
|
+
primaryRuntimeService: defaultPrimaryRuntimeServiceConfig(),
|
|
288
420
|
};
|
|
289
421
|
|
|
290
422
|
function json<T>(path: string): T | undefined {
|
|
@@ -292,16 +424,22 @@ function json<T>(path: string): T | undefined {
|
|
|
292
424
|
}
|
|
293
425
|
|
|
294
426
|
function writeJson(path: string, value: unknown): void {
|
|
427
|
+
const content = `${JSON.stringify(value, null, 2)}\n`;
|
|
428
|
+
assertStorageHeadroom(path, {
|
|
429
|
+
operation: 'standalone_recovery_state_write',
|
|
430
|
+
requiredBytes: Buffer.byteLength(content),
|
|
431
|
+
reserveBytes: 16 * 1024 * 1024,
|
|
432
|
+
});
|
|
295
433
|
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
296
434
|
const temporary = `${path}.${process.pid}.tmp`;
|
|
297
|
-
writeFileSync(temporary,
|
|
435
|
+
writeFileSync(temporary, content, { encoding: 'utf8', mode: 0o600 });
|
|
298
436
|
try { chmodSync(temporary, 0o600); } catch { /* best effort */ }
|
|
299
437
|
renameSync(temporary, path);
|
|
300
438
|
}
|
|
301
439
|
|
|
302
440
|
function recoveryRoot(config: RecoveryConfig): string { return join(resolve(config.controllerHome), 'recovery'); }
|
|
303
441
|
function statePath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'state', 'known-good.json'); }
|
|
304
|
-
function lockPath(config: RecoveryConfig): string { return
|
|
442
|
+
function lockPath(config: RecoveryConfig): string { return recoveryOperationLockPath(config.controllerHome); }
|
|
305
443
|
function auditPath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'audit', 'recovery.jsonl'); }
|
|
306
444
|
function watchdogDiagnosticPath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'state', 'watchdog-diagnostics.json'); }
|
|
307
445
|
function quarantinePath(config: RecoveryConfig): string { return join(recoveryRoot(config), 'state', 'quarantine.json'); }
|
|
@@ -321,7 +459,7 @@ export function saveWatchdogState(config: RecoveryConfig, state: WatchdogState):
|
|
|
321
459
|
return persisted;
|
|
322
460
|
}
|
|
323
461
|
|
|
324
|
-
function configuredRecoveryTunnel(config: RecoveryConfig):
|
|
462
|
+
function configuredRecoveryTunnel(config: RecoveryConfig): RecoveryTunnelServiceConfig | undefined {
|
|
325
463
|
return config.recoveryTunnelService;
|
|
326
464
|
}
|
|
327
465
|
|
|
@@ -356,6 +494,7 @@ export function loadRecoveryConfig(controllerHome: string, explicit?: string): R
|
|
|
356
494
|
...(loaded.primaryPublicTunnelService ? { primaryPublicTunnelService: loaded.primaryPublicTunnelService } : {}),
|
|
357
495
|
...(loaded.primaryRuntimeService ? { primaryRuntimeService: loaded.primaryRuntimeService } : {}),
|
|
358
496
|
...(typeof loaded.primaryRuntimeSourceRoot === 'string' ? { primaryRuntimeSourceRoot: resolve(loaded.primaryRuntimeSourceRoot) } : {}),
|
|
497
|
+
...(typeof loaded.primaryRuntimeSourceRepositoryId === 'string' && loaded.primaryRuntimeSourceRepositoryId.trim() ? { primaryRuntimeSourceRepositoryId: loaded.primaryRuntimeSourceRepositoryId.trim() } : {}),
|
|
359
498
|
...(loaded.primaryConnectorService ? { primaryConnectorService: loaded.primaryConnectorService } : {}),
|
|
360
499
|
...(typeof loaded.mainMcpTokenFile === 'string' ? { mainMcpTokenFile: loaded.mainMcpTokenFile } : {}),
|
|
361
500
|
...(typeof loaded.expectedToolFingerprint === 'string' ? { expectedToolFingerprint: loaded.expectedToolFingerprint } : {}),
|
|
@@ -454,7 +593,7 @@ function watchdogProbeComponent(name: string): RecoveryWatchdogDiagnosticEvidenc
|
|
|
454
593
|
if (name === 'active_gateway') return 'gateway';
|
|
455
594
|
if (name === 'recovery_gateway') return 'recovery_gateway';
|
|
456
595
|
if (name === 'recovery_watchdog') return 'recovery_watchdog';
|
|
457
|
-
if (name === 'recovery_external_http') return 'recovery_tunnel';
|
|
596
|
+
if (name === 'recovery_external_http' || name === 'recovery_tunnel_runtime') return 'recovery_tunnel';
|
|
458
597
|
return 'public_mcp';
|
|
459
598
|
}
|
|
460
599
|
|
|
@@ -618,9 +757,11 @@ async function withLock<T>(
|
|
|
618
757
|
}
|
|
619
758
|
|
|
620
759
|
export async function runtimeStatus(config: RecoveryConfig) {
|
|
760
|
+
const identity = recoveryMachineIdentity(config);
|
|
621
761
|
const observation = observeRuntimeStatus(config.controllerHome);
|
|
622
762
|
const watchdog = loadWatchdogState(config);
|
|
623
763
|
return {
|
|
764
|
+
identity,
|
|
624
765
|
...observation,
|
|
625
766
|
recoveryWatchdog: {
|
|
626
767
|
lastDecision: watchdog.lastDecision,
|
|
@@ -648,7 +789,11 @@ async function probe(transport: RecoveryHttpTransport, url: string, timeoutMs =
|
|
|
648
789
|
} finally { clearTimeout(timer); }
|
|
649
790
|
}
|
|
650
791
|
|
|
651
|
-
async function probeExternalMcp(
|
|
792
|
+
async function probeExternalMcp(
|
|
793
|
+
transport: RecoveryHttpTransport,
|
|
794
|
+
url: string,
|
|
795
|
+
options: { acceptOAuthChallenge?: boolean } = {},
|
|
796
|
+
): Promise<{ ok: boolean; detail: string; status?: number }> {
|
|
652
797
|
const controller = new AbortController();
|
|
653
798
|
const timer = setTimeout(() => controller.abort('RECOVERY_HTTP_TIMEOUT'), 4_000);
|
|
654
799
|
try {
|
|
@@ -669,8 +814,13 @@ async function probeExternalMcp(transport: RecoveryHttpTransport, url: string):
|
|
|
669
814
|
signal: controller.signal,
|
|
670
815
|
});
|
|
671
816
|
const challenge = response.headers['www-authenticate'] ?? '';
|
|
672
|
-
const
|
|
673
|
-
|
|
817
|
+
const oauthChallenge = response.status === 401 && /\bBearer\b/i.test(challenge);
|
|
818
|
+
const acceptsOAuthChallenge = options.acceptOAuthChallenge ?? true;
|
|
819
|
+
const ok = response.ok || (acceptsOAuthChallenge && oauthChallenge);
|
|
820
|
+
const detail = oauthChallenge
|
|
821
|
+
? acceptsOAuthChallenge ? 'HTTP 401 OAuth challenge' : 'HTTP 401 unexpected OAuth challenge for unauthenticated Connector'
|
|
822
|
+
: `HTTP ${response.status}`;
|
|
823
|
+
return { ok, detail, status: response.status };
|
|
674
824
|
} catch (error) {
|
|
675
825
|
return { ok: false, detail: error instanceof Error ? error.message.slice(0, 180) : 'request failed' };
|
|
676
826
|
} finally { clearTimeout(timer); }
|
|
@@ -687,6 +837,7 @@ function publicGatewayReadinessEndpoint(endpoint: string): string {
|
|
|
687
837
|
async function probePublicGatewayReadiness(
|
|
688
838
|
transport: RecoveryHttpTransport,
|
|
689
839
|
endpoint: string,
|
|
840
|
+
options: { acceptOAuthChallenge?: boolean } = {},
|
|
690
841
|
): Promise<{ ok: boolean; detail: string; status?: number; value?: unknown }> {
|
|
691
842
|
const controller = new AbortController();
|
|
692
843
|
const timer = setTimeout(() => controller.abort('RECOVERY_HTTP_TIMEOUT'), 4_000);
|
|
@@ -707,7 +858,7 @@ async function probePublicGatewayReadiness(
|
|
|
707
858
|
// initialize response or the expected OAuth Bearer challenge proves the
|
|
708
859
|
// legacy Connector transport is reachable without weakening 5xx/timeout
|
|
709
860
|
// failure handling.
|
|
710
|
-
const legacy = await probeExternalMcp(transport, endpoint);
|
|
861
|
+
const legacy = await probeExternalMcp(transport, endpoint, options);
|
|
711
862
|
return { ...legacy, detail: `legacy-mcp ${legacy.detail}` };
|
|
712
863
|
}
|
|
713
864
|
let sessionCapacity: unknown;
|
|
@@ -742,6 +893,19 @@ function connectorCapacityRecoveryRecommended(verified: VerifyResult): boolean {
|
|
|
742
893
|
);
|
|
743
894
|
}
|
|
744
895
|
|
|
896
|
+
function canonicalRuntimeSafeForTargetedConnectorRecovery(verified: VerifyResult): boolean {
|
|
897
|
+
// Targeted Connector/public-transport recovery mutates no Runtime release or
|
|
898
|
+
// Work authority. Use durable Runtime ownership + release/execution evidence
|
|
899
|
+
// as the safety fence; a single failed HTTP gateway/MCP observation is the
|
|
900
|
+
// symptom this action may need to repair and must not deadlock the recovery path.
|
|
901
|
+
return verified.runtime.ok
|
|
902
|
+
&& verified.runtime.running
|
|
903
|
+
&& verified.runtime.ready
|
|
904
|
+
&& !verified.runtime.stale
|
|
905
|
+
&& verified.releases.coherent !== false
|
|
906
|
+
&& verified.probes.runtime_execution_surface?.ok !== false;
|
|
907
|
+
}
|
|
908
|
+
|
|
745
909
|
function mainToken(config: RecoveryConfig): string | undefined {
|
|
746
910
|
const candidate = config.mainMcpTokenFile ?? join(config.controllerHome, 'mcp', 'mcp.tokens.json');
|
|
747
911
|
const parsed = json<{ bearerToken?: unknown }>(candidate);
|
|
@@ -839,6 +1003,146 @@ function runtimeHealthEndpoint(endpoint: string): string {
|
|
|
839
1003
|
return url.toString();
|
|
840
1004
|
}
|
|
841
1005
|
|
|
1006
|
+
async function observeOpenAiTunnelRuntime(
|
|
1007
|
+
configured: OpenAiSecureTunnelServiceConfig,
|
|
1008
|
+
runCommand: CommandRunner = command,
|
|
1009
|
+
): Promise<OpenAiSecureTunnelRuntimeObservation> {
|
|
1010
|
+
let args: string[];
|
|
1011
|
+
try {
|
|
1012
|
+
args = openAiSecureTunnelStatusArgs(configured.alias, configured);
|
|
1013
|
+
} catch (error) {
|
|
1014
|
+
return {
|
|
1015
|
+
ok: false, running: false, healthy: false, ready: false, tunnelMatches: false, endpointMatches: false,
|
|
1016
|
+
alias: configured.alias, tunnelId: configured.tunnelId,
|
|
1017
|
+
detail: error instanceof Error ? error.message : 'OpenAI tunnel runtime configuration is invalid',
|
|
1018
|
+
};
|
|
1019
|
+
}
|
|
1020
|
+
const status = await runCommand('tunnel-client', args, 15_000);
|
|
1021
|
+
if (!status.stdout.trim()) {
|
|
1022
|
+
const local = await observeOpenAiTunnelLocalHealthFallback(configured);
|
|
1023
|
+
if (local) return local;
|
|
1024
|
+
return {
|
|
1025
|
+
ok: false, running: false, healthy: false, ready: false, tunnelMatches: false, endpointMatches: false,
|
|
1026
|
+
alias: configured.alias, tunnelId: configured.tunnelId,
|
|
1027
|
+
detail: status.stderr.trim() || 'OpenAI tunnel runtime status is unavailable',
|
|
1028
|
+
};
|
|
1029
|
+
}
|
|
1030
|
+
return parseOpenAiSecureTunnelRuntimeStatus(status.stdout, {
|
|
1031
|
+
alias: configured.alias, tunnelId: configured.tunnelId, mcpServerUrl: configured.mcpServerUrl,
|
|
1032
|
+
});
|
|
1033
|
+
}
|
|
1034
|
+
|
|
1035
|
+
/**
|
|
1036
|
+
* tunnel-client's structured status command can itself be delayed by a
|
|
1037
|
+
* control-plane query. When that happens, retain a bounded observation of the
|
|
1038
|
+
* locally supervised runtime instead of treating a live, identity-matched
|
|
1039
|
+
* alias as failed. This reads only the client-owned profile and loopback
|
|
1040
|
+
* health URL; it never creates, repoints, or restarts a tunnel.
|
|
1041
|
+
*/
|
|
1042
|
+
export async function observeOpenAiTunnelLocalHealthFallback(
|
|
1043
|
+
configured: OpenAiSecureTunnelServiceConfig,
|
|
1044
|
+
options: {
|
|
1045
|
+
request?: (url: string, init: { signal: AbortSignal }) => Promise<{ ok: boolean; status: number }>;
|
|
1046
|
+
} = {},
|
|
1047
|
+
): Promise<OpenAiSecureTunnelRuntimeObservation | undefined> {
|
|
1048
|
+
if (!configured.profile?.trim() || !configured.profileDir?.trim()) return undefined;
|
|
1049
|
+
const profilePath = join(configured.profileDir, `${configured.profile}.yaml`);
|
|
1050
|
+
let profile: string;
|
|
1051
|
+
try { profile = readFileSync(profilePath, 'utf8'); } catch { return undefined; }
|
|
1052
|
+
const identityMatches = profile.includes(configured.tunnelId);
|
|
1053
|
+
const endpointMatches = profile.includes(configured.mcpServerUrl);
|
|
1054
|
+
const urlFileMatch = profile.match(/["']?url_file["']?\s*:\s*["']([^"']+)["']/i);
|
|
1055
|
+
if (!urlFileMatch) return undefined;
|
|
1056
|
+
let base: URL;
|
|
1057
|
+
try {
|
|
1058
|
+
base = new URL(readFileSync(urlFileMatch[1].trim(), 'utf8').trim());
|
|
1059
|
+
if (base.protocol !== 'http:' || !['127.0.0.1', 'localhost', '::1'].includes(base.hostname)) return undefined;
|
|
1060
|
+
} catch { return undefined; }
|
|
1061
|
+
const request = options.request ?? ((url: string, init: { signal: AbortSignal }) => fetch(url, init));
|
|
1062
|
+
const probe = async (pathname: '/healthz' | '/readyz'): Promise<boolean> => {
|
|
1063
|
+
const url = new URL(base);
|
|
1064
|
+
url.pathname = pathname;
|
|
1065
|
+
url.search = '';
|
|
1066
|
+
url.hash = '';
|
|
1067
|
+
const controller = new AbortController();
|
|
1068
|
+
const timer = setTimeout(() => controller.abort('RECOVERY_TUNNEL_HEALTH_TIMEOUT'), 4_000);
|
|
1069
|
+
try { return (await request(url.toString(), { signal: controller.signal })).ok; } catch { return false; } finally { clearTimeout(timer); }
|
|
1070
|
+
};
|
|
1071
|
+
const [healthy, ready] = await Promise.all([probe('/healthz'), probe('/readyz')]);
|
|
1072
|
+
const ok = healthy && ready && identityMatches && endpointMatches;
|
|
1073
|
+
const failures: string[] = [];
|
|
1074
|
+
if (!identityMatches) failures.push('profile tunnel id does not match');
|
|
1075
|
+
if (!endpointMatches) failures.push('profile MCP endpoint does not match');
|
|
1076
|
+
if (!healthy) failures.push('local healthz failed');
|
|
1077
|
+
if (!ready) failures.push('local readyz failed');
|
|
1078
|
+
return {
|
|
1079
|
+
ok,
|
|
1080
|
+
running: healthy,
|
|
1081
|
+
healthy,
|
|
1082
|
+
ready,
|
|
1083
|
+
tunnelMatches: identityMatches,
|
|
1084
|
+
endpointMatches,
|
|
1085
|
+
alias: configured.alias,
|
|
1086
|
+
tunnelId: configured.tunnelId,
|
|
1087
|
+
...(identityMatches ? { observedTunnelId: configured.tunnelId } : {}),
|
|
1088
|
+
detail: ok
|
|
1089
|
+
? `managed runtime ${configured.alias} is locally healthy and ready for ${configured.tunnelId} (status command unavailable)`
|
|
1090
|
+
: failures.join('; ') || 'OpenAI tunnel local health fallback failed',
|
|
1091
|
+
profilePath,
|
|
1092
|
+
};
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1095
|
+
async function observeOpenAiRecoveryTunnel(
|
|
1096
|
+
config: RecoveryConfig,
|
|
1097
|
+
runCommand: CommandRunner = command,
|
|
1098
|
+
): Promise<OpenAiSecureTunnelRuntimeObservation | undefined> {
|
|
1099
|
+
const configured = configuredRecoveryTunnel(config);
|
|
1100
|
+
if (!configured || configured.platform !== 'openai-secure-tunnel') return undefined;
|
|
1101
|
+
return observeOpenAiTunnelRuntime(configured, runCommand);
|
|
1102
|
+
}
|
|
1103
|
+
|
|
1104
|
+
async function ensureOpenAiTunnelRuntimeStarted(
|
|
1105
|
+
configured: OpenAiSecureTunnelServiceConfig,
|
|
1106
|
+
runCommand: CommandRunner = command,
|
|
1107
|
+
): Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string; serviceLabel: string; serviceTarget: string }> {
|
|
1108
|
+
const serviceLabel = configured.alias;
|
|
1109
|
+
const serviceTarget = `tunnel-client:${configured.alias}`;
|
|
1110
|
+
const observed = await observeOpenAiTunnelRuntime(configured, runCommand);
|
|
1111
|
+
if (observed.ok) return { ok: true, attempted: false, noOp: true, detail: observed.detail, serviceLabel, serviceTarget };
|
|
1112
|
+
if (observed.observedTunnelId && !observed.tunnelMatches) {
|
|
1113
|
+
return { ok: false, attempted: false, noOp: true, detail: `OpenAI tunnel alias ${configured.alias} is already bound to a different tunnel id`, serviceLabel, serviceTarget };
|
|
1114
|
+
}
|
|
1115
|
+
if (observed.profilePath && !observed.endpointMatches) {
|
|
1116
|
+
return { ok: false, attempted: false, noOp: true, detail: `OpenAI tunnel alias ${configured.alias} is already bound to a different MCP endpoint`, serviceLabel, serviceTarget };
|
|
1117
|
+
}
|
|
1118
|
+
let args: string[];
|
|
1119
|
+
try {
|
|
1120
|
+
args = openAiSecureTunnelConnectArgs({
|
|
1121
|
+
alias: configured.alias,
|
|
1122
|
+
tunnelId: configured.tunnelId,
|
|
1123
|
+
mcpServerUrl: configured.mcpServerUrl,
|
|
1124
|
+
runtimeApiKeyRef: configured.runtimeApiKeyRef,
|
|
1125
|
+
profile: configured.profile,
|
|
1126
|
+
profileDir: configured.profileDir,
|
|
1127
|
+
adminProfile: configured.adminProfile,
|
|
1128
|
+
});
|
|
1129
|
+
} catch (error) {
|
|
1130
|
+
return { ok: false, attempted: false, noOp: true, detail: error instanceof Error ? error.message : 'OpenAI tunnel runtime configuration is invalid', serviceLabel, serviceTarget };
|
|
1131
|
+
}
|
|
1132
|
+
const connected = await runCommand('tunnel-client', args, Math.max(30_000, configured.postRestartVerifyTimeoutMs ?? 20_000));
|
|
1133
|
+
return connected.ok
|
|
1134
|
+
? { ok: true, attempted: true, detail: `OpenAI Secure MCP Tunnel runtime ${configured.alias} connect dispatched`, serviceLabel, serviceTarget }
|
|
1135
|
+
: { ok: false, attempted: true, detail: `OpenAI Secure MCP Tunnel runtime connect failed: ${connected.stderr || connected.stdout || connected.status}`, serviceLabel, serviceTarget };
|
|
1136
|
+
}
|
|
1137
|
+
|
|
1138
|
+
async function probeOpenAiRecoveryTunnel(
|
|
1139
|
+
config: RecoveryConfig,
|
|
1140
|
+
runCommand: CommandRunner = command,
|
|
1141
|
+
): Promise<{ ok: boolean; detail: string; value?: unknown } | undefined> {
|
|
1142
|
+
const observed = await observeOpenAiRecoveryTunnel(config, runCommand);
|
|
1143
|
+
return observed ? { ok: observed.ok, detail: observed.detail, value: observed } : undefined;
|
|
1144
|
+
}
|
|
1145
|
+
|
|
842
1146
|
async function probeMcp(config: RecoveryConfig, transport: RecoveryHttpTransport): Promise<Record<string, { ok: boolean; detail: string; value?: unknown }>> {
|
|
843
1147
|
const url = config.publicMcpUrl ?? runtimeEndpoint(config);
|
|
844
1148
|
if (!url) return { mcp_initialize: { ok: false, detail: 'canonical Runtime MCP endpoint is unavailable' } };
|
|
@@ -890,6 +1194,24 @@ async function probeMcp(config: RecoveryConfig, transport: RecoveryHttpTransport
|
|
|
890
1194
|
};
|
|
891
1195
|
}
|
|
892
1196
|
|
|
1197
|
+
function probeRuntimeExecutionSurface(
|
|
1198
|
+
config: RecoveryConfig,
|
|
1199
|
+
release: ReleaseEvidence | undefined,
|
|
1200
|
+
dependencies: RuntimeReleaseExecutionCanaryDependencies = {},
|
|
1201
|
+
): VerifyResult['probes'][string] {
|
|
1202
|
+
if (!release) return { ok: false, detail: 'active immutable Runtime release is unavailable' };
|
|
1203
|
+
try {
|
|
1204
|
+
const surface = assertRuntimeReleaseExecutionCanaries(release.path, config.controllerHome, dependencies);
|
|
1205
|
+
return {
|
|
1206
|
+
ok: true,
|
|
1207
|
+
detail: 'Process Runtime release entries are manifest-attested, executable, and canary-ready',
|
|
1208
|
+
value: { entries: surface.entries.map((entry) => ({ name: entry.name, artifactIdentity: entry.artifactIdentity })) },
|
|
1209
|
+
};
|
|
1210
|
+
} catch (error) {
|
|
1211
|
+
return { ok: false, detail: error instanceof Error ? error.message : String(error) };
|
|
1212
|
+
}
|
|
1213
|
+
}
|
|
1214
|
+
|
|
893
1215
|
export async function verifyStableRuntime(
|
|
894
1216
|
config: RecoveryConfig,
|
|
895
1217
|
transport = createRecoveryHttpTransport(config.controllerHome),
|
|
@@ -912,10 +1234,18 @@ export async function verifyStableRuntime(
|
|
|
912
1234
|
probes.active_gateway = endpoint
|
|
913
1235
|
? await probe(transport, runtimeHealthEndpoint(endpoint))
|
|
914
1236
|
: { ok: false, detail: 'canonical Runtime endpoint is unavailable' };
|
|
1237
|
+
probes.runtime_execution_surface = probeRuntimeExecutionSurface(config, active, options);
|
|
915
1238
|
if (config.publicMcpUrl) {
|
|
916
1239
|
probes.external_mcp_http = await probeExternalMcp(transport, config.publicMcpUrl);
|
|
917
1240
|
const connectorReadinessEndpoint = config.primaryConnectorService?.localMcpUrl?.trim() || config.publicMcpUrl;
|
|
918
|
-
probes.primary_connector_ready = await probePublicGatewayReadiness(transport, connectorReadinessEndpoint
|
|
1241
|
+
probes.primary_connector_ready = await probePublicGatewayReadiness(transport, connectorReadinessEndpoint, {
|
|
1242
|
+
acceptOAuthChallenge: packageConnectorAuthMode(config.controllerHome) === 'oauth',
|
|
1243
|
+
});
|
|
1244
|
+
}
|
|
1245
|
+
const primaryTunnel = configuredPrimaryPublicTunnel(config);
|
|
1246
|
+
if (primaryTunnel?.platform === 'openai-secure-tunnel') {
|
|
1247
|
+
const observed = await observeOpenAiTunnelRuntime(primaryTunnel);
|
|
1248
|
+
probes.primary_tunnel_runtime = { ok: observed.ok, detail: observed.detail, value: observed };
|
|
919
1249
|
}
|
|
920
1250
|
const primaryConnectorLocal = await probePrimaryConnectorLocal(config, transport);
|
|
921
1251
|
if (primaryConnectorLocal) probes.primary_connector_local = primaryConnectorLocal;
|
|
@@ -934,6 +1264,8 @@ export async function verifyStableRuntime(
|
|
|
934
1264
|
};
|
|
935
1265
|
const recoveryPublicUrl = configuredRecoveryPublicUrl(config);
|
|
936
1266
|
if (recoveryPublicUrl) probes.recovery_external_http = await probeExternalMcp(transport, recoveryPublicUrl);
|
|
1267
|
+
const recoveryTunnelRuntime = await probeOpenAiRecoveryTunnel(config);
|
|
1268
|
+
if (recoveryTunnelRuntime) probes.recovery_tunnel_runtime = recoveryTunnelRuntime;
|
|
937
1269
|
if (options.probeMcpProtocol !== false) Object.assign(probes, await probeMcp(config, transport));
|
|
938
1270
|
const coreChecks = Object.entries(probes)
|
|
939
1271
|
.filter(([name]) => !name.startsWith('recovery_'))
|
|
@@ -978,7 +1310,7 @@ export async function verifyStableRuntime(
|
|
|
978
1310
|
return result;
|
|
979
1311
|
}
|
|
980
1312
|
/** Explicitly records evidence only after the full independent verification passed. */
|
|
981
|
-
function persistVerifiedKnownGood(config: RecoveryConfig, verified: VerifyResult): ReleaseEvidence {
|
|
1313
|
+
function persistVerifiedKnownGood(config: RecoveryConfig, verified: VerifyResult, performance: RuntimePerformanceEvidence): ReleaseEvidence {
|
|
982
1314
|
const authority = releaseAuthority(config);
|
|
983
1315
|
const active = verified.releases.active;
|
|
984
1316
|
if (
|
|
@@ -992,8 +1324,10 @@ function persistVerifiedKnownGood(config: RecoveryConfig, verified: VerifyResult
|
|
|
992
1324
|
) {
|
|
993
1325
|
throw new Error('RECOVERY_KNOWN_GOOD_ATTESTATION_REQUIRES_FULL_VERIFY_AND_RELEASE_AUTHORITY');
|
|
994
1326
|
}
|
|
1327
|
+
assertRuntimePerformanceEvidence(performance, runtimePerformanceIdentity(config));
|
|
995
1328
|
const attested: ReleaseEvidence = {
|
|
996
1329
|
...active,
|
|
1330
|
+
performance,
|
|
997
1331
|
controllerHome: resolve(config.controllerHome),
|
|
998
1332
|
releaseAuthorityRevision: authority.revision,
|
|
999
1333
|
releaseFencingTokenSha256: createHash('sha256').update(authority.fencingToken).digest('hex'),
|
|
@@ -1014,10 +1348,51 @@ function persistVerifiedKnownGood(config: RecoveryConfig, verified: VerifyResult
|
|
|
1014
1348
|
return attested;
|
|
1015
1349
|
}
|
|
1016
1350
|
|
|
1017
|
-
|
|
1018
|
-
const
|
|
1019
|
-
|
|
1020
|
-
|
|
1351
|
+
function runtimePerformanceIdentity(config: RecoveryConfig): RuntimePerformanceIdentity {
|
|
1352
|
+
const authority = releaseAuthority(config);
|
|
1353
|
+
const status = observeRuntimeStatus(config.controllerHome);
|
|
1354
|
+
const snapshot = status.snapshot;
|
|
1355
|
+
if (!authority || !status.running || !status.ready || status.stale || !snapshot
|
|
1356
|
+
|| snapshot.releaseId !== authority.active.releaseId
|
|
1357
|
+
|| snapshot.artifactIdentity !== authority.active.artifactIdentity) {
|
|
1358
|
+
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: coherent live Runtime required');
|
|
1359
|
+
}
|
|
1360
|
+
return {
|
|
1361
|
+
releaseId: authority.active.releaseId, authorityRevision: authority.revision,
|
|
1362
|
+
pid: snapshot.pid, runtimeInstanceId: snapshot.runtimeInstanceId, startedAt: snapshot.startedAt,
|
|
1363
|
+
};
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
export async function attestKnownGood(
|
|
1367
|
+
config: RecoveryConfig,
|
|
1368
|
+
dependencies: RuntimePerformanceDependencies = {},
|
|
1369
|
+
): Promise<ReleaseEvidence> {
|
|
1370
|
+
const initialIdentity = runtimePerformanceIdentity(config);
|
|
1371
|
+
const before = await verifyStableRuntime(config);
|
|
1372
|
+
if (!before.ok) throw new Error('RECOVERY_KNOWN_GOOD_ATTESTATION_REQUIRES_FULL_VERIFY_AND_RELEASE_AUTHORITY');
|
|
1373
|
+
const verifiedIdentity = runtimePerformanceIdentity(config);
|
|
1374
|
+
if (!samePerformanceIdentity(initialIdentity, verifiedIdentity)
|
|
1375
|
+
|| before.releases.active?.revision !== verifiedIdentity.releaseId) {
|
|
1376
|
+
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: Runtime or release authority changed during functional verification');
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
// Performance observation is read-only and deliberately stays outside the
|
|
1380
|
+
// single Recovery mutation lock. Runtime/release identity is fenced on every
|
|
1381
|
+
// sample; a concurrent restart/rollback/activation therefore invalidates the
|
|
1382
|
+
// evidence instead of being blocked for the six-minute observation window.
|
|
1383
|
+
const performance = await measureRuntimePerformance(() => {
|
|
1384
|
+
const current = runtimePerformanceIdentity(config);
|
|
1385
|
+
if (!samePerformanceIdentity(verifiedIdentity, current)) {
|
|
1386
|
+
throw new Error('RECOVERY_PERFORMANCE_UNKNOWN: Runtime or release authority changed after functional verification');
|
|
1387
|
+
}
|
|
1388
|
+
return current;
|
|
1389
|
+
}, dependencies);
|
|
1390
|
+
|
|
1391
|
+
const locked = await withLock(config, { action: 'attest_known_good' }, async () => {
|
|
1392
|
+
const verified = await verifyStableRuntime(config);
|
|
1393
|
+
if (!verified.ok) throw new Error('RECOVERY_KNOWN_GOOD_ATTESTATION_REQUIRES_FULL_VERIFY_AND_RELEASE_AUTHORITY');
|
|
1394
|
+
return persistVerifiedKnownGood(config, verified, performance);
|
|
1395
|
+
});
|
|
1021
1396
|
if (!locked.acquired) throw new Error(recoveryBusyDetail(locked.owner));
|
|
1022
1397
|
return locked.value;
|
|
1023
1398
|
}
|
|
@@ -1032,9 +1407,6 @@ function quarantine(config: RecoveryConfig, release: ReleaseEvidence | undefined
|
|
|
1032
1407
|
async function rollbackPreviousLocked(config: RecoveryConfig, reason: string): Promise<RollbackResult> {
|
|
1033
1408
|
const before = await verifyStableRuntime(config);
|
|
1034
1409
|
const active = before.releases.active;
|
|
1035
|
-
if (before.ok && matchingKnownGood(config, active)) {
|
|
1036
|
-
return { ok: true, noOp: true, detail: 'active whole-Runtime release is currently healthy and independently attested known-good', verify: before };
|
|
1037
|
-
}
|
|
1038
1410
|
if (before.runtime.running) {
|
|
1039
1411
|
return {
|
|
1040
1412
|
ok: false,
|
|
@@ -1055,6 +1427,19 @@ async function rollbackPreviousLocked(config: RecoveryConfig, reason: string): P
|
|
|
1055
1427
|
verify: before,
|
|
1056
1428
|
};
|
|
1057
1429
|
}
|
|
1430
|
+
const targetExecution = probeRuntimeExecutionSurface(config, target);
|
|
1431
|
+
if (!targetExecution.ok) {
|
|
1432
|
+
audit(config, 'rollback_refused', {
|
|
1433
|
+
reason: 'attested previous release failed Process Runtime execution canary',
|
|
1434
|
+
targetRevision: target.revision,
|
|
1435
|
+
detail: targetExecution.detail,
|
|
1436
|
+
});
|
|
1437
|
+
return {
|
|
1438
|
+
ok: false,
|
|
1439
|
+
detail: `rollback refused: previous whole-Runtime release failed Process Runtime execution verification: ${targetExecution.detail}`,
|
|
1440
|
+
verify: before,
|
|
1441
|
+
};
|
|
1442
|
+
}
|
|
1058
1443
|
const operationId = `recovery-rollback-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
|
1059
1444
|
try {
|
|
1060
1445
|
const committed = rollbackRuntimeRelease(config.controllerHome, operationId);
|
|
@@ -1094,10 +1479,14 @@ export async function reconnectMain(config: RecoveryConfig): Promise<{ ok: boole
|
|
|
1094
1479
|
// intentionally a bounded health/reconnect observation, never a rollout.
|
|
1095
1480
|
const verified = await verifyStableRuntime(config);
|
|
1096
1481
|
const publicProbe = verified.probes.external_mcp_http;
|
|
1482
|
+
const primaryTunnel = configuredPrimaryPublicTunnel(config);
|
|
1483
|
+
const remoteTransportOk = primaryTunnel?.platform === 'openai-secure-tunnel'
|
|
1484
|
+
? verified.probes.primary_tunnel_runtime?.ok === true
|
|
1485
|
+
: (publicProbe?.ok === true || publicProbe?.status === 401);
|
|
1097
1486
|
const ok = verified.probes.active_gateway?.ok === true
|
|
1098
1487
|
&& verified.probes.primary_connector_ready?.ok !== false
|
|
1099
|
-
&&
|
|
1100
|
-
audit(config, 'reconnect_main', { ok, externalStatus: publicProbe?.status });
|
|
1488
|
+
&& remoteTransportOk;
|
|
1489
|
+
audit(config, 'reconnect_main', { ok, externalStatus: publicProbe?.status, primaryTunnelPlatform: primaryTunnel?.platform });
|
|
1101
1490
|
return { ok, detail: ok ? 'canonical Runtime Gateway and primary endpoint are reachable; client session may refresh' : 'primary endpoint remains unavailable; recovery channel remains independent', verify: verified };
|
|
1102
1491
|
}
|
|
1103
1492
|
|
|
@@ -1113,14 +1502,19 @@ async function verifyLocalRuntime(
|
|
|
1113
1502
|
...config,
|
|
1114
1503
|
publicMcpUrl: undefined,
|
|
1115
1504
|
recoveryPublicUrl: undefined,
|
|
1505
|
+
recoveryTunnelService: undefined,
|
|
1506
|
+
primaryPublicTunnelService: undefined,
|
|
1116
1507
|
readOnlyTool: { name: STABLE_RECOVERY_READ_ONLY_TOOL.name, arguments: { ...STABLE_RECOVERY_READ_ONLY_TOOL.arguments } },
|
|
1117
1508
|
}, createRecoveryHttpTransport(config.controllerHome), options);
|
|
1118
1509
|
}
|
|
1119
1510
|
|
|
1120
1511
|
function isExternalTunnelFailure(config: RecoveryConfig, verified: VerifyResult, localVerify: VerifyResult): boolean {
|
|
1121
|
-
const
|
|
1512
|
+
const configured = configuredRecoveryTunnel(config);
|
|
1122
1513
|
const localRecoveryGatewayHealthy = localVerify.probes.recovery_gateway?.ok ?? true;
|
|
1123
|
-
|
|
1514
|
+
if (!configured || !localVerify.ok || !localRecoveryGatewayHealthy) return false;
|
|
1515
|
+
if (configured.platform === 'openai-secure-tunnel') return verified.probes.recovery_tunnel_runtime?.ok !== true;
|
|
1516
|
+
const external = verified.probes.recovery_external_http ?? verified.probes.external_mcp_http;
|
|
1517
|
+
return Boolean(configuredRecoveryPublicUrl(config) && external?.ok !== true);
|
|
1124
1518
|
}
|
|
1125
1519
|
|
|
1126
1520
|
export const WATCHDOG_RUNTIME_STARTUP_GRACE_MS = 60_000;
|
|
@@ -1347,6 +1741,23 @@ interface CommandResult { ok: boolean; status: number | null; stdout: string; st
|
|
|
1347
1741
|
type CommandRunner = (commandName: string, args: string[], timeoutMs?: number) => Promise<CommandResult>;
|
|
1348
1742
|
interface LaunchdService { uid: number; domain: string; target: string; label: string; plistPath: string; }
|
|
1349
1743
|
|
|
1744
|
+
/**
|
|
1745
|
+
* systemd --user does not necessarily inherit the interactive shell PATH.
|
|
1746
|
+
* Recovery owns tunnel repair, so its non-interactive command environment must
|
|
1747
|
+
* still find user-installed runtime tools without adding a second service
|
|
1748
|
+
* owner or accepting an executable path through RPC.
|
|
1749
|
+
*/
|
|
1750
|
+
export function recoveryCommandPath(
|
|
1751
|
+
inheritedPath = process.env.PATH ?? '',
|
|
1752
|
+
platform: NodeJS.Platform = process.platform,
|
|
1753
|
+
accountHome = homedir(),
|
|
1754
|
+
): string {
|
|
1755
|
+
if (platform === 'win32') return inheritedPath;
|
|
1756
|
+
const userBin = join(accountHome, '.local', 'bin');
|
|
1757
|
+
const entries = inheritedPath.split(delimiter).filter(Boolean);
|
|
1758
|
+
return entries.includes(userBin) ? inheritedPath : [userBin, ...entries].join(delimiter);
|
|
1759
|
+
}
|
|
1760
|
+
|
|
1350
1761
|
export interface PublicTunnelRepairDependencies {
|
|
1351
1762
|
platform?: NodeJS.Platform;
|
|
1352
1763
|
currentUid?: () => Promise<number | undefined>;
|
|
@@ -1359,12 +1770,19 @@ export interface PublicTunnelRepairDependencies {
|
|
|
1359
1770
|
|
|
1360
1771
|
function command(commandName: string, args: string[], timeoutMs = 10_000, options: { cwd?: string; maxOutputBytes?: number } = {}): Promise<CommandResult> {
|
|
1361
1772
|
return new Promise((resolveCommand) => {
|
|
1362
|
-
const child = spawn(commandName, args, {
|
|
1773
|
+
const child = spawn(commandName, args, {
|
|
1774
|
+
shell: false,
|
|
1775
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
1776
|
+
windowsHide: true,
|
|
1777
|
+
cwd: options.cwd,
|
|
1778
|
+
env: { ...process.env, PATH: recoveryCommandPath() },
|
|
1779
|
+
});
|
|
1363
1780
|
const stdout: Buffer[] = [];
|
|
1364
1781
|
const stderr: Buffer[] = [];
|
|
1365
1782
|
let size = 0;
|
|
1366
1783
|
const maxOutputBytes = options.maxOutputBytes ?? 256 * 1024;
|
|
1367
1784
|
let settled = false;
|
|
1785
|
+
let stopping = false;
|
|
1368
1786
|
let killTimer: ReturnType<typeof setTimeout> | undefined;
|
|
1369
1787
|
const finish = (result: CommandResult) => {
|
|
1370
1788
|
if (settled) return;
|
|
@@ -1377,19 +1795,28 @@ function command(commandName: string, args: string[], timeoutMs = 10_000, option
|
|
|
1377
1795
|
if (!child.pid || child.exitCode != null) return;
|
|
1378
1796
|
try { process.kill(child.pid, signalName); } catch { /* Process already exited. */ }
|
|
1379
1797
|
};
|
|
1380
|
-
const stop = () => {
|
|
1381
|
-
if (child.exitCode != null) return;
|
|
1798
|
+
const stop = (detail: string) => {
|
|
1799
|
+
if (child.exitCode != null || stopping) return;
|
|
1800
|
+
stopping = true;
|
|
1382
1801
|
signalChild('SIGTERM');
|
|
1383
|
-
killTimer = setTimeout(() =>
|
|
1802
|
+
killTimer = setTimeout(() => {
|
|
1803
|
+
signalChild('SIGKILL');
|
|
1804
|
+
finish({
|
|
1805
|
+
ok: false,
|
|
1806
|
+
status: null,
|
|
1807
|
+
stdout: Buffer.concat(stdout).toString('utf8'),
|
|
1808
|
+
stderr: [Buffer.concat(stderr).toString('utf8').trim(), detail].filter(Boolean).join('\n'),
|
|
1809
|
+
});
|
|
1810
|
+
}, 1_000);
|
|
1384
1811
|
};
|
|
1385
|
-
const timeout = setTimeout(stop, timeoutMs);
|
|
1812
|
+
const timeout = setTimeout(() => stop(`command timed out after ${timeoutMs}ms`), timeoutMs);
|
|
1386
1813
|
child.stdout.on('data', (chunk: Buffer) => {
|
|
1387
1814
|
size += chunk.length;
|
|
1388
|
-
if (size <= maxOutputBytes) stdout.push(Buffer.from(chunk)); else stop();
|
|
1815
|
+
if (size <= maxOutputBytes) stdout.push(Buffer.from(chunk)); else stop(`command output exceeded ${maxOutputBytes} bytes`);
|
|
1389
1816
|
});
|
|
1390
1817
|
child.stderr.on('data', (chunk: Buffer) => {
|
|
1391
1818
|
size += chunk.length;
|
|
1392
|
-
if (size <= maxOutputBytes) stderr.push(Buffer.from(chunk)); else stop();
|
|
1819
|
+
if (size <= maxOutputBytes) stderr.push(Buffer.from(chunk)); else stop(`command output exceeded ${maxOutputBytes} bytes`);
|
|
1393
1820
|
});
|
|
1394
1821
|
child.once('error', () => finish({ ok: false, status: null, stdout: '', stderr: 'command spawn failed' }));
|
|
1395
1822
|
child.once('close', (status) => finish({ ok: status === 0, status, stdout: Buffer.concat(stdout).toString('utf8'), stderr: Buffer.concat(stderr).toString('utf8') }));
|
|
@@ -1430,6 +1857,8 @@ export interface PrimaryRuntimeRecoveryDependencies {
|
|
|
1430
1857
|
currentUid?: () => Promise<number | undefined>;
|
|
1431
1858
|
runCommand?: CommandRunner;
|
|
1432
1859
|
verifyLocal?: (config: RecoveryConfig) => Promise<VerifyResult>;
|
|
1860
|
+
/** Rebind the independently supervised OAuth Connector after a whole-Runtime release switch. */
|
|
1861
|
+
repairPrimaryConnectorBinding?: (config: RecoveryConfig) => Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string }>;
|
|
1433
1862
|
runtimeRunning?: (config: RecoveryConfig) => boolean;
|
|
1434
1863
|
ensureRuntimeLaunchContract?: (controllerHome: string) => void;
|
|
1435
1864
|
now?: () => number;
|
|
@@ -1445,23 +1874,120 @@ export interface RuntimeReleaseActivationGuard {
|
|
|
1445
1874
|
expectedActiveReleaseId?: string | null;
|
|
1446
1875
|
}
|
|
1447
1876
|
|
|
1448
|
-
function configuredPrimaryRuntimeService(config: RecoveryConfig): PrimaryRuntimeServiceConfig {
|
|
1449
|
-
return config.primaryRuntimeService ??
|
|
1877
|
+
function configuredPrimaryRuntimeService(config: RecoveryConfig, platform: NodeJS.Platform = process.platform): PrimaryRuntimeServiceConfig {
|
|
1878
|
+
return config.primaryRuntimeService ?? defaultPrimaryRuntimeServiceConfig(platform);
|
|
1450
1879
|
}
|
|
1451
1880
|
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1881
|
+
interface PrimaryRuntimeServiceOwner {
|
|
1882
|
+
platform: 'launchd' | 'systemd-user';
|
|
1883
|
+
target: string;
|
|
1884
|
+
uid: number;
|
|
1885
|
+
launchd?: LaunchdService;
|
|
1886
|
+
unitName?: string;
|
|
1887
|
+
unitPath?: string;
|
|
1888
|
+
}
|
|
1889
|
+
|
|
1890
|
+
function primaryRuntimeServiceOwner(
|
|
1891
|
+
config: RecoveryConfig,
|
|
1892
|
+
platform: NodeJS.Platform,
|
|
1893
|
+
uid: number | undefined,
|
|
1894
|
+
): PrimaryRuntimeServiceOwner | undefined {
|
|
1895
|
+
if (uid === undefined) return undefined;
|
|
1896
|
+
const configured = configuredPrimaryRuntimeService(config, platform);
|
|
1455
1897
|
const paths = forgeRuntimeServicePaths(config.controllerHome);
|
|
1456
|
-
if (
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1898
|
+
if (platform === 'darwin' && configured.platform === 'launchd') {
|
|
1899
|
+
if (!existsSync(paths.installedPlistPath)) return undefined;
|
|
1900
|
+
const domain = `gui/${uid}`;
|
|
1901
|
+
const launchd: LaunchdService = {
|
|
1902
|
+
uid,
|
|
1903
|
+
domain,
|
|
1904
|
+
target: `${domain}/${paths.label}`,
|
|
1905
|
+
label: paths.label,
|
|
1906
|
+
plistPath: paths.installedPlistPath,
|
|
1907
|
+
};
|
|
1908
|
+
return { platform: 'launchd', target: launchd.target, uid, launchd };
|
|
1909
|
+
}
|
|
1910
|
+
if (platform === 'linux' && configured.platform === 'systemd-user') {
|
|
1911
|
+
const unitName = systemdUserUnitName(paths.label);
|
|
1912
|
+
const unitPath = systemdUserUnitPath(unitName);
|
|
1913
|
+
if (!existsSync(unitPath)) return undefined;
|
|
1914
|
+
return { platform: 'systemd-user', target: unitName, uid, unitName, unitPath };
|
|
1915
|
+
}
|
|
1916
|
+
return undefined;
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1919
|
+
async function ensurePrimaryRuntimeServiceStarted(
|
|
1920
|
+
owner: PrimaryRuntimeServiceOwner,
|
|
1921
|
+
runCommand: CommandRunner,
|
|
1922
|
+
mode: 'start' | 'restart' = 'start',
|
|
1923
|
+
): Promise<{ ok: boolean; detail: string }> {
|
|
1924
|
+
if (owner.platform === 'launchd') return ensureLaunchdServiceStarted(owner.launchd!, runCommand);
|
|
1925
|
+
const action = mode === 'restart' ? 'restart' : 'start';
|
|
1926
|
+
const result = await runCommand('systemctl', ['--user', action, owner.unitName!], 15_000);
|
|
1927
|
+
return result.ok
|
|
1928
|
+
? { ok: true, detail: owner.target }
|
|
1929
|
+
: { ok: false, detail: `systemd-user ${action} failed: ${result.stderr || result.stdout || result.status}` };
|
|
1930
|
+
}
|
|
1931
|
+
|
|
1932
|
+
async function stopPrimaryRuntimeServiceOwner(
|
|
1933
|
+
owner: PrimaryRuntimeServiceOwner,
|
|
1934
|
+
runCommand: CommandRunner,
|
|
1935
|
+
): Promise<{ ok: boolean; detail: string }> {
|
|
1936
|
+
if (owner.platform === 'launchd') {
|
|
1937
|
+
const stopped = await runCommand('launchctl', ['bootout', owner.launchd!.target], 15_000);
|
|
1938
|
+
const stoppedCleanly = stopped.ok || /not found|no such process|could not find service|service is not loaded/i.test(`${stopped.stderr}\n${stopped.stdout}`);
|
|
1939
|
+
return stoppedCleanly
|
|
1940
|
+
? { ok: true, detail: owner.target }
|
|
1941
|
+
: { ok: false, detail: `primary Runtime bootout failed: ${stopped.stderr || stopped.stdout || stopped.status}` };
|
|
1942
|
+
}
|
|
1943
|
+
const stopped = await runCommand('systemctl', ['--user', 'stop', owner.unitName!], 15_000);
|
|
1944
|
+
return stopped.ok
|
|
1945
|
+
? { ok: true, detail: owner.target }
|
|
1946
|
+
: { ok: false, detail: `primary Runtime systemd-user stop failed: ${stopped.stderr || stopped.stdout || stopped.status}` };
|
|
1947
|
+
}
|
|
1948
|
+
|
|
1949
|
+
async function waitForPrimaryRuntimeServiceStopped(input: {
|
|
1950
|
+
owner: PrimaryRuntimeServiceOwner;
|
|
1951
|
+
timeoutMs: number;
|
|
1952
|
+
now: () => number;
|
|
1953
|
+
wait: (ms: number) => Promise<void>;
|
|
1954
|
+
runCommand: CommandRunner;
|
|
1955
|
+
}): Promise<boolean> {
|
|
1956
|
+
if (input.owner.platform === 'launchd') {
|
|
1957
|
+
return waitForLaunchdServiceUnloaded({
|
|
1958
|
+
service: input.owner.launchd!,
|
|
1959
|
+
timeoutMs: input.timeoutMs,
|
|
1960
|
+
now: input.now,
|
|
1961
|
+
wait: input.wait,
|
|
1962
|
+
runCommand: input.runCommand,
|
|
1963
|
+
});
|
|
1964
|
+
}
|
|
1965
|
+
const observe = async () => {
|
|
1966
|
+
const result = await input.runCommand('systemctl', ['--user', 'show', '--property', 'ActiveState', '--value', input.owner.unitName!], 5_000);
|
|
1967
|
+
if (!result.ok) return false;
|
|
1968
|
+
return /^(inactive|failed)$/.test(result.stdout.trim());
|
|
1464
1969
|
};
|
|
1970
|
+
const deadline = input.now() + input.timeoutMs;
|
|
1971
|
+
while (input.now() < deadline) {
|
|
1972
|
+
if (await observe()) return true;
|
|
1973
|
+
await input.wait(250);
|
|
1974
|
+
}
|
|
1975
|
+
return observe();
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
function primaryRuntimeServiceContractMatches(config: RecoveryConfig, owner: PrimaryRuntimeServiceOwner): boolean {
|
|
1979
|
+
if (owner.platform === 'launchd') {
|
|
1980
|
+
return inspectForgeRuntimeLaunchAgentContract({
|
|
1981
|
+
controllerHome: config.controllerHome,
|
|
1982
|
+
inspectUserLaunchAgent: true,
|
|
1983
|
+
}).matches;
|
|
1984
|
+
}
|
|
1985
|
+
try {
|
|
1986
|
+
const unit = readFileSync(owner.unitPath!, 'utf8');
|
|
1987
|
+
return unit === renderPackageRuntimeSystemdUserService(config.controllerHome);
|
|
1988
|
+
} catch {
|
|
1989
|
+
return false;
|
|
1990
|
+
}
|
|
1465
1991
|
}
|
|
1466
1992
|
|
|
1467
1993
|
export interface PrimaryConnectorRecoveryDependencies {
|
|
@@ -1479,14 +2005,55 @@ export interface PrimaryConnectorRecoveryDependencies {
|
|
|
1479
2005
|
sleep?: (ms: number) => Promise<void>;
|
|
1480
2006
|
}
|
|
1481
2007
|
|
|
1482
|
-
|
|
2008
|
+
interface PrimaryConnectorServiceOwner {
|
|
2009
|
+
platform: 'launchd' | 'systemd-user';
|
|
2010
|
+
label: string;
|
|
2011
|
+
target: string;
|
|
2012
|
+
uid: number;
|
|
2013
|
+
launchd?: LaunchdService;
|
|
2014
|
+
unitName?: string;
|
|
2015
|
+
unitPath?: string;
|
|
2016
|
+
}
|
|
2017
|
+
|
|
2018
|
+
function primaryConnectorServiceOwner(
|
|
2019
|
+
config: RecoveryConfig,
|
|
2020
|
+
platform: NodeJS.Platform,
|
|
2021
|
+
uid: number | undefined,
|
|
2022
|
+
): PrimaryConnectorServiceOwner | undefined {
|
|
1483
2023
|
const configured = config.primaryConnectorService;
|
|
1484
|
-
if (!configured ||
|
|
1485
|
-
const
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
2024
|
+
if (!configured || uid === undefined) return undefined;
|
|
2025
|
+
const paths = packageConnectorServicePaths(config.controllerHome);
|
|
2026
|
+
if (configured.platform === 'launchd') {
|
|
2027
|
+
if (platform !== 'darwin' || !configured.label.trim()) return undefined;
|
|
2028
|
+
const label = configured.label.trim();
|
|
2029
|
+
const plistPath = resolve(configured.plistPath ?? join(homedir(), 'Library', 'LaunchAgents', `${label}.plist`));
|
|
2030
|
+
if (!existsSync(plistPath)) return undefined;
|
|
2031
|
+
const domain = `gui/${uid}`;
|
|
2032
|
+
const launchd = { uid, domain, target: `${domain}/${label}`, label, plistPath };
|
|
2033
|
+
return { platform: 'launchd', label, target: launchd.target, uid, launchd };
|
|
2034
|
+
}
|
|
2035
|
+
if (platform !== 'linux') return undefined;
|
|
2036
|
+
if (configured.label?.trim() && configured.label.trim() !== paths.label) return undefined;
|
|
2037
|
+
const unitName = systemdUserUnitName(paths.label);
|
|
2038
|
+
return {
|
|
2039
|
+
platform: 'systemd-user',
|
|
2040
|
+
label: paths.label,
|
|
2041
|
+
target: `systemd-user:${unitName}`,
|
|
2042
|
+
uid,
|
|
2043
|
+
unitName,
|
|
2044
|
+
unitPath: systemdUserUnitPath(paths.label),
|
|
2045
|
+
};
|
|
2046
|
+
}
|
|
2047
|
+
|
|
2048
|
+
async function ensurePrimaryConnectorServiceStarted(
|
|
2049
|
+
owner: PrimaryConnectorServiceOwner,
|
|
2050
|
+
runCommand: CommandRunner,
|
|
2051
|
+
): Promise<{ ok: boolean; detail: string }> {
|
|
2052
|
+
if (owner.platform === 'launchd') return ensureLaunchdServiceStarted(owner.launchd!, runCommand);
|
|
2053
|
+
const restarted = await runCommand('systemctl', ['--user', 'restart', owner.unitName!], 15_000);
|
|
2054
|
+
return restarted.ok
|
|
2055
|
+
? { ok: true, detail: owner.target }
|
|
2056
|
+
: { ok: false, detail: `systemd-user restart failed: ${restarted.stderr || restarted.stdout || restarted.status}` };
|
|
1490
2057
|
}
|
|
1491
2058
|
|
|
1492
2059
|
async function probePrimaryConnectorLocal(
|
|
@@ -1495,29 +2062,47 @@ async function probePrimaryConnectorLocal(
|
|
|
1495
2062
|
): Promise<{ ok: boolean; detail: string; status?: number } | undefined> {
|
|
1496
2063
|
const localMcpUrl = config.primaryConnectorService?.localMcpUrl?.trim();
|
|
1497
2064
|
if (!localMcpUrl) return undefined;
|
|
1498
|
-
return probeExternalMcp(transport, localMcpUrl
|
|
2065
|
+
return probeExternalMcp(transport, localMcpUrl, {
|
|
2066
|
+
acceptOAuthChallenge: packageConnectorAuthMode(config.controllerHome) === 'oauth',
|
|
2067
|
+
});
|
|
1499
2068
|
}
|
|
1500
2069
|
|
|
1501
2070
|
function activePackageConnectorReleaseBinding(config: RecoveryConfig): PackageConnectorReleaseBinding | undefined {
|
|
1502
2071
|
const configured = config.primaryConnectorService;
|
|
1503
|
-
if (!configured ||
|
|
2072
|
+
if (!configured || !configured.localMcpUrl?.trim()) return undefined;
|
|
1504
2073
|
const authority = readRuntimeReleaseAuthority(config.controllerHome);
|
|
1505
2074
|
if (!authority) return undefined;
|
|
1506
2075
|
const releaseRoot = dirname(resolve(authority.active.manifestPath));
|
|
1507
2076
|
const packageRoot = join(releaseRoot, 'package');
|
|
1508
2077
|
const paths = packageConnectorServicePaths(config.controllerHome);
|
|
1509
|
-
|
|
1510
|
-
|
|
2078
|
+
if (configured.platform === 'launchd') {
|
|
2079
|
+
const configuredPlist = resolve(configured.plistPath ?? paths.installedPlistPath);
|
|
2080
|
+
if (configured.label !== paths.label || configuredPlist !== resolve(paths.installedPlistPath)) return undefined;
|
|
2081
|
+
} else if (configured.label?.trim() && configured.label.trim() !== paths.label) {
|
|
2082
|
+
return undefined;
|
|
2083
|
+
}
|
|
1511
2084
|
if (!existsSync(join(packageRoot, 'src', 'cli', 'index.ts')) || !existsSync(join(packageRoot, 'src', 'runtime', 'shared', 'node-ts-loader.mjs'))) return undefined;
|
|
1512
2085
|
return { releaseId: authority.active.releaseId, releaseRoot, packageRoot };
|
|
1513
2086
|
}
|
|
1514
2087
|
|
|
1515
|
-
|
|
2088
|
+
export function resolveRecoveryPackageConnectorExecutable(
|
|
2089
|
+
execPath: string = process.execPath,
|
|
2090
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
2091
|
+
accountHome: string = homedir(),
|
|
2092
|
+
): string {
|
|
2093
|
+
return resolveBunExecutable(execPath, env, accountHome);
|
|
2094
|
+
}
|
|
2095
|
+
|
|
2096
|
+
async function repairPrimaryConnectorBinding(
|
|
2097
|
+
config: RecoveryConfig,
|
|
2098
|
+
platform: NodeJS.Platform = process.platform,
|
|
2099
|
+
): Promise<{ ok: boolean; attempted: boolean; noOp?: boolean; detail: string }> {
|
|
1516
2100
|
const release = activePackageConnectorReleaseBinding(config);
|
|
1517
2101
|
const endpoint = config.primaryConnectorService?.localMcpUrl?.trim();
|
|
1518
2102
|
if (!release || !endpoint) return { ok: true, attempted: false, noOp: true, detail: 'primary Connector does not use the canonical package-release binding contract' };
|
|
1519
2103
|
try {
|
|
1520
|
-
const
|
|
2104
|
+
const executable = resolveRecoveryPackageConnectorExecutable();
|
|
2105
|
+
const result = await ensurePackageConnectorService({ release, controllerHome: config.controllerHome, endpoint, executable, platform });
|
|
1521
2106
|
const attempted = result.reused !== true;
|
|
1522
2107
|
return {
|
|
1523
2108
|
ok: true,
|
|
@@ -1534,7 +2119,7 @@ async function repairPrimaryConnectorBinding(config: RecoveryConfig): Promise<{
|
|
|
1534
2119
|
|
|
1535
2120
|
async function probePrimaryConnectorOwnership(
|
|
1536
2121
|
config: RecoveryConfig,
|
|
1537
|
-
service:
|
|
2122
|
+
service: PrimaryConnectorServiceOwner,
|
|
1538
2123
|
runCommand: CommandRunner = command,
|
|
1539
2124
|
): Promise<{ ok: boolean; detail: string }> {
|
|
1540
2125
|
const localMcpUrl = config.primaryConnectorService?.localMcpUrl?.trim();
|
|
@@ -1548,11 +2133,18 @@ async function probePrimaryConnectorOwnership(
|
|
|
1548
2133
|
} catch {
|
|
1549
2134
|
return { ok: false, detail: `primary Connector local MCP endpoint has no verifiable TCP port: ${localMcpUrl}` };
|
|
1550
2135
|
}
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
2136
|
+
let pid: number | undefined;
|
|
2137
|
+
if (service.platform === 'launchd') {
|
|
2138
|
+
const printed = await runCommand('launchctl', ['print', service.target], 5_000);
|
|
2139
|
+
const pidMatch = printed.ok ? printed.stdout.match(/\bpid\s*=\s*(\d+)/) : null;
|
|
2140
|
+
pid = pidMatch ? Number(pidMatch[1]) : undefined;
|
|
2141
|
+
} else {
|
|
2142
|
+
const shown = await runCommand('systemctl', ['--user', 'show', '--property', 'MainPID', '--value', service.unitName!], 5_000);
|
|
2143
|
+
const parsed = Number(shown.stdout.trim());
|
|
2144
|
+
pid = shown.ok && Number.isInteger(parsed) && parsed > 0 ? parsed : undefined;
|
|
2145
|
+
}
|
|
1554
2146
|
if (!pid || !Number.isInteger(pid)) {
|
|
1555
|
-
return { ok: false, detail: `configured primary Connector
|
|
2147
|
+
return { ok: false, detail: `configured primary Connector ${service.platform} service has no live pid: ${service.target}` };
|
|
1556
2148
|
}
|
|
1557
2149
|
const listening = await runCommand('lsof', ['-nP', '-a', '-p', String(pid), `-iTCP:${port}`, '-sTCP:LISTEN', '-t'], 5_000);
|
|
1558
2150
|
const listenerPids = listening.stdout.split(/\s+/).map((value) => Number(value)).filter(Number.isInteger);
|
|
@@ -1575,27 +2167,21 @@ export async function restartPrimaryConnector(
|
|
|
1575
2167
|
// That probe is expected to be false when this recovery action is needed,
|
|
1576
2168
|
// so it must not prevent a healthy canonical Runtime from repairing the
|
|
1577
2169
|
// Connector service.
|
|
1578
|
-
const canonicalRuntimeHealthy = initialLocal
|
|
1579
|
-
&& initialLocal.runtime.running
|
|
1580
|
-
&& initialLocal.runtime.ready
|
|
1581
|
-
&& !initialLocal.runtime.stale
|
|
1582
|
-
&& initialLocal.probes.active_gateway?.ok === true;
|
|
2170
|
+
const canonicalRuntimeHealthy = canonicalRuntimeSafeForTargetedConnectorRecovery(initialLocal);
|
|
1583
2171
|
if (!canonicalRuntimeHealthy) {
|
|
1584
2172
|
return {
|
|
1585
2173
|
ok: false,
|
|
1586
2174
|
attempted: false,
|
|
1587
2175
|
noOp: true,
|
|
1588
|
-
detail: 'Canonical Runtime must be
|
|
2176
|
+
detail: 'Canonical Runtime ownership/release execution authority must be healthy before the primary Connector is restarted',
|
|
1589
2177
|
verify: initialLocal,
|
|
1590
2178
|
};
|
|
1591
2179
|
}
|
|
1592
|
-
|
|
1593
|
-
return { ok: false, attempted: false, noOp: true, detail: 'primary Connector restart currently requires the configured macOS launchd service', verify: initialLocal };
|
|
1594
|
-
}
|
|
2180
|
+
const platform = dependencies.platform ?? process.platform;
|
|
1595
2181
|
const uid = await (dependencies.currentUid ?? currentUid)();
|
|
1596
|
-
const service =
|
|
2182
|
+
const service = primaryConnectorServiceOwner(config, platform, uid);
|
|
1597
2183
|
if (!service) {
|
|
1598
|
-
return { ok: false, attempted: false, noOp: true, detail:
|
|
2184
|
+
return { ok: false, attempted: false, noOp: true, detail: `primary Connector ${config.primaryConnectorService?.platform ?? 'service'} is not configured for ${platform}`, verify: initialLocal };
|
|
1599
2185
|
}
|
|
1600
2186
|
const reconnect = dependencies.reconnect ?? reconnectMain;
|
|
1601
2187
|
const probeConnectorLocal = dependencies.probeConnectorLocal ?? probePrimaryConnectorLocal;
|
|
@@ -1607,7 +2193,7 @@ export async function restartPrimaryConnector(
|
|
|
1607
2193
|
const timeoutMs = config.primaryConnectorService?.postRestartVerifyTimeoutMs ?? 30_000;
|
|
1608
2194
|
const locked = await withLock(config, { action: 'restart_primary_connector', requestId: dependencies.requestId }, async () => {
|
|
1609
2195
|
const tunnelConfigured = Boolean(configuredPrimaryPublicTunnel(config));
|
|
1610
|
-
const repairConnectorBinding = dependencies.repairConnectorBinding ?? repairPrimaryConnectorBinding;
|
|
2196
|
+
const repairConnectorBinding = dependencies.repairConnectorBinding ?? ((candidateConfig: RecoveryConfig) => repairPrimaryConnectorBinding(candidateConfig, platform));
|
|
1611
2197
|
const bindingRepair = await repairConnectorBinding(config);
|
|
1612
2198
|
if (bindingRepair.attempted) {
|
|
1613
2199
|
audit(config, bindingRepair.ok ? 'primary_connector_binding_repaired' : 'primary_connector_binding_repair_failed', { detail: bindingRepair.detail });
|
|
@@ -1636,7 +2222,7 @@ export async function restartPrimaryConnector(
|
|
|
1636
2222
|
// restart and repair the public hop directly.
|
|
1637
2223
|
const restartConnector = !localConnector?.ok || !connectorOwnership?.ok || !tunnelConfigured;
|
|
1638
2224
|
if (restartConnector) {
|
|
1639
|
-
const restarted = await
|
|
2225
|
+
const restarted = await ensurePrimaryConnectorServiceStarted(service, runCommand);
|
|
1640
2226
|
if (!restarted.ok) {
|
|
1641
2227
|
audit(config, 'primary_connector_restart_failed', { serviceTarget: service.target, detail: restarted.detail });
|
|
1642
2228
|
return { ok: false, attempted: true, detail: restarted.detail, serviceTarget: service.target, verify: initialLocal } satisfies PrimaryConnectorRestartResult;
|
|
@@ -1658,7 +2244,7 @@ export async function restartPrimaryConnector(
|
|
|
1658
2244
|
ok: false,
|
|
1659
2245
|
attempted: true,
|
|
1660
2246
|
detail: localConnector.ok
|
|
1661
|
-
?
|
|
2247
|
+
? `primary Connector restarted but the configured ${service.platform} service does not own its local MCP listener`
|
|
1662
2248
|
: 'primary Connector restarted but its local MCP endpoint did not recover before timeout',
|
|
1663
2249
|
serviceTarget: service.target,
|
|
1664
2250
|
verify: initialLocal,
|
|
@@ -1667,30 +2253,38 @@ export async function restartPrimaryConnector(
|
|
|
1667
2253
|
}
|
|
1668
2254
|
|
|
1669
2255
|
observed ??= await reconnect(config);
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
attempted: true,
|
|
1679
|
-
detail: `primary Connector is locally healthy but public tunnel restart failed: ${tunnelRestarted.detail}`,
|
|
1680
|
-
serviceTarget: service.target,
|
|
1681
|
-
verify: observed.verify,
|
|
1682
|
-
} satisfies PrimaryConnectorRestartResult;
|
|
2256
|
+
const primaryTunnel = configuredPrimaryPublicTunnel(config);
|
|
2257
|
+
if (!observed.ok && localConnector?.ok && connectorOwnership?.ok && primaryTunnel) {
|
|
2258
|
+
let tunnelRestarted: { ok: boolean; attempted: boolean; detail: string; serviceTarget: string } | undefined;
|
|
2259
|
+
if (primaryTunnel.platform === 'launchd') {
|
|
2260
|
+
const tunnel = primaryPublicTunnelService(config, service.uid);
|
|
2261
|
+
if (tunnel) {
|
|
2262
|
+
const result = await ensureLaunchdServiceStarted(tunnel, runCommand);
|
|
2263
|
+
tunnelRestarted = { ok: result.ok, attempted: true, detail: result.detail, serviceTarget: tunnel.target };
|
|
1683
2264
|
}
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
audit(config,
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
2265
|
+
} else {
|
|
2266
|
+
tunnelRestarted = await ensureOpenAiTunnelRuntimeStarted(primaryTunnel, runCommand);
|
|
2267
|
+
}
|
|
2268
|
+
if (!tunnelRestarted?.ok) {
|
|
2269
|
+
const detail = tunnelRestarted?.detail ?? 'primary public tunnel service configuration is invalid or unavailable';
|
|
2270
|
+
audit(config, 'primary_public_tunnel_restart_failed', { serviceTarget: tunnelRestarted?.serviceTarget, detail });
|
|
2271
|
+
return {
|
|
2272
|
+
ok: false,
|
|
2273
|
+
attempted: tunnelRestarted?.attempted ?? false,
|
|
2274
|
+
detail: `primary Connector is locally healthy but public tunnel restart failed: ${detail}`,
|
|
2275
|
+
serviceTarget: service.target,
|
|
2276
|
+
verify: observed.verify,
|
|
2277
|
+
} satisfies PrimaryConnectorRestartResult;
|
|
1693
2278
|
}
|
|
2279
|
+
const tunnelDeadline = now() + (primaryTunnel.postRestartVerifyTimeoutMs ?? 20_000);
|
|
2280
|
+
while (!observed.ok && now() < tunnelDeadline) {
|
|
2281
|
+
await wait(1_000);
|
|
2282
|
+
observed = await reconnect(config);
|
|
2283
|
+
}
|
|
2284
|
+
audit(config, observed.ok ? 'primary_public_tunnel_restart_succeeded' : 'primary_public_tunnel_restart_unverified', {
|
|
2285
|
+
serviceTarget: tunnelRestarted.serviceTarget,
|
|
2286
|
+
detail: observed.detail,
|
|
2287
|
+
});
|
|
1694
2288
|
} else {
|
|
1695
2289
|
const deadline = now() + timeoutMs;
|
|
1696
2290
|
while (!observed.ok && now() < deadline) {
|
|
@@ -1909,17 +2503,14 @@ interface PrimaryRuntimeStoppedTransition {
|
|
|
1909
2503
|
*/
|
|
1910
2504
|
async function stopPrimaryRuntimeForReleaseTransition(input: {
|
|
1911
2505
|
config: RecoveryConfig;
|
|
1912
|
-
service:
|
|
2506
|
+
service: PrimaryRuntimeServiceOwner;
|
|
1913
2507
|
now: () => number;
|
|
1914
2508
|
wait: (ms: number) => Promise<void>;
|
|
1915
2509
|
runCommand: CommandRunner;
|
|
1916
2510
|
runtimeRunning: (config: RecoveryConfig) => boolean;
|
|
1917
2511
|
}): Promise<PrimaryRuntimeStoppedTransition> {
|
|
1918
|
-
const stopped = await input.
|
|
1919
|
-
|
|
1920
|
-
if (!stoppedCleanly) {
|
|
1921
|
-
return { ok: false, detail: `primary Runtime bootout failed: ${stopped.stderr || stopped.stdout || stopped.status}` };
|
|
1922
|
-
}
|
|
2512
|
+
const stopped = await stopPrimaryRuntimeServiceOwner(input.service, input.runCommand);
|
|
2513
|
+
if (!stopped.ok) return { ok: false, detail: stopped.detail };
|
|
1923
2514
|
const runtimeStopped = await waitForPrimaryRuntimeState({
|
|
1924
2515
|
config: input.config,
|
|
1925
2516
|
expectedRunning: false,
|
|
@@ -1928,15 +2519,24 @@ async function stopPrimaryRuntimeForReleaseTransition(input: {
|
|
|
1928
2519
|
wait: input.wait,
|
|
1929
2520
|
runtimeRunning: input.runtimeRunning,
|
|
1930
2521
|
});
|
|
1931
|
-
if (!runtimeStopped)
|
|
1932
|
-
|
|
1933
|
-
|
|
2522
|
+
if (!runtimeStopped) {
|
|
2523
|
+
return { ok: false, detail: `primary Runtime owner remained live after bounded ${input.service.platform} stop` };
|
|
2524
|
+
}
|
|
2525
|
+
const serviceStopped = await waitForPrimaryRuntimeServiceStopped({
|
|
2526
|
+
owner: input.service,
|
|
1934
2527
|
timeoutMs: 20_000,
|
|
1935
2528
|
now: input.now,
|
|
1936
2529
|
wait: input.wait,
|
|
1937
2530
|
runCommand: input.runCommand,
|
|
1938
2531
|
});
|
|
1939
|
-
if (!
|
|
2532
|
+
if (!serviceStopped) {
|
|
2533
|
+
return {
|
|
2534
|
+
ok: false,
|
|
2535
|
+
detail: input.service.platform === 'launchd'
|
|
2536
|
+
? 'primary Runtime launchd service remained loaded after bounded bootout'
|
|
2537
|
+
: 'primary Runtime systemd-user service remained active after bounded stop',
|
|
2538
|
+
};
|
|
2539
|
+
}
|
|
1940
2540
|
|
|
1941
2541
|
let portReleased = await waitForPrimaryRuntimePortReleased({
|
|
1942
2542
|
config: input.config,
|
|
@@ -1959,11 +2559,11 @@ async function stopPrimaryRuntimeForReleaseTransition(input: {
|
|
|
1959
2559
|
if (!portReleased) {
|
|
1960
2560
|
return {
|
|
1961
2561
|
ok: false,
|
|
1962
|
-
detail: `primary Runtime TCP port remained occupied after bounded
|
|
2562
|
+
detail: `primary Runtime TCP port remained occupied after bounded ${input.service.platform} stop: ${staleListenerCleanup?.detail ?? 'listener did not release'}`,
|
|
1963
2563
|
staleListenerCleanup,
|
|
1964
2564
|
};
|
|
1965
2565
|
}
|
|
1966
|
-
return { ok: true, detail:
|
|
2566
|
+
return { ok: true, detail: `primary Runtime ${input.service.platform} service, owner, and TCP listener are stopped`, staleListenerCleanup };
|
|
1967
2567
|
}
|
|
1968
2568
|
|
|
1969
2569
|
async function verifyPrimaryRuntimeAfterStart(input: {
|
|
@@ -1996,7 +2596,7 @@ interface PrimaryRuntimeRebindStartResult {
|
|
|
1996
2596
|
*/
|
|
1997
2597
|
async function rebindStartAndVerifyPrimaryRuntime(input: {
|
|
1998
2598
|
config: RecoveryConfig;
|
|
1999
|
-
service:
|
|
2599
|
+
service: PrimaryRuntimeServiceOwner;
|
|
2000
2600
|
runCommand: CommandRunner;
|
|
2001
2601
|
now: () => number;
|
|
2002
2602
|
wait: (ms: number) => Promise<void>;
|
|
@@ -2008,13 +2608,28 @@ async function rebindStartAndVerifyPrimaryRuntime(input: {
|
|
|
2008
2608
|
successDetail: string;
|
|
2009
2609
|
}): Promise<PrimaryRuntimeRebindStartResult> {
|
|
2010
2610
|
try {
|
|
2011
|
-
|
|
2012
|
-
|
|
2013
|
-
|
|
2611
|
+
if (input.service.platform === 'launchd') {
|
|
2612
|
+
const ensureRuntimeLaunchContract = input.ensureRuntimeLaunchContract
|
|
2613
|
+
?? ((controllerHome: string) => { ensureForgeRuntimeLaunchAgentContract({ controllerHome, installUserLaunchAgent: true }); });
|
|
2614
|
+
ensureRuntimeLaunchContract(input.config.controllerHome);
|
|
2615
|
+
} else {
|
|
2616
|
+
const unitPath = writePackageRuntimeSystemdUserService(input.config.controllerHome);
|
|
2617
|
+
if (resolve(unitPath) !== resolve(input.service.unitPath!)) {
|
|
2618
|
+
throw new Error('canonical package Runtime systemd-user unit path changed during Recovery rebind');
|
|
2619
|
+
}
|
|
2620
|
+
for (const args of [
|
|
2621
|
+
['--user', 'daemon-reload'],
|
|
2622
|
+
['--user', 'enable', input.service.unitName!],
|
|
2623
|
+
]) {
|
|
2624
|
+
const result = await input.runCommand('systemctl', args, 15_000);
|
|
2625
|
+
if (!result.ok) throw new Error(`systemctl ${args.join(' ')} failed: ${result.stderr || result.stdout || result.status}`);
|
|
2626
|
+
}
|
|
2627
|
+
}
|
|
2014
2628
|
} catch (error) {
|
|
2629
|
+
const manager = input.service.platform === 'launchd' ? 'launchd' : 'systemd-user';
|
|
2015
2630
|
return {
|
|
2016
2631
|
ok: false,
|
|
2017
|
-
detail: `primary Runtime
|
|
2632
|
+
detail: `primary Runtime ${manager} contract rebuild failed ${input.contractFailureContext ?? 'after release transition'}: ${error instanceof Error ? error.message : String(error)}`,
|
|
2018
2633
|
verify: await input.verifyLocal(input.config),
|
|
2019
2634
|
};
|
|
2020
2635
|
}
|
|
@@ -2029,7 +2644,7 @@ async function rebindStartAndVerifyPrimaryRuntime(input: {
|
|
|
2029
2644
|
};
|
|
2030
2645
|
}
|
|
2031
2646
|
|
|
2032
|
-
const started = await
|
|
2647
|
+
const started = await ensurePrimaryRuntimeServiceStarted(input.service, input.runCommand, 'start');
|
|
2033
2648
|
if (!started.ok) {
|
|
2034
2649
|
return { ok: false, detail: started.detail, verify: await input.verifyLocal(input.config) };
|
|
2035
2650
|
}
|
|
@@ -2041,7 +2656,7 @@ async function rebindStartAndVerifyPrimaryRuntime(input: {
|
|
|
2041
2656
|
verifyLocal: input.verifyLocal,
|
|
2042
2657
|
});
|
|
2043
2658
|
if (verify.ok) return { ok: true, detail: input.successDetail, verify };
|
|
2044
|
-
await input.
|
|
2659
|
+
await stopPrimaryRuntimeServiceOwner(input.service, input.runCommand);
|
|
2045
2660
|
return {
|
|
2046
2661
|
ok: false,
|
|
2047
2662
|
detail: 'release transition started a Runtime that failed whole-Runtime verification; service was stopped to prevent a restart loop',
|
|
@@ -2056,18 +2671,18 @@ export async function restartPrimaryRuntime(
|
|
|
2056
2671
|
const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
|
|
2057
2672
|
const initial = await verifyLocal(config);
|
|
2058
2673
|
if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime is already healthy', verify: initial };
|
|
2059
|
-
|
|
2060
|
-
return { ok: false, attempted: false, noOp: true, detail: 'primary Runtime automatic restart currently requires the configured macOS launchd service', verify: initial };
|
|
2061
|
-
}
|
|
2674
|
+
const platform = dependencies.platform ?? process.platform;
|
|
2062
2675
|
const uid = await (dependencies.currentUid ?? currentUid)();
|
|
2063
|
-
const service =
|
|
2064
|
-
if (!service)
|
|
2676
|
+
const service = primaryRuntimeServiceOwner(config, platform, uid);
|
|
2677
|
+
if (!service) {
|
|
2678
|
+
return { ok: false, attempted: false, noOp: true, detail: `primary Forge Runtime ${configuredPrimaryRuntimeService(config, platform).platform} service is not installed for ${platform}`, verify: initial };
|
|
2679
|
+
}
|
|
2065
2680
|
const now = dependencies.now ?? Date.now;
|
|
2066
2681
|
const wait = dependencies.sleep ?? sleep;
|
|
2067
2682
|
const locked = await withLock(config, { action: 'restart_primary_runtime' }, async () => {
|
|
2068
2683
|
const before = await verifyLocal(config);
|
|
2069
2684
|
if (before.ok) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime recovered before restart', serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRestartResult;
|
|
2070
|
-
const started = await
|
|
2685
|
+
const started = await ensurePrimaryRuntimeServiceStarted(service, dependencies.runCommand ?? command, 'restart');
|
|
2071
2686
|
if (!started.ok) {
|
|
2072
2687
|
audit(config, 'primary_runtime_restart_failed', { serviceTarget: service.target, detail: started.detail });
|
|
2073
2688
|
return { ok: false, attempted: true, detail: started.detail, serviceTarget: service.target, verify: before } satisfies PrimaryRuntimeRestartResult;
|
|
@@ -2098,12 +2713,12 @@ export async function recoverPrimaryRuntime(
|
|
|
2098
2713
|
const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
|
|
2099
2714
|
const initial = await verifyLocal(config);
|
|
2100
2715
|
if (initial.ok) return { ok: true, attempted: false, noOp: true, detail: 'Canonical Forge Runtime recovered before rollback', verify: initial };
|
|
2101
|
-
|
|
2102
|
-
return { ok: false, attempted: false, noOp: true, detail: 'automatic whole-Runtime recovery currently requires the configured macOS launchd service', verify: initial };
|
|
2103
|
-
}
|
|
2716
|
+
const platform = dependencies.platform ?? process.platform;
|
|
2104
2717
|
const uid = await (dependencies.currentUid ?? currentUid)();
|
|
2105
|
-
const service =
|
|
2106
|
-
if (!service)
|
|
2718
|
+
const service = primaryRuntimeServiceOwner(config, platform, uid);
|
|
2719
|
+
if (!service) {
|
|
2720
|
+
return { ok: false, attempted: false, noOp: true, detail: `primary Forge Runtime ${configuredPrimaryRuntimeService(config, platform).platform} service is not installed for ${platform}`, verify: initial };
|
|
2721
|
+
}
|
|
2107
2722
|
const runCommand = dependencies.runCommand ?? command;
|
|
2108
2723
|
const now = dependencies.now ?? Date.now;
|
|
2109
2724
|
const wait = dependencies.sleep ?? sleep;
|
|
@@ -2151,8 +2766,8 @@ export async function recoverPrimaryRuntime(
|
|
|
2151
2766
|
audit(config, 'primary_runtime_recovery_succeeded', { serviceTarget: service.target, rollbackOperationId: rollback.operationId, restoredRelease: restarted.verify.releases.active?.revision });
|
|
2152
2767
|
return { ok: true, attempted: true, detail: restarted.detail, serviceTarget: service.target, rollback, verify: restarted.verify } satisfies PrimaryRuntimeRecoveryResult;
|
|
2153
2768
|
}
|
|
2154
|
-
const contractFailure =
|
|
2155
|
-
audit(config, contractFailure ? '
|
|
2769
|
+
const contractFailure = /^primary Runtime (launchd|systemd-user) contract rebuild failed/.test(restarted.detail);
|
|
2770
|
+
audit(config, contractFailure ? 'primary_runtime_recovery_service_contract_failed' : 'primary_runtime_recovery_unverified', {
|
|
2156
2771
|
serviceTarget: service.target,
|
|
2157
2772
|
rollbackOperationId: rollback.operationId,
|
|
2158
2773
|
detail: restarted.detail,
|
|
@@ -2208,6 +2823,7 @@ function validateRuntimeReleaseCandidate(
|
|
|
2208
2823
|
if (identity !== manifest.artifactIdentity) {
|
|
2209
2824
|
throw new Error(`RUNTIME_RELEASE_CANDIDATE_ARTIFACT_MISMATCH: expected ${manifest.artifactIdentity} observed ${identity}`);
|
|
2210
2825
|
}
|
|
2826
|
+
assertRuntimeReleaseExecutionCanaries(resolvedManifestPath, config.controllerHome);
|
|
2211
2827
|
return { manifest, releaseRoot, manifestPath: resolvedManifestPath };
|
|
2212
2828
|
}
|
|
2213
2829
|
|
|
@@ -2234,17 +2850,18 @@ export async function activateRuntimeRelease(
|
|
|
2234
2850
|
return { ok: false, attempted: false, noOp: true, detail };
|
|
2235
2851
|
}
|
|
2236
2852
|
const platform = dependencies.platform ?? process.platform;
|
|
2237
|
-
if (platform !== 'darwin') {
|
|
2238
|
-
return { ok: false, attempted: false, noOp: true, detail: 'Runtime release activation currently requires the configured macOS launchd service' };
|
|
2239
|
-
}
|
|
2240
2853
|
const uid = await (dependencies.currentUid ?? currentUid)();
|
|
2241
|
-
const service =
|
|
2242
|
-
if (!service)
|
|
2854
|
+
const service = primaryRuntimeServiceOwner(config, platform, uid);
|
|
2855
|
+
if (!service) {
|
|
2856
|
+
return { ok: false, attempted: false, noOp: true, detail: `primary Forge Runtime ${configuredPrimaryRuntimeService(config, platform).platform} service is not installed for ${platform}`, verify: await verifyStableRuntime(config) };
|
|
2857
|
+
}
|
|
2243
2858
|
const runCommand = dependencies.runCommand ?? command;
|
|
2244
2859
|
const now = dependencies.now ?? Date.now;
|
|
2245
2860
|
const wait = dependencies.sleep ?? sleep;
|
|
2246
2861
|
const runtimeRunning = dependencies.runtimeRunning ?? ((value: RecoveryConfig) => observeRuntimeStatus(value.controllerHome).running);
|
|
2247
2862
|
const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
|
|
2863
|
+
const repairConnectorBinding = dependencies.repairPrimaryConnectorBinding
|
|
2864
|
+
?? ((value: RecoveryConfig) => repairPrimaryConnectorBinding(value, platform));
|
|
2248
2865
|
const operationId = `recovery-activate-runtime-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
|
2249
2866
|
const lockRequestId = guard.requestId?.trim() || operationId;
|
|
2250
2867
|
const locked = await withLock(config, { action: 'activate_runtime_release', requestId: lockRequestId }, async () => {
|
|
@@ -2296,10 +2913,10 @@ export async function activateRuntimeRelease(
|
|
|
2296
2913
|
return { ok: false, attempted: false, noOp: true, detail, operationId } satisfies RuntimeReleaseActivationResult;
|
|
2297
2914
|
}
|
|
2298
2915
|
return {
|
|
2299
|
-
ok:
|
|
2916
|
+
ok: (await repairConnectorBinding(config)).ok,
|
|
2300
2917
|
attempted: false,
|
|
2301
2918
|
noOp: true,
|
|
2302
|
-
detail: 'requested Runtime release is already the active whole-Runtime release',
|
|
2919
|
+
detail: 'requested Runtime release is already the active whole-Runtime release; its persistent Connector binding was reconciled',
|
|
2303
2920
|
operationId,
|
|
2304
2921
|
verify: await verifyLocal(config),
|
|
2305
2922
|
} satisfies RuntimeReleaseActivationResult;
|
|
@@ -2309,10 +2926,7 @@ export async function activateRuntimeRelease(
|
|
|
2309
2926
|
let serviceContractMatches = false;
|
|
2310
2927
|
if (behaviorEquivalent && !storageMigrationRequired) {
|
|
2311
2928
|
try {
|
|
2312
|
-
serviceContractMatches =
|
|
2313
|
-
controllerHome: config.controllerHome,
|
|
2314
|
-
inspectUserLaunchAgent: true,
|
|
2315
|
-
}).matches;
|
|
2929
|
+
serviceContractMatches = primaryRuntimeServiceContractMatches(config, service);
|
|
2316
2930
|
} catch (error) {
|
|
2317
2931
|
audit(config, 'runtime_release_activation_service_contract_inspection_failed', {
|
|
2318
2932
|
operationId,
|
|
@@ -2337,8 +2951,8 @@ export async function activateRuntimeRelease(
|
|
|
2337
2951
|
attempted: false,
|
|
2338
2952
|
noOp: true,
|
|
2339
2953
|
detail: verify.ok
|
|
2340
|
-
? 'candidate Runtime behavior and
|
|
2341
|
-
: 'candidate Runtime behavior and
|
|
2954
|
+
? 'candidate Runtime behavior and installed service contract are identical to the active release; restart skipped'
|
|
2955
|
+
: 'candidate Runtime behavior and installed service contract are identical to the active release, but the active Runtime is unhealthy',
|
|
2342
2956
|
operationId,
|
|
2343
2957
|
verify,
|
|
2344
2958
|
} satisfies RuntimeReleaseActivationResult;
|
|
@@ -2386,45 +3000,81 @@ export async function activateRuntimeRelease(
|
|
|
2386
3000
|
audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId });
|
|
2387
3001
|
return { ok: false, attempted: true, detail, serviceTarget: service.target, verify: await verifyLocal(config) } satisfies RuntimeReleaseActivationResult;
|
|
2388
3002
|
}
|
|
3003
|
+
// The Connector is independently supervised, but its executable is part of
|
|
3004
|
+
// this immutable Runtime release. Rebind it immediately after publishing
|
|
3005
|
+
// authority and before starting the candidate: full Runtime verification
|
|
3006
|
+
// includes the Connector and must never observe a stale previous release.
|
|
3007
|
+
const candidateConnectorBinding = await repairConnectorBinding(config);
|
|
2389
3008
|
let storageMigration: ControllerHomeStorageMigration | undefined;
|
|
2390
|
-
const activated = await rebindStartAndVerifyPrimaryRuntime({
|
|
2391
|
-
config,
|
|
2392
|
-
service,
|
|
2393
|
-
runCommand,
|
|
2394
|
-
now,
|
|
2395
|
-
wait,
|
|
2396
|
-
verifyLocal,
|
|
2397
|
-
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
|
|
2398
|
-
successDetail: 'requested Runtime release started and passed whole-Runtime verification',
|
|
2399
|
-
beforeStart: () => {
|
|
2400
|
-
storageMigration = migrateStoppedRepoLocalControllerHomeStorage(config.controllerHome, platform);
|
|
2401
|
-
if (storageMigration.migrated) {
|
|
2402
|
-
audit(config, 'runtime_controller_home_noindex_migrated', {
|
|
2403
|
-
serviceTarget: service.target,
|
|
2404
|
-
operationId,
|
|
2405
|
-
logicalHome: storageMigration.logicalHome,
|
|
2406
|
-
physicalHome: storageMigration.physicalHome,
|
|
2407
|
-
});
|
|
2408
|
-
}
|
|
2409
|
-
},
|
|
2410
|
-
});
|
|
2411
|
-
let after = activated.verify;
|
|
2412
3009
|
let activationFailureDetail: string | undefined;
|
|
3010
|
+
let activated: PrimaryRuntimeRebindStartResult;
|
|
3011
|
+
if (!candidateConnectorBinding.ok) {
|
|
3012
|
+
const detail = `persistent Connector binding failed before Runtime activation: ${candidateConnectorBinding.detail}`;
|
|
3013
|
+
activationFailureDetail = detail;
|
|
3014
|
+
audit(config, 'runtime_release_activation_connector_binding_failed', {
|
|
3015
|
+
serviceTarget: service.target,
|
|
3016
|
+
operationId,
|
|
3017
|
+
requestId: lockRequestId,
|
|
3018
|
+
detail: candidateConnectorBinding.detail,
|
|
3019
|
+
});
|
|
3020
|
+
activated = { ok: false, detail, verify: await verifyLocal(config) };
|
|
3021
|
+
} else {
|
|
3022
|
+
activated = await rebindStartAndVerifyPrimaryRuntime({
|
|
3023
|
+
config,
|
|
3024
|
+
service,
|
|
3025
|
+
runCommand,
|
|
3026
|
+
now,
|
|
3027
|
+
wait,
|
|
3028
|
+
verifyLocal,
|
|
3029
|
+
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
|
|
3030
|
+
successDetail: 'requested Runtime release started and passed whole-Runtime verification',
|
|
3031
|
+
beforeStart: () => {
|
|
3032
|
+
storageMigration = migrateStoppedRepoLocalControllerHomeStorage(config.controllerHome, platform);
|
|
3033
|
+
if (storageMigration.migrated) {
|
|
3034
|
+
audit(config, 'runtime_controller_home_noindex_migrated', {
|
|
3035
|
+
serviceTarget: service.target,
|
|
3036
|
+
operationId,
|
|
3037
|
+
logicalHome: storageMigration.logicalHome,
|
|
3038
|
+
physicalHome: storageMigration.physicalHome,
|
|
3039
|
+
});
|
|
3040
|
+
}
|
|
3041
|
+
},
|
|
3042
|
+
});
|
|
3043
|
+
}
|
|
3044
|
+
let after = activated.verify;
|
|
2413
3045
|
if (activated.ok && after.releases.active?.revision === candidate.manifest.releaseId) {
|
|
2414
|
-
audit(config, 'runtime_release_activation_succeeded', {
|
|
2415
|
-
|
|
3046
|
+
audit(config, 'runtime_release_activation_succeeded', {
|
|
3047
|
+
serviceTarget: service.target,
|
|
3048
|
+
operationId,
|
|
3049
|
+
requestId: lockRequestId,
|
|
3050
|
+
activeRevision: after.releases.active?.revision,
|
|
3051
|
+
expectedAuthorityRevision: guard.expectedAuthorityRevision,
|
|
3052
|
+
expectedActiveReleaseId: guard.expectedActiveReleaseId,
|
|
3053
|
+
controllerHomeStorageMigrated: storageMigration?.migrated === true,
|
|
3054
|
+
connectorBindingRepaired: candidateConnectorBinding.attempted,
|
|
3055
|
+
});
|
|
3056
|
+
return {
|
|
3057
|
+
ok: true,
|
|
3058
|
+
attempted: true,
|
|
3059
|
+
detail: storageMigration?.migrated
|
|
3060
|
+
? 'requested Runtime release activated, Controller Home migrated to .noindex storage, persistent Connector rebound, and whole-Runtime verification passed'
|
|
3061
|
+
: 'requested Runtime release activated, persistent Connector rebound, and whole-Runtime verification passed',
|
|
3062
|
+
serviceTarget: service.target,
|
|
3063
|
+
operationId,
|
|
3064
|
+
verify: after,
|
|
3065
|
+
} satisfies RuntimeReleaseActivationResult;
|
|
2416
3066
|
}
|
|
2417
|
-
if (!activated.ok) {
|
|
3067
|
+
if (!activated.ok && !activationFailureDetail) {
|
|
2418
3068
|
activationFailureDetail = activated.detail;
|
|
2419
|
-
const auditAction =
|
|
2420
|
-
? '
|
|
3069
|
+
const auditAction = /^primary Runtime (launchd|systemd-user) contract rebuild failed/.test(activated.detail)
|
|
3070
|
+
? 'runtime_release_activation_service_contract_failed'
|
|
2421
3071
|
: activated.detail.startsWith('release transition pre-start preparation failed')
|
|
2422
3072
|
? 'runtime_controller_home_noindex_migration_failed'
|
|
2423
3073
|
: activated.detail.includes('start')
|
|
2424
3074
|
? 'runtime_release_activation_start_failed'
|
|
2425
3075
|
: 'runtime_release_activation_unverified';
|
|
2426
3076
|
audit(config, auditAction, { serviceTarget: service.target, operationId, detail: activated.detail });
|
|
2427
|
-
} else {
|
|
3077
|
+
} else if (!activationFailureDetail) {
|
|
2428
3078
|
activationFailureDetail = `activated Runtime release identity mismatch: expected ${candidate.manifest.releaseId}, observed ${after.releases.active?.revision ?? 'none'}`;
|
|
2429
3079
|
audit(config, 'runtime_release_activation_commit_mismatch', { serviceTarget: service.target, operationId, detail: activationFailureDetail });
|
|
2430
3080
|
}
|
|
@@ -2473,7 +3123,23 @@ export async function activateRuntimeRelease(
|
|
|
2473
3123
|
timeoutMs: configuredPrimaryRuntimeService(config).postRestartVerifyTimeoutMs ?? 60_000,
|
|
2474
3124
|
successDetail: 'previous whole-Runtime release and SQLite backup restored, restarted, and verified',
|
|
2475
3125
|
});
|
|
2476
|
-
|
|
3126
|
+
const connectorBinding = restarted.ok ? await repairConnectorBinding(config) : undefined;
|
|
3127
|
+
if (connectorBinding && !connectorBinding.ok) {
|
|
3128
|
+
audit(config, 'runtime_release_activation_rollback_connector_binding_failed', {
|
|
3129
|
+
serviceTarget: service.target,
|
|
3130
|
+
operationId,
|
|
3131
|
+
rollbackOperationId,
|
|
3132
|
+
detail: connectorBinding.detail,
|
|
3133
|
+
});
|
|
3134
|
+
}
|
|
3135
|
+
rollback = {
|
|
3136
|
+
ok: restarted.ok && (connectorBinding?.ok ?? true),
|
|
3137
|
+
operationId: rollbackOperationId,
|
|
3138
|
+
detail: connectorBinding && !connectorBinding.ok
|
|
3139
|
+
? `${restarted.detail}; previous persistent Connector binding failed: ${connectorBinding.detail}`
|
|
3140
|
+
: restarted.detail,
|
|
3141
|
+
verify: restarted.verify,
|
|
3142
|
+
};
|
|
2477
3143
|
} catch (error) {
|
|
2478
3144
|
rollback = { ok: false, detail: error instanceof Error ? error.message : 'previous whole-Runtime release rollback failed' };
|
|
2479
3145
|
}
|
|
@@ -2519,6 +3185,10 @@ export async function stageAndActivateConfiguredRuntimeRelease(
|
|
|
2519
3185
|
if (!sourceRoot) {
|
|
2520
3186
|
return { ok: false, attempted: false, noOp: true, detail: 'primary Runtime source root is not configured in standalone Recovery' };
|
|
2521
3187
|
}
|
|
3188
|
+
const sourceRepositoryId = config.primaryRuntimeSourceRepositoryId?.trim();
|
|
3189
|
+
if (!sourceRepositoryId) {
|
|
3190
|
+
return { ok: false, attempted: false, noOp: true, detail: 'primary Runtime source repository id is not configured in standalone Recovery' };
|
|
3191
|
+
}
|
|
2522
3192
|
// Capture the base before staging. Staging can take long enough for another
|
|
2523
3193
|
// activation to win; the later publish must prove this base is still current.
|
|
2524
3194
|
const expectedAuthority = releaseAuthority(config);
|
|
@@ -2527,7 +3197,7 @@ export async function stageAndActivateConfiguredRuntimeRelease(
|
|
|
2527
3197
|
...(requestId?.trim() ? { requestId: requestId.trim() } : {}),
|
|
2528
3198
|
}, async () => {
|
|
2529
3199
|
try {
|
|
2530
|
-
const staged = (dependencies.stage ?? stageRuntimeReleaseFromCandidateSource)({ controllerHome: config.controllerHome, sourceRoot });
|
|
3200
|
+
const staged = (dependencies.stage ?? stageRuntimeReleaseFromCandidateSource)({ controllerHome: config.controllerHome, sourceRoot, sourceRepositoryId });
|
|
2531
3201
|
assertRuntimeReleaseFiles(staged);
|
|
2532
3202
|
return { ok: true as const, staged };
|
|
2533
3203
|
} catch (error) {
|
|
@@ -2570,7 +3240,7 @@ export async function stageAndActivateConfiguredRuntimeRelease(
|
|
|
2570
3240
|
};
|
|
2571
3241
|
}
|
|
2572
3242
|
|
|
2573
|
-
function tunnelLaunchdService(configured:
|
|
3243
|
+
function tunnelLaunchdService(configured: RecoveryTunnelServiceConfig | undefined, uid: number): LaunchdService | undefined {
|
|
2574
3244
|
if (!configured || configured.platform !== 'launchd') return undefined;
|
|
2575
3245
|
if (!/^com\.[A-Za-z0-9._-]{1,180}$/.test(configured.label)) return undefined;
|
|
2576
3246
|
const plistPath = configured.plistPath;
|
|
@@ -2588,6 +3258,14 @@ function recoveryTunnelService(config: RecoveryConfig, uid: number): LaunchdServ
|
|
|
2588
3258
|
return tunnelLaunchdService(configuredRecoveryTunnel(config), uid);
|
|
2589
3259
|
}
|
|
2590
3260
|
|
|
3261
|
+
function recoverySystemdTunnelUnit(config: RecoveryConfig): string | undefined {
|
|
3262
|
+
const configured = configuredRecoveryTunnel(config);
|
|
3263
|
+
if (!configured || configured.platform !== 'systemd-user') return undefined;
|
|
3264
|
+
const unitName = configured.unitName.trim();
|
|
3265
|
+
if (!/^[A-Za-z0-9_.@-]{1,180}\.service$/.test(unitName)) return undefined;
|
|
3266
|
+
return unitName;
|
|
3267
|
+
}
|
|
3268
|
+
|
|
2591
3269
|
function primaryPublicTunnelService(config: RecoveryConfig, uid: number): LaunchdService | undefined {
|
|
2592
3270
|
return tunnelLaunchdService(configuredPrimaryPublicTunnel(config), uid);
|
|
2593
3271
|
}
|
|
@@ -2603,54 +3281,146 @@ export async function repairPublicTunnel(config: RecoveryConfig, dependencies: P
|
|
|
2603
3281
|
const verifyLocal = dependencies.verifyLocal ?? verifyLocalRuntime;
|
|
2604
3282
|
const now = dependencies.now ?? Date.now;
|
|
2605
3283
|
const wait = dependencies.sleep ?? sleep;
|
|
3284
|
+
const runCommand = dependencies.runCommand ?? command;
|
|
2606
3285
|
const initial = await verify(config);
|
|
2607
3286
|
const configured = configuredRecoveryTunnel(config);
|
|
2608
3287
|
if (!configured) return { ok: false, attempted: false, noOp: true, detail: 'public tunnel repair is not configured', verify: initial };
|
|
2609
3288
|
const localVerify = await verifyLocal(config);
|
|
3289
|
+
const tunnelProbe = configured.platform === 'openai-secure-tunnel'
|
|
3290
|
+
? initial.probes.recovery_tunnel_runtime
|
|
3291
|
+
: (initial.probes.recovery_external_http ?? initial.probes.external_mcp_http);
|
|
2610
3292
|
if (!isExternalTunnelFailure(config, initial, localVerify)) {
|
|
2611
|
-
return { ok:
|
|
3293
|
+
return { ok: tunnelProbe?.ok === true, attempted: false, noOp: true, detail: 'Recovery tunnel repair requires a healthy local runtime and failed managed Recovery tunnel', verify: initial, localVerify };
|
|
2612
3294
|
}
|
|
2613
|
-
|
|
2614
|
-
|
|
3295
|
+
|
|
3296
|
+
let serviceLabel: string;
|
|
3297
|
+
let serviceTarget: string;
|
|
3298
|
+
let launchdService: LaunchdService | undefined;
|
|
3299
|
+
let systemdUnitName: string | undefined;
|
|
3300
|
+
if (configured.platform === 'launchd') {
|
|
3301
|
+
if ((dependencies.platform ?? process.platform) !== 'darwin') {
|
|
3302
|
+
return { ok: false, attempted: false, noOp: true, detail: 'configured launchd public tunnel can only be repaired on macOS', verify: initial, localVerify };
|
|
3303
|
+
}
|
|
3304
|
+
const uid = await (dependencies.currentUid ?? currentUid)();
|
|
3305
|
+
launchdService = uid === undefined ? undefined : recoveryTunnelService(config, uid);
|
|
3306
|
+
if (!launchdService) return { ok: false, attempted: false, noOp: true, detail: 'public tunnel launchd configuration is invalid or unavailable', verify: initial, localVerify };
|
|
3307
|
+
serviceLabel = launchdService.label;
|
|
3308
|
+
serviceTarget = launchdService.target;
|
|
3309
|
+
} else if (configured.platform === 'systemd-user') {
|
|
3310
|
+
if ((dependencies.platform ?? process.platform) !== 'linux') {
|
|
3311
|
+
return { ok: false, attempted: false, noOp: true, detail: 'configured systemd-user public tunnel can only be repaired on Linux', verify: initial, localVerify };
|
|
3312
|
+
}
|
|
3313
|
+
systemdUnitName = recoverySystemdTunnelUnit(config);
|
|
3314
|
+
if (!systemdUnitName) return { ok: false, attempted: false, noOp: true, detail: 'public tunnel systemd-user configuration is invalid', verify: initial, localVerify };
|
|
3315
|
+
const loaded = await runCommand('systemctl', ['--user', 'show', '--property=LoadState', '--value', systemdUnitName], 5_000);
|
|
3316
|
+
if (!loaded.ok || loaded.stdout.trim() !== 'loaded') {
|
|
3317
|
+
return { ok: false, attempted: false, noOp: true, detail: `public tunnel systemd-user unit is not loaded: ${systemdUnitName}`, verify: initial, localVerify };
|
|
3318
|
+
}
|
|
3319
|
+
serviceLabel = systemdUnitName;
|
|
3320
|
+
serviceTarget = systemdUnitName;
|
|
3321
|
+
} else {
|
|
3322
|
+
serviceLabel = configured.alias;
|
|
3323
|
+
serviceTarget = `tunnel-client:${configured.alias}`;
|
|
3324
|
+
const observed = await observeOpenAiRecoveryTunnel(config, runCommand);
|
|
3325
|
+
if (observed?.observedTunnelId && !observed.tunnelMatches) {
|
|
3326
|
+
return { ok: false, attempted: false, noOp: true, detail: `OpenAI tunnel alias ${configured.alias} is already bound to a different tunnel id`, serviceLabel, serviceTarget, verify: initial, localVerify };
|
|
3327
|
+
}
|
|
3328
|
+
if (observed?.profilePath && !observed.endpointMatches) {
|
|
3329
|
+
return { ok: false, attempted: false, noOp: true, detail: `OpenAI tunnel alias ${configured.alias} is already bound to a different MCP endpoint`, serviceLabel, serviceTarget, verify: initial, localVerify };
|
|
3330
|
+
}
|
|
2615
3331
|
}
|
|
2616
|
-
|
|
2617
|
-
const service = uid === undefined ? undefined : recoveryTunnelService(config, uid);
|
|
2618
|
-
if (!service) return { ok: false, attempted: false, noOp: true, detail: 'public tunnel launchd configuration is invalid or unavailable', verify: initial, localVerify };
|
|
3332
|
+
|
|
2619
3333
|
if (!tunnelRepairAllowed(config, now())) {
|
|
2620
|
-
return { ok: false, attempted: false, noOp: true, detail: 'public tunnel repair is in cooldown', serviceLabel
|
|
3334
|
+
return { ok: false, attempted: false, noOp: true, detail: 'public tunnel repair is in cooldown', serviceLabel, serviceTarget, verify: initial, localVerify };
|
|
2621
3335
|
}
|
|
3336
|
+
|
|
2622
3337
|
const locked = await withLock(config, { action: 'repair_public_tunnel' }, async () => {
|
|
2623
3338
|
// Recheck after acquiring ownership so a concurrent watchdog never causes a restart storm.
|
|
2624
3339
|
const before = await verify(config);
|
|
2625
3340
|
const localBefore = await verifyLocal(config);
|
|
2626
3341
|
if (!isExternalTunnelFailure(config, before, localBefore)) {
|
|
2627
|
-
|
|
3342
|
+
const recoveredProbe = configured.platform === 'openai-secure-tunnel'
|
|
3343
|
+
? before.probes.recovery_tunnel_runtime
|
|
3344
|
+
: (before.probes.recovery_external_http ?? before.probes.external_mcp_http);
|
|
3345
|
+
return { ok: recoveredProbe?.ok === true, attempted: false, noOp: true, detail: 'Recovery tunnel recovered before restart', serviceLabel, serviceTarget, verify: before, localVerify: localBefore };
|
|
2628
3346
|
}
|
|
2629
|
-
writeJson(publicTunnelRepairStatePath(config), { lastAttemptAt: now(), serviceLabel
|
|
2630
|
-
|
|
2631
|
-
if (
|
|
2632
|
-
|
|
2633
|
-
|
|
3347
|
+
writeJson(publicTunnelRepairStatePath(config), { lastAttemptAt: now(), serviceLabel, serviceTarget });
|
|
3348
|
+
|
|
3349
|
+
if (configured.platform === 'launchd') {
|
|
3350
|
+
const started = await ensureLaunchdServiceStarted(launchdService!, runCommand);
|
|
3351
|
+
if (!started.ok) {
|
|
3352
|
+
audit(config, 'public_tunnel_restart_failed', { serviceLabel, serviceTarget, detail: started.detail });
|
|
3353
|
+
return { ok: false, attempted: true, detail: started.detail, serviceLabel, serviceTarget, verify: before, localVerify: localBefore };
|
|
3354
|
+
}
|
|
3355
|
+
} else if (configured.platform === 'systemd-user') {
|
|
3356
|
+
const restarted = await runCommand('systemctl', ['--user', 'restart', systemdUnitName!], 15_000);
|
|
3357
|
+
if (!restarted.ok) {
|
|
3358
|
+
const detail = `systemd-user restart failed: ${restarted.stderr || restarted.stdout || restarted.status}`;
|
|
3359
|
+
audit(config, 'public_tunnel_restart_failed', { serviceLabel, serviceTarget, detail });
|
|
3360
|
+
return { ok: false, attempted: true, detail, serviceLabel, serviceTarget, verify: before, localVerify: localBefore };
|
|
3361
|
+
}
|
|
3362
|
+
} else {
|
|
3363
|
+
const observed = await observeOpenAiRecoveryTunnel(config, runCommand);
|
|
3364
|
+
if (observed?.observedTunnelId && !observed.tunnelMatches) {
|
|
3365
|
+
const detail = `OpenAI tunnel alias ${configured.alias} changed ownership to ${observed.observedTunnelId}; refusing to repoint it`;
|
|
3366
|
+
audit(config, 'public_tunnel_restart_failed', { serviceLabel, serviceTarget, detail });
|
|
3367
|
+
return { ok: false, attempted: false, noOp: true, detail, serviceLabel, serviceTarget, verify: before, localVerify: localBefore };
|
|
3368
|
+
}
|
|
3369
|
+
if (observed?.profilePath && !observed.endpointMatches) {
|
|
3370
|
+
const detail = `OpenAI tunnel alias ${configured.alias} targets a different MCP endpoint; refusing to repoint it`;
|
|
3371
|
+
audit(config, 'public_tunnel_restart_failed', { serviceLabel, serviceTarget, detail });
|
|
3372
|
+
return { ok: false, attempted: false, noOp: true, detail, serviceLabel, serviceTarget, verify: before, localVerify: localBefore };
|
|
3373
|
+
}
|
|
3374
|
+
let args: string[];
|
|
3375
|
+
try {
|
|
3376
|
+
args = openAiSecureTunnelConnectArgs({
|
|
3377
|
+
alias: configured.alias,
|
|
3378
|
+
tunnelId: configured.tunnelId,
|
|
3379
|
+
mcpServerUrl: configured.mcpServerUrl,
|
|
3380
|
+
runtimeApiKeyRef: configured.runtimeApiKeyRef,
|
|
3381
|
+
profile: configured.profile,
|
|
3382
|
+
profileDir: configured.profileDir,
|
|
3383
|
+
adminProfile: configured.adminProfile,
|
|
3384
|
+
});
|
|
3385
|
+
} catch (error) {
|
|
3386
|
+
const detail = error instanceof Error ? error.message : 'OpenAI tunnel runtime configuration is invalid';
|
|
3387
|
+
audit(config, 'public_tunnel_restart_failed', { serviceLabel, serviceTarget, detail });
|
|
3388
|
+
return { ok: false, attempted: false, noOp: true, detail, serviceLabel, serviceTarget, verify: before, localVerify: localBefore };
|
|
3389
|
+
}
|
|
3390
|
+
const connected = await runCommand('tunnel-client', args, Math.max(30_000, configured.postRestartVerifyTimeoutMs ?? 20_000));
|
|
3391
|
+
if (!connected.ok) {
|
|
3392
|
+
const detail = `OpenAI Secure MCP Tunnel runtime connect failed: ${connected.stderr || connected.stdout || connected.status}`;
|
|
3393
|
+
audit(config, 'public_tunnel_restart_failed', { serviceLabel, serviceTarget, detail });
|
|
3394
|
+
return { ok: false, attempted: true, detail, serviceLabel, serviceTarget, verify: before, localVerify: localBefore };
|
|
3395
|
+
}
|
|
2634
3396
|
}
|
|
3397
|
+
|
|
2635
3398
|
const timeoutMs = configured.postRestartVerifyTimeoutMs ?? 20_000;
|
|
2636
3399
|
const deadline = now() + timeoutMs;
|
|
2637
3400
|
let after = before;
|
|
2638
3401
|
while (now() < deadline) {
|
|
2639
3402
|
await wait(1_000);
|
|
3403
|
+
if (configured.platform === 'openai-secure-tunnel') {
|
|
3404
|
+
const observed = await observeOpenAiRecoveryTunnel(config, runCommand);
|
|
3405
|
+
if (!observed?.ok) continue;
|
|
3406
|
+
}
|
|
2640
3407
|
after = await verify(config);
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
3408
|
+
const recovered = configured.platform === 'openai-secure-tunnel'
|
|
3409
|
+
? after.probes.recovery_tunnel_runtime?.ok === true
|
|
3410
|
+
: (after.probes.recovery_external_http ?? after.probes.external_mcp_http)?.ok === true;
|
|
3411
|
+
if (recovered) {
|
|
3412
|
+
audit(config, 'public_tunnel_restart_succeeded', { serviceLabel, serviceTarget });
|
|
3413
|
+
return { ok: true, attempted: true, detail: 'public tunnel service restarted and external tunnel readiness verified', serviceLabel, serviceTarget, verify: after, localVerify: await verifyLocal(config) };
|
|
2644
3414
|
}
|
|
2645
3415
|
}
|
|
2646
|
-
|
|
2647
|
-
|
|
3416
|
+
after = await verify(config);
|
|
3417
|
+
audit(config, 'public_tunnel_restart_unverified', { serviceLabel, serviceTarget });
|
|
3418
|
+
return { ok: false, attempted: true, detail: 'public tunnel service restarted but external tunnel readiness did not recover before timeout', serviceLabel, serviceTarget, verify: after, localVerify: await verifyLocal(config) };
|
|
2648
3419
|
});
|
|
2649
|
-
if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), serviceLabel
|
|
3420
|
+
if (!locked.acquired) return { ok: false, attempted: false, noOp: true, detail: recoveryBusyDetail(locked.owner), serviceLabel, serviceTarget, verify: initial, localVerify };
|
|
2650
3421
|
return locked.value;
|
|
2651
3422
|
}
|
|
2652
3423
|
|
|
2653
|
-
|
|
2654
3424
|
export interface RecoveryGatewayRestartResult {
|
|
2655
3425
|
ok: boolean;
|
|
2656
3426
|
attempted: boolean;
|
|
@@ -2795,7 +3565,7 @@ export function gatewayToken(config: RecoveryConfig): string | undefined {
|
|
|
2795
3565
|
export function initializeStandaloneRecovery(
|
|
2796
3566
|
controllerHome: string,
|
|
2797
3567
|
port = 8787,
|
|
2798
|
-
extensions: Partial<Pick<RecoveryConfig, 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryPublicTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
|
|
3568
|
+
extensions: Partial<Pick<RecoveryConfig, 'publicMcpUrl' | 'recoveryPublicUrl' | 'recoveryTunnelService' | 'primaryPublicTunnelService' | 'primaryRuntimeService' | 'primaryRuntimeSourceRoot' | 'primaryRuntimeSourceRepositoryId' | 'primaryConnectorService' | 'readOnlyTool'>> = {},
|
|
2799
3569
|
): RecoveryConfig {
|
|
2800
3570
|
const root = resolve(controllerHome);
|
|
2801
3571
|
const tokenPath = join(root, 'recovery', 'config', 'gateway-token.json');
|
|
@@ -2902,22 +3672,18 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
2902
3672
|
}
|
|
2903
3673
|
const recoveryHealthy = verified.probes.recovery_gateway?.ok !== false
|
|
2904
3674
|
&& verified.probes.recovery_external_http?.ok !== false;
|
|
2905
|
-
const primaryRuntimeHealthy = localVerify
|
|
2906
|
-
&& localVerify.runtime.running
|
|
2907
|
-
&& localVerify.runtime.ready
|
|
2908
|
-
&& !localVerify.runtime.stale
|
|
2909
|
-
&& localVerify.probes.active_gateway?.ok === true;
|
|
3675
|
+
const primaryRuntimeHealthy = canonicalRuntimeSafeForTargetedConnectorRecovery(localVerify);
|
|
2910
3676
|
const primaryConnectorConfigured = Boolean(config.primaryConnectorService);
|
|
2911
3677
|
const primaryConnectorLocalFailed = verified.probes.primary_connector_local?.ok === false;
|
|
2912
3678
|
const primaryConnectorCapacityFailed = connectorCapacityRecoveryRecommended(verified);
|
|
2913
|
-
const
|
|
3679
|
+
const primaryPublicTunnel = configuredPrimaryPublicTunnel(config);
|
|
3680
|
+
const primaryPublicTransportManaged = Boolean(primaryPublicTunnel);
|
|
2914
3681
|
const primaryPublicTransportFailed = Boolean(
|
|
2915
|
-
|
|
2916
|
-
&& primaryRuntimeHealthy
|
|
3682
|
+
primaryRuntimeHealthy
|
|
2917
3683
|
&& !primaryConnectorCapacityFailed
|
|
2918
3684
|
&& (
|
|
2919
|
-
verified.probes.
|
|
2920
|
-
|| verified.probes.primary_connector_ready?.ok === false
|
|
3685
|
+
(primaryPublicTunnel?.platform === 'openai-secure-tunnel' && verified.probes.primary_tunnel_runtime?.ok === false)
|
|
3686
|
+
|| (Boolean(config.publicMcpUrl) && (verified.probes.external_mcp_http?.ok === false || verified.probes.primary_connector_ready?.ok === false))
|
|
2921
3687
|
),
|
|
2922
3688
|
);
|
|
2923
3689
|
const primaryConnectorFailed = Boolean(
|
|
@@ -2930,18 +3696,6 @@ export async function watchdogTick(config: RecoveryConfig, prior: WatchdogState)
|
|
|
2930
3696
|
),
|
|
2931
3697
|
);
|
|
2932
3698
|
if (primaryRuntimeHealthy && !primaryConnectorFailed && !primaryPublicTransportFailed && recoveryHealthy) {
|
|
2933
|
-
if (fullVerificationPerformed && verified.ok && !matchingKnownGood(config, verified.releases.active)) {
|
|
2934
|
-
try {
|
|
2935
|
-
const attestation = await withLock(config, { action: 'attest_known_good' }, async () => persistVerifiedKnownGood(config, verified));
|
|
2936
|
-
if (!attestation.acquired) {
|
|
2937
|
-
audit(config, 'known_good_attestation_deferred', { reason: recoveryBusyDetail(attestation.owner) });
|
|
2938
|
-
}
|
|
2939
|
-
} catch (error) {
|
|
2940
|
-
audit(config, 'known_good_attestation_deferred', {
|
|
2941
|
-
reason: error instanceof Error ? error.message : 'watchdog known-good attestation failed',
|
|
2942
|
-
});
|
|
2943
|
-
}
|
|
2944
|
-
}
|
|
2945
3699
|
const stable = recordWatchdogRuntimeHealthy(
|
|
2946
3700
|
scopeWatchdogStateToRuntimeRelease(scopedPrior, verified.releases.active),
|
|
2947
3701
|
now,
|