@bitkyc08/opencodex 2.100.2 → 2.100.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS_INSTALL.md +23 -1
- package/README.md +1 -1
- package/bin/ocx.mjs +40 -19
- package/gui/dist/assets/{App-COo8Yopd.css → App-BGP8I0V4.css} +1 -1
- package/gui/dist/assets/App-DcpyI78f.js +60 -0
- package/gui/dist/assets/{Tray-BcaI-VCr.js → Tray-CzjPJprT.js} +1 -1
- package/gui/dist/assets/brand-mark-BTIi7e69.js +1 -0
- package/gui/dist/assets/index-DJxM-imn.css +2 -0
- package/gui/dist/assets/index-DVgU6S3y.js +138 -0
- package/gui/dist/assets/{segment-thumb-features-e9A_FogJ.js → segment-thumb-features-Bn6Y-g3R.js} +1 -1
- package/gui/dist/classic/assets/App-eYJAyNHM.js +59 -0
- package/gui/dist/classic/assets/{Tray-DT146AhS.js → Tray-xHMYxLkV.js} +1 -1
- package/gui/dist/classic/assets/index-DSaVRW4F.js +138 -0
- package/gui/dist/classic/assets/quota-summary-DyzHlkFl.js +1 -0
- package/gui/dist/classic/index.html +1 -1
- package/gui/dist/index.html +2 -2
- package/native/ocx-child/Cargo.toml +9 -0
- package/native/ocx-child/src/cli.rs +404 -20
- package/native/ocx-child/src/codex_home.rs +99 -14
- package/native/ocx-child/src/daemon.rs +2 -1
- package/native/ocx-child/src/lib.rs +3 -0
- package/native/ocx-child/src/link_ws.rs +2598 -0
- package/native/ocx-child/src/link_ws_advert.rs +346 -0
- package/native/ocx-child/src/link_ws_frame.rs +749 -0
- package/native/ocx-child/src/parity.rs +5 -1
- package/native/ocx-child/src/platform.rs +152 -6
- package/native/ocx-child/src/relay.rs +150 -19
- package/package.json +3 -3
- package/src/adapters/anthropic/advisor-tool.ts +176 -0
- package/src/adapters/anthropic/beta-body-pairs.ts +6 -0
- package/src/adapters/anthropic/client-identity.ts +3 -1
- package/src/adapters/anthropic/passthrough.ts +9 -3
- package/src/adapters/anthropic/per-message-effort.ts +296 -0
- package/src/adapters/anthropic.ts +34 -14
- package/src/adapters/claude-cli/adapter.ts +1 -0
- package/src/adapters/client-fingerprint.ts +3 -3
- package/src/adapters/cursor/discovery.ts +17 -0
- package/src/adapters/cursor/native-exec-desktop.ts +2 -1
- package/src/adapters/cursor/native-exec-shell.ts +2 -1
- package/src/adapters/cursor/protobuf-request.ts +8 -8
- package/src/adapters/cursor.ts +7 -6
- package/src/adapters/devin/cloud-direct/chat.ts +71 -14
- package/src/adapters/google-antigravity-wire.ts +7 -5
- package/src/adapters/google-errors.ts +14 -2
- package/src/adapters/google-wire-compiler.ts +2 -2
- package/src/adapters/google.ts +9 -2
- package/src/adapters/kiro/payload.ts +7 -4
- package/src/adapters/kiro/reasoning.ts +44 -1
- package/src/adapters/kiro/structured-output.ts +10 -8
- package/src/adapters/kiro-tools.ts +61 -15
- package/src/adapters/openai-chat/destination-policy.ts +16 -0
- package/src/adapters/openai-chat/reasoning-replay.ts +4 -1
- package/src/adapters/qoder/adapter.ts +9 -1
- package/src/adapters/zed.ts +7 -2
- package/src/advisor/consult.ts +15 -4
- package/src/advisor/context.ts +12 -3
- package/src/advisor/runtime.ts +35 -5
- package/src/advisor/settings.ts +8 -0
- package/src/advisor/synthetic-tool.ts +21 -8
- package/src/bridge/response-json.ts +11 -2
- package/src/bridge/sse.ts +43 -8
- package/src/claude/auth-detect.ts +21 -33
- package/src/claude/desktop-3p.ts +11 -5
- package/src/claude/desktop-gateway-state.ts +2 -0
- package/src/claude/desktop-remote-store-artifact.ts +14 -3
- package/src/claude/desktop-remote-store.ts +82 -12
- package/src/claude/inbound-content-options.ts +31 -16
- package/src/claude/outbound.ts +13 -3
- package/src/cli/account-api.ts +21 -1
- package/src/cli/account-auth.ts +83 -22
- package/src/cli/account-extended.ts +33 -2
- package/src/cli/account-key-quota.ts +34 -5
- package/src/cli/account.ts +4 -2
- package/src/cli/advisor.ts +7 -4
- package/src/cli/alias.ts +38 -1
- package/src/cli/capabilities-access-remote.ts +128 -3
- package/src/cli/capabilities-accounts.ts +6 -6
- package/src/cli/capabilities-agents-routing.ts +17 -10
- package/src/cli/capabilities-base.ts +49 -19
- package/src/cli/capabilities-integrations.ts +2 -2
- package/src/cli/capabilities-observe-system.ts +2 -2
- package/src/cli/capabilities-provider-models.ts +18 -2
- package/src/cli/capabilities-route-to.ts +43 -0
- package/src/cli/capabilities.ts +2 -0
- package/src/cli/capability-contracts.ts +1 -1
- package/src/cli/claude-desktop.ts +30 -11
- package/src/cli/claude.ts +17 -3
- package/src/cli/codex-launcher.ts +3 -1
- package/src/cli/codex-shim-autorestore.ts +5 -42
- package/src/cli/config-command.ts +14 -1
- package/src/cli/config-root-startup.ts +34 -0
- package/src/cli/connect.ts +88 -16
- package/src/cli/desktop-runtime-guidance.ts +1 -1
- package/src/cli/disconnect-runtime.ts +178 -0
- package/src/cli/dispatch.ts +61 -13
- package/src/cli/export-command.ts +1 -1
- package/src/cli/gui-pair-client.ts +6 -3
- package/src/cli/gui.ts +10 -5
- package/src/cli/help-catalog.ts +11 -0
- package/src/cli/help.ts +4 -3
- package/src/cli/index.ts +85 -94
- package/src/cli/inspect.ts +11 -2
- package/src/cli/integrations.ts +26 -1
- package/src/cli/link-apply-restart.ts +4 -2
- package/src/cli/link-capture.ts +64 -0
- package/src/cli/link-child-ops.ts +218 -40
- package/src/cli/link-cleanup.ts +8 -5
- package/src/cli/link-diagnostics.ts +67 -0
- package/src/cli/link-enrollment.ts +158 -12
- package/src/cli/link-operations.ts +52 -0
- package/src/cli/link-recover.ts +152 -0
- package/src/cli/link-routes.ts +90 -0
- package/src/cli/link-status-child.ts +44 -1
- package/src/cli/link-status-traffic.ts +34 -0
- package/src/cli/link.ts +315 -37
- package/src/cli/message-command.ts +1 -1
- package/src/cli/message-mcp-command.ts +23 -10
- package/src/cli/models-runtime.ts +3 -3
- package/src/cli/opencode.ts +3 -1
- package/src/cli/pi-sync.ts +35 -0
- package/src/cli/preflight-exemptions.ts +49 -0
- package/src/cli/registry.ts +33 -14
- package/src/cli/resolve.ts +23 -8
- package/src/cli/restart-failure.ts +15 -2
- package/src/cli/root.ts +5 -1
- package/src/cli/route-to.ts +172 -0
- package/src/cli/runtime-api.ts +9 -8
- package/src/cli/status.ts +7 -2
- package/src/cli/stop-approval.ts +66 -19
- package/src/cli/stop-report.ts +23 -0
- package/src/cli/stop-windows-checked.ts +304 -0
- package/src/cli/sync-command.ts +70 -15
- package/src/cli/sync-fanout.ts +74 -19
- package/src/cli/sync-flags.ts +40 -3
- package/src/cli/sync-link-status.ts +18 -2
- package/src/cli/sync-update-guard.ts +7 -2
- package/src/cli/system-command.ts +1 -1
- package/src/cli/system-restart-client.ts +10 -3
- package/src/cli/system-settings-parity.ts +2 -1
- package/src/cli/uninstall-plan.ts +77 -5
- package/src/cli/uninstall-stop.ts +152 -0
- package/src/cli/v2-input.ts +10 -6
- package/src/cli/v2-local-output.ts +3 -2
- package/src/cli/v2-runtime.ts +10 -3
- package/src/cli/v2.ts +22 -7
- package/src/client/connect.ts +376 -35
- package/src/client/desktop-refusal.ts +38 -0
- package/src/client/lifecycle-lock.ts +29 -4
- package/src/client/link-home-features.ts +27 -0
- package/src/client/link-home.ts +92 -4
- package/src/client/link-ingress.ts +21 -3
- package/src/client/link-join.ts +12 -2
- package/src/client/link-parity-reporter.ts +60 -6
- package/src/client/link-relay-transport.ts +161 -36
- package/src/client/link-relay.ts +150 -6
- package/src/client/link-request-compression.ts +84 -0
- package/src/client/link-state.ts +11 -5
- package/src/client/link-status.ts +39 -1
- package/src/client/link-teardown.ts +1 -0
- package/src/client/link-tunnel.ts +56 -5
- package/src/client/link-websocket-limits.ts +89 -0
- package/src/client/link-websocket.ts +586 -0
- package/src/client/link-ws-advertisement.ts +100 -0
- package/src/client/link-ws-framer.ts +241 -0
- package/src/client/link-ws-transport.ts +177 -0
- package/src/client/machine-api.ts +13 -2
- package/src/client/machine-listener.ts +48 -7
- package/src/client/route-intent-apply.ts +293 -0
- package/src/client/route-intent-file-originals.ts +97 -0
- package/src/client/route-intent-native-originals.ts +201 -0
- package/src/client/route-intent-watch.ts +141 -0
- package/src/client/runtime-restart.ts +141 -7
- package/src/client/runtime.ts +63 -15
- package/src/client/state.ts +44 -0
- package/src/client/sync-expectation.ts +36 -2
- package/src/client/sync-runtime.ts +7 -0
- package/src/clients/config-export/alias-selectors.ts +63 -40
- package/src/clients/config-export/contracts.ts +2 -0
- package/src/clients/config-export/nested-selectors.ts +42 -0
- package/src/clients/config-export.ts +22 -3
- package/src/codex/account-pause.ts +4 -0
- package/src/codex/account-runtime-state.ts +4 -0
- package/src/codex/account-store.ts +12 -3
- package/src/codex/account-usability.ts +13 -0
- package/src/codex/auth-api/account-list.ts +29 -3
- package/src/codex/auth-api/main-account-probe.ts +3 -1
- package/src/codex/auth-api/main-hard-lock-refusal-read.ts +172 -0
- package/src/codex/auth-collision.ts +23 -22
- package/src/codex/auth-context.ts +61 -8
- package/src/codex/catalog/build-entries.ts +10 -0
- package/src/codex/catalog/derive-entry.ts +7 -0
- package/src/codex/catalog/gather-capture.ts +8 -2
- package/src/codex/catalog/parsing.ts +7 -3
- package/src/codex/catalog/provider-models.ts +54 -15
- package/src/codex/catalog/reserve-warn.ts +7 -5
- package/src/codex/catalog/reserve.ts +4 -3
- package/src/codex/catalog/retained-sync.ts +4 -1
- package/src/codex/catalog/routed-defaults.ts +35 -0
- package/src/codex/catalog/routed-gather.ts +1 -1
- package/src/codex/catalog-auto-refresh.ts +11 -0
- package/src/codex/cli-update-job.ts +35 -13
- package/src/codex/convergence.ts +4 -2
- package/src/codex/desktop-compatibility/native-identity.ts +4 -8
- package/src/codex/features.ts +2 -2
- package/src/codex/history-provider.ts +31 -0
- package/src/codex/inject/multi-agent-v2.ts +9 -3
- package/src/codex/inject/paginated-openai-compat.ts +54 -1
- package/src/codex/inject/plan.ts +20 -2
- package/src/codex/inject.ts +2 -0
- package/src/codex/journal-config-restore.ts +217 -0
- package/src/codex/journal.ts +75 -6
- package/src/codex/loopback-target.ts +22 -8
- package/src/codex/main-account.ts +109 -57
- package/src/codex/native-main-auth-file.ts +118 -0
- package/src/codex/native-main-credentials.ts +270 -0
- package/src/codex/native-main-owner.ts +3 -0
- package/src/codex/native-main-refresh-mode.ts +14 -1
- package/src/codex/native-profile-manager.ts +4 -3
- package/src/codex/native-profile-store.ts +3 -61
- package/src/codex/orca-import.ts +10 -1
- package/src/codex/quota-history.ts +13 -0
- package/src/codex/quota-window-conflict.ts +103 -0
- package/src/codex/quota.ts +86 -15
- package/src/codex/subagent-model-fallback.ts +3 -1
- package/src/codex/websocket-activity.ts +171 -16
- package/src/codex/websocket-registry.ts +11 -4
- package/src/combos/cooldown-store.ts +25 -5
- package/src/combos/failover.ts +342 -3
- package/src/combos/index.ts +5 -0
- package/src/combos/resolve.ts +115 -19
- package/src/config/dashboard-trusted-proxy.ts +83 -0
- package/src/config/diagnostics.ts +31 -0
- package/src/config/feature-flags.ts +9 -0
- package/src/config/live-reconcile.ts +16 -4
- package/src/config/load-degrade.ts +14 -1
- package/src/config/multi-agent-surface.ts +28 -5
- package/src/config/pending-teardown.ts +60 -3
- package/src/config/persisted-mutation.ts +2 -2
- package/src/config/persisted-projections.ts +18 -0
- package/src/config/process-state.ts +176 -4
- package/src/config/provider-validation.ts +4 -4
- package/src/config/reserve-capability.ts +96 -0
- package/src/config/schema/config-schema.ts +12 -0
- package/src/config/schema/leaf-validators.ts +11 -2
- package/src/config/schema/routed-catalog-defaults.ts +49 -0
- package/src/config/windows-process-command-line.ts +107 -0
- package/src/config.ts +7 -7
- package/src/desktop-update/acquire.ts +202 -0
- package/src/desktop-update/cli.ts +168 -0
- package/src/desktop-update/drain.ts +40 -0
- package/src/desktop-update/headless.ts +285 -0
- package/src/desktop-update/image.ts +85 -0
- package/src/desktop-update/install.ts +53 -0
- package/src/desktop-update/msi.ts +72 -0
- package/src/desktop-update/release.ts +128 -0
- package/src/desktop-update/runtime.ts +155 -0
- package/src/desktop-update/staging.ts +118 -0
- package/src/desktop-update/trust.ts +45 -0
- package/src/generated/compatibility-version.json +1114 -442
- package/src/generated/model-metadata.ts +1 -1
- package/src/images/fulfill-video.ts +3 -3
- package/src/images/loop.ts +9 -2
- package/src/images/plan.ts +147 -11
- package/src/images/types.ts +13 -4
- package/src/images/xai-video-client.ts +72 -10
- package/src/integrations/model-name-guard.ts +25 -0
- package/src/integrations/route-intent-catalog.ts +26 -0
- package/src/integrations/route-intent-connected.ts +262 -0
- package/src/integrations/route-intent-contract.ts +55 -0
- package/src/integrations/route-intent-disconnect.ts +109 -0
- package/src/integrations/route-intent-initial.ts +64 -0
- package/src/integrations/route-intent-state.ts +117 -0
- package/src/integrations/route-intent-tick.ts +65 -0
- package/src/integrations/route-intent.ts +212 -0
- package/src/integrations/writer.ts +44 -11
- package/src/lib/client-retry-after.ts +29 -0
- package/src/lib/config-ownership.ts +13 -0
- package/src/lib/desktop-gui-origin.ts +28 -0
- package/src/lib/errors.ts +1 -1
- package/src/lib/gcp-adc.ts +48 -4
- package/src/lib/gui-pair-capability.ts +2 -1
- package/src/lib/messages-error-body.ts +4 -1
- package/src/lib/offline-resend.ts +150 -0
- package/src/lib/outbound-credential-registry.ts +362 -4
- package/src/lib/paced-stream.ts +34 -0
- package/src/lib/passive-key-fetch-proof.ts +30 -0
- package/src/lib/passive-read.ts +5 -0
- package/src/lib/process-control.ts +76 -12
- package/src/lib/provider-outbound.ts +17 -6
- package/src/lib/redact.ts +84 -8
- package/src/lib/request-failure-model.ts +2 -0
- package/src/lib/response-body-inactivity.ts +23 -1
- package/src/lib/state-store-registrations.ts +11 -0
- package/src/lib/state-store-sweeper.ts +4 -0
- package/src/lib/supervisor-markers.ts +18 -0
- package/src/lib/terminal-safe-text.ts +58 -0
- package/src/lib/translator-budget.ts +3 -2
- package/src/lib/upstream-retry.ts +52 -9
- package/src/link/alias-lock.ts +119 -0
- package/src/link/child-activity.ts +28 -2
- package/src/link/child-binary-cache-fill.d.mts +50 -0
- package/src/link/child-binary-cache-fill.mjs +494 -0
- package/src/link/child-binary-cache-start.ts +73 -0
- package/src/link/child-binary-source.ts +12 -5
- package/src/link/child-gated-ocx.ts +29 -11
- package/src/link/child-observations.ts +2 -1
- package/src/link/child-observe.ts +2 -2
- package/src/link/child-ops-engine.ts +159 -91
- package/src/link/child-ops-host.ts +138 -0
- package/src/link/child-ops-runners.ts +319 -0
- package/src/link/child-ops-scheduler.ts +207 -0
- package/src/link/child-ops-types.ts +89 -0
- package/src/link/child-parity.ts +23 -4
- package/src/link/child-policy.ts +93 -0
- package/src/link/child-update-contract.ts +12 -3
- package/src/link/codex-settings.ts +1 -1
- package/src/link/compatibility.ts +18 -1
- package/src/link/diagnostics-read.ts +163 -0
- package/src/link/diagnostics.ts +20 -7
- package/src/link/explicit-route.ts +209 -0
- package/src/link/link-recover.ts +113 -0
- package/src/link/management-routes.ts +3 -1
- package/src/link/operation-recovery-slot.ts +44 -0
- package/src/link/operations-journal.ts +408 -0
- package/src/link/peer-report.ts +1 -1
- package/src/link/relay-auth.ts +14 -0
- package/src/link/remote-plant.ts +152 -14
- package/src/link/remote-platform.ts +21 -10
- package/src/link/remote-probe.ts +11 -3
- package/src/link/remote-windows.ts +124 -36
- package/src/link/request-compression.ts +30 -0
- package/src/link/residual-recovery.ts +66 -0
- package/src/link/retirement.ts +128 -0
- package/src/link/route-intent-watch.ts +41 -0
- package/src/link/route-intent.ts +209 -0
- package/src/link/route-proof.ts +82 -0
- package/src/link/route-rounds.ts +579 -0
- package/src/link/route-select.ts +95 -0
- package/src/link/route-trust.ts +96 -0
- package/src/link/ssh-argv.ts +172 -24
- package/src/link/ssh-capture-redact.ts +253 -0
- package/src/link/ssh-capture.ts +497 -0
- package/src/link/ssh-runner.ts +141 -14
- package/src/link/status-projection.ts +87 -0
- package/src/link/store.ts +54 -1
- package/src/link/supervisor.ts +543 -55
- package/src/link/sync-fanout-classify.ts +16 -0
- package/src/link/sync-then-restart.ts +65 -0
- package/src/link/tombstones.ts +209 -0
- package/src/link/traffic.ts +178 -0
- package/src/link/ws-activity-holds.ts +474 -0
- package/src/local-credentials/publication-fence.ts +20 -1
- package/src/messaging/budget.ts +23 -5
- package/src/messaging/mcp-queue-rpc.ts +1 -1
- package/src/messaging/mcp-queue-server.ts +16 -1
- package/src/messaging/mcp-resume-tools.ts +25 -0
- package/src/messaging/remote-enrollment.ts +3 -3
- package/src/messaging/remote-tunnels.ts +2 -2
- package/src/messaging/resume-permissions.ts +27 -0
- package/src/messaging/rpc.ts +50 -6
- package/src/oauth/anthropic-continuity.ts +23 -7
- package/src/oauth/anthropic-routing.ts +5 -2
- package/src/oauth/anthropic.ts +22 -2
- package/src/oauth/antigravity-risk.ts +126 -0
- package/src/oauth/claude-keychain-read.ts +78 -68
- package/src/oauth/generic-account-failover.ts +89 -18
- package/src/oauth/generic-session-affinity.ts +10 -3
- package/src/oauth/index.ts +24 -6
- package/src/oauth/local-token-detect.ts +8 -29
- package/src/oauth/login-cli-errors.ts +5 -1
- package/src/oauth/login-cli.ts +68 -19
- package/src/oauth/login-flow-state.ts +26 -3
- package/src/oauth/meta-muse.ts +1 -1
- package/src/oauth/pool-kernel.ts +3 -3
- package/src/oauth/store.ts +7 -0
- package/src/oauth/xai.ts +248 -14
- package/src/protocols/contract.ts +4 -0
- package/src/protocols/encoders/messages.ts +1 -0
- package/src/providers/alias-template.ts +48 -0
- package/src/providers/anthropic-billing-ownership.ts +2 -0
- package/src/providers/anthropic-billing-refusal.ts +27 -3
- package/src/providers/anthropic-output-caps.ts +46 -0
- package/src/providers/anthropic-reset-grant-ledger.ts +9 -3
- package/src/providers/antigravity-effort-families.ts +11 -3
- package/src/providers/antigravity-models.ts +32 -9
- package/src/providers/antigravity-wire-snapshot.ts +3 -2
- package/src/providers/api-key-pause.ts +3 -0
- package/src/providers/api-key-resolve.ts +49 -7
- package/src/providers/api-key-response-ownership.ts +2 -0
- package/src/providers/api-key-selection.ts +51 -9
- package/src/providers/api-keys.ts +6 -2
- package/src/providers/cache-affinity-identity.ts +73 -0
- package/src/providers/cache-affinity.ts +256 -0
- package/src/providers/default-aliases.ts +98 -20
- package/src/providers/google-vertex-model-discovery.ts +515 -0
- package/src/providers/key-failover.ts +123 -8
- package/src/providers/key-store.ts +3 -3
- package/src/providers/kiro-models.ts +3 -5
- package/src/providers/label.ts +31 -5
- package/src/providers/mistral-transport.ts +17 -4
- package/src/providers/model-rename-fields.ts +5 -0
- package/src/providers/muse-key-quota.ts +14 -6
- package/src/providers/new-model-policy-runtime.ts +17 -4
- package/src/providers/new-model-policy.ts +30 -4
- package/src/providers/passive-key-observation.ts +252 -0
- package/src/providers/quota/account-cache.ts +2 -1
- package/src/providers/quota/anthropic-cooldown-recovery-worker.ts +121 -0
- package/src/providers/quota/passive-key-limits.ts +272 -0
- package/src/providers/quota/passive-wire-model.ts +126 -0
- package/src/providers/quota/report-cache.ts +61 -2
- package/src/providers/quota/workbuddy.ts +6 -2
- package/src/providers/quota-key-accounts.ts +11 -6
- package/src/providers/quota-types.ts +18 -0
- package/src/providers/quota.ts +11 -12
- package/src/providers/registry/entries-core.ts +34 -16
- package/src/providers/registry/entries-extended.ts +32 -0
- package/src/providers/registry/model-seeds.ts +35 -0
- package/src/providers/resolved-model-policy.ts +7 -0
- package/src/providers/routed-config.ts +8 -3
- package/src/providers/workbuddy-credit-balance.ts +125 -16
- package/src/providers/xai-transport.ts +3 -0
- package/src/remote/features.ts +17 -0
- package/src/responses/anthropic-progress.ts +10 -4
- package/src/responses/citation-markers.ts +231 -0
- package/src/responses/effort-history.ts +732 -0
- package/src/responses/spill-store.ts +5 -1
- package/src/responses/state/replay-input.ts +50 -0
- package/src/responses/state/replay-owner.ts +24 -0
- package/src/responses/state/snapshot-codec.ts +9 -0
- package/src/responses/state/spill-queue.ts +1 -0
- package/src/responses/state.ts +42 -60
- package/src/router.ts +2 -2
- package/src/routing/history/integrity.ts +41 -4
- package/src/routing/home-route-capture.ts +123 -0
- package/src/routing/home-route-intent.ts +200 -0
- package/src/routing/route-intent-state.ts +178 -0
- package/src/routing/route-to-live-settings.ts +35 -0
- package/src/routing/route-to-local.ts +145 -0
- package/src/routing/route-to-status.ts +94 -0
- package/src/routing/route-to-sync-child.ts +32 -0
- package/src/server/adapter-resolve.ts +21 -1
- package/src/server/admitted-http-turn.ts +3 -0
- package/src/server/auth-cors.ts +111 -3
- package/src/server/bare-upstream-error.ts +220 -0
- package/src/server/chat-native.ts +14 -5
- package/src/server/claude-messages.ts +6 -4
- package/src/server/codex-model-list.ts +4 -2
- package/src/server/configured-credentials.ts +37 -5
- package/src/server/direct-local-http.ts +26 -3
- package/src/server/gui-desktop-authorization.ts +36 -0
- package/src/server/gui-pair-delivery.ts +6 -3
- package/src/server/gui-session.ts +10 -1
- package/src/server/inbound-body-admission.ts +17 -4
- package/src/server/index/link-listener.ts +196 -12
- package/src/server/index/link-management.ts +15 -1
- package/src/server/index/link-relay-sessions.ts +87 -8
- package/src/server/index/link-websocket-limits.ts +136 -0
- package/src/server/index/link-websocket.ts +692 -0
- package/src/server/index/optional-listeners.ts +97 -7
- package/src/server/index/route-intent-watch.ts +47 -0
- package/src/server/index/serve-contract.ts +86 -0
- package/src/server/index/serve-options.ts +36 -86
- package/src/server/index/startup-warnings.ts +8 -2
- package/src/server/index/websocket-handler.ts +26 -4
- package/src/server/index.ts +15 -4
- package/src/server/inference/client-encoder-delivery.ts +2 -0
- package/src/server/inspection-tee.ts +39 -2
- package/src/server/lifecycle.ts +2 -0
- package/src/server/link-turn.ts +149 -0
- package/src/server/management/advisor-routes.ts +10 -2
- package/src/server/management/agent-settings-routes.ts +50 -16
- package/src/server/management/anthropic-reset-grant-routes.ts +1 -1
- package/src/server/management/config-routes.ts +39 -3
- package/src/server/management/context.ts +9 -2
- package/src/server/management/desktop-authorization.ts +41 -0
- package/src/server/management/desktop-compatibility-routes.ts +9 -8
- package/src/server/management/desktop-compatibility-runtime-routes.ts +3 -2
- package/src/server/management/desktop-compatibility-settings-routes.ts +5 -3
- package/src/server/management/home-integration-decision.ts +89 -0
- package/src/server/management/integration-routes.ts +208 -81
- package/src/server/management/link-child-ops-routes.ts +145 -43
- package/src/server/management/link-child-preflight.ts +31 -0
- package/src/server/management/link-diagnostics-route.ts +29 -0
- package/src/server/management/link-enrollment.ts +199 -47
- package/src/server/management/link-operation.ts +609 -0
- package/src/server/management/link-recover-route.ts +103 -0
- package/src/server/management/link-route-edit.ts +192 -0
- package/src/server/management/link-route-request.ts +62 -0
- package/src/server/management/link-route-state.ts +83 -7
- package/src/server/management/link-routes.ts +553 -51
- package/src/server/management/link-traffic-status.ts +13 -0
- package/src/server/management/model-routes.ts +122 -36
- package/src/server/management/model-rows.ts +2 -1
- package/src/server/management/native-integration-routes.ts +176 -70
- package/src/server/management/oauth-account-routes.ts +58 -18
- package/src/server/management/protocol-routes.ts +7 -0
- package/src/server/management/provider-completion-test.ts +4 -4
- package/src/server/management/provider-overwrite-carry.ts +14 -0
- package/src/server/management/provider-routes.ts +68 -65
- package/src/server/management/provider-routing-quota.ts +45 -0
- package/src/server/management/route-registry.ts +22 -3
- package/src/server/management/route-to-routes.ts +257 -0
- package/src/server/management/static-catalog-probe.ts +33 -0
- package/src/server/management/system-restart.ts +4 -2
- package/src/server/management/usage-aggregate-cache.ts +101 -66
- package/src/server/management-api.ts +17 -3
- package/src/server/management-auth.ts +13 -1
- package/src/server/messages-native-preflight.ts +27 -4
- package/src/server/messages-native.ts +17 -5
- package/src/server/persistent-gui-session.ts +6 -3
- package/src/server/proxy-liveness.ts +87 -15
- package/src/server/relay-eager.ts +36 -12
- package/src/server/relay.ts +189 -117
- package/src/server/request-body-facts.ts +48 -0
- package/src/server/request-decompress.ts +65 -2
- package/src/server/request-log-account-rotation.ts +6 -0
- package/src/server/request-log-attempt-usage.ts +69 -0
- package/src/server/request-log-cache-affinity.ts +60 -0
- package/src/server/request-log-conversation.ts +45 -2
- package/src/server/request-log-terminal-status.ts +210 -24
- package/src/server/request-log.ts +76 -80
- package/src/server/reserve-capability-startup.ts +32 -0
- package/src/server/responses/adapter-continuation.ts +2 -1
- package/src/server/responses/adapter-delivery.ts +4 -0
- package/src/server/responses/adapter-dispatch.ts +36 -17
- package/src/server/responses/advisor-plan-slot.ts +33 -1
- package/src/server/responses/advisor-question.ts +21 -0
- package/src/server/responses/advisor-slot.ts +10 -3
- package/src/server/responses/antigravity-fault-rotation.ts +24 -0
- package/src/server/responses/antigravity-validation-refusal.ts +43 -6
- package/src/server/responses/bare-namespace-aliases.ts +18 -0
- package/src/server/responses/buffered-sse-json.ts +9 -3
- package/src/server/responses/code-mode-collab-surface.ts +11 -0
- package/src/server/responses/codex-tool-catalog.ts +156 -0
- package/src/server/responses/combo-required-tool-fulfillments.ts +4 -2
- package/src/server/responses/combo-selection-keepalive.ts +254 -0
- package/src/server/responses/combo-stream-preflight.ts +557 -54
- package/src/server/responses/compact.ts +32 -8
- package/src/server/responses/core-codex-account.ts +17 -3
- package/src/server/responses/core-combo-failure.ts +8 -0
- package/src/server/responses/core-combo.ts +55 -12
- package/src/server/responses/core-errors.ts +13 -7
- package/src/server/responses/core-lifetime.ts +3 -2
- package/src/server/responses/core-normalize.ts +3 -1
- package/src/server/responses/core-options.ts +14 -0
- package/src/server/responses/core.ts +23 -0
- package/src/server/responses/estimated-usage.ts +115 -0
- package/src/server/responses/fetch-helpers.ts +15 -10
- package/src/server/responses/jev-model-invoke.ts +7 -2
- package/src/server/responses/native-replay-guard.ts +57 -0
- package/src/server/responses/opengateway-completion-guard.ts +10 -4
- package/src/server/responses/outbound-request-body.ts +29 -0
- package/src/server/responses/passthrough-completion.ts +109 -9
- package/src/server/responses/passthrough-delivery.ts +77 -18
- package/src/server/responses/passthrough-dispatch.ts +77 -73
- package/src/server/responses/policy-fallback.ts +13 -1
- package/src/server/responses/request-prepare.ts +59 -11
- package/src/server/responses/request-transport.ts +10 -3
- package/src/server/responses/reset-replay.ts +46 -2
- package/src/server/responses/run-turn-execution.ts +2 -0
- package/src/server/responses/sidecar-execution.ts +19 -0
- package/src/server/responses/terminal-error-redaction.ts +222 -25
- package/src/server/responses/ws-upstream.ts +2 -2
- package/src/server/sse-payload-rewrite.ts +3 -2
- package/src/server/stop-service-route.ts +85 -0
- package/src/server/system-env-shell.ts +12 -1
- package/src/server/system-env.ts +5 -3
- package/src/server/windows-tcp-drop.ts +21 -4
- package/src/server/workflow-refusal.ts +8 -0
- package/src/server/ws-bridge.ts +191 -12
- package/src/service/cli-windows-stop.ts +189 -0
- package/src/service/cli.ts +41 -12
- package/src/service/desktop-supervision.d.mts +2 -0
- package/src/service/desktop-supervision.mjs +5 -0
- package/src/service/disconnected-client-restart.ts +60 -0
- package/src/service/guarded-manager-target.ts +18 -1
- package/src/service/health.ts +99 -6
- package/src/service/launchd.ts +84 -31
- package/src/service/orchestration.ts +418 -91
- package/src/service/repair.ts +223 -6
- package/src/service/restart-proxy-stop.ts +79 -0
- package/src/service/service-child-ownership.ts +6 -2
- package/src/service/windows-checked-stop-live.ts +467 -0
- package/src/service/windows-checked-stop.ts +435 -0
- package/src/service/windows-command-line-token.ts +23 -0
- package/src/service/windows-guarded-manager.ts +77 -34
- package/src/service/windows-ops.ts +38 -56
- package/src/service/windows-process-evidence.ts +292 -0
- package/src/service/windows-process-native.ts +262 -0
- package/src/service/windows-service-stop.ts +321 -0
- package/src/service/windows-stop-binding.ts +187 -0
- package/src/service/windows-stop-budget.ts +123 -0
- package/src/service/windows-stop-confirm.ts +158 -0
- package/src/service/windows-stop-deadline.ts +56 -0
- package/src/service.ts +2 -2
- package/src/stall-timeout.ts +11 -0
- package/src/types/config.ts +57 -110
- package/src/types/link.ts +20 -0
- package/src/types/multi-agent.ts +13 -0
- package/src/types/provider.ts +60 -2
- package/src/types/request.ts +23 -0
- package/src/types/routed-catalog-defaults.ts +17 -0
- package/src/types/routing-profile.ts +90 -0
- package/src/types.ts +4 -0
- package/src/update/guarded-runtime-ops.ts +5 -7
- package/src/update/index.ts +26 -11
- package/src/update/install-detection.mjs +3 -1
- package/src/update/job-pinned-start.ts +2 -0
- package/src/update/job-windows-restart.ts +19 -32
- package/src/update/job.ts +12 -5
- package/src/update/npm-install.d.mts +11 -0
- package/src/update/npm-install.mjs +44 -7
- package/src/update/orchestrated-entry.ts +1 -1
- package/src/update/orchestrated-node-driver.mjs +3 -2
- package/src/update/orchestrated-pnpm.d.mts +4 -0
- package/src/update/orchestrated-pnpm.mjs +50 -6
- package/src/update/orchestrated-session.d.mts +3 -1
- package/src/update/orchestrated-session.mjs +21 -6
- package/src/update/pnpm-global-install.d.mts +21 -3
- package/src/update/pnpm-global-install.mjs +322 -19
- package/src/update/pnpm-store-layout.mjs +46 -0
- package/src/update/sync-update-mutex.d.mts +3 -2
- package/src/update/sync-update-mutex.mjs +57 -8
- package/src/update/transactional-install.d.mts +3 -1
- package/src/update/transactional-install.mjs +545 -12
- package/src/update/windows-installer-marker.d.mts +1 -0
- package/src/update/windows-installer-marker.mjs +44 -0
- package/src/usage/expected-prices.ts +8 -0
- package/src/usage/filter-index.ts +326 -77
- package/src/usage/ledger-scanner.ts +15 -0
- package/src/usage/link-log-fields.ts +48 -0
- package/src/usage/log.ts +57 -6
- package/src/usage/request-body.ts +37 -0
- package/src/usage/telemetry-contract.ts +1 -0
- package/src/web-search/loop.ts +3 -0
- package/src/web-search/passthrough-bridge.ts +7 -0
- package/gui/dist/assets/App-BpJOzw4F.js +0 -59
- package/gui/dist/assets/brand-mark-DU-0Vdkr.js +0 -1
- package/gui/dist/assets/index-Bbor_y80.js +0 -94
- package/gui/dist/assets/index-DJEkIDAi.css +0 -2
- package/gui/dist/classic/assets/App-BrIlUiLc.js +0 -58
- package/gui/dist/classic/assets/index-DyJHagvT.js +0 -94
- package/gui/dist/classic/assets/quota-summary-CakPe9Ni.js +0 -1
- package/src/server/management/desktop-compatibility-authorization.ts +0 -12
package/src/link/supervisor.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { linkDir, linkKnownHostsPath, linkStorePath } from "./paths";
|
|
|
5
5
|
import { buildTunnelArgv } from "./ssh-argv";
|
|
6
6
|
import { nextLinkConnectionGeneration } from "./relay-auth";
|
|
7
7
|
import { createSshRunner, type SshChild, type SshRunner } from "./ssh-runner";
|
|
8
|
+
import { defaultSshCaptureOpener, openSshCapture, retainSshCapture, type SshCaptureOpener } from "./ssh-capture";
|
|
8
9
|
import type { LinkDiagnosticSink } from "./diagnostics";
|
|
9
10
|
import { createTunnelHistoryBook, exitCause, type TunnelCause, type TunnelHistory } from "./tunnel-history";
|
|
10
11
|
import {
|
|
@@ -17,7 +18,10 @@ import {
|
|
|
17
18
|
type TunnelFailure,
|
|
18
19
|
type TunnelState,
|
|
19
20
|
} from "./tunnel-state";
|
|
20
|
-
import { emptyLinkStore, readLinkStore, type LinkRecord, type LinkStore } from "./store";
|
|
21
|
+
import { effectiveLinkRoutes, emptyLinkStore, readLinkStore, type LinkRecord, type LinkStore } from "./store";
|
|
22
|
+
import { CANDIDATE_KILL_GRACE_MS, createRouteRounds, type LinkRouteStatus } from "./route-rounds";
|
|
23
|
+
import { linkSuspendedByRevoke, operationsJournalPath, revokeSuspensions } from "./operations-journal";
|
|
24
|
+
import type { RouteProofRegistry } from "./route-proof";
|
|
21
25
|
|
|
22
26
|
export type LinkTunnelStatus =
|
|
23
27
|
| {
|
|
@@ -28,6 +32,10 @@ export type LinkTunnelStatus =
|
|
|
28
32
|
orphan?: "orphan-unverified";
|
|
29
33
|
/** Attempts, last change and last failure cause of this tunnel (`tunnel-history.ts`). */
|
|
30
34
|
history?: TunnelHistory;
|
|
35
|
+
/** Links with several routes only (src/link/route-rounds.ts). */
|
|
36
|
+
routes?: LinkRouteStatus;
|
|
37
|
+
/** An unsettled revoke keeps this link down (revokeSuspendsLink); absent otherwise. */
|
|
38
|
+
suspended?: { operationId: string; since: string };
|
|
31
39
|
}
|
|
32
40
|
| {
|
|
33
41
|
linkId: string;
|
|
@@ -46,8 +54,9 @@ export interface LinkSupervisor {
|
|
|
46
54
|
/**
|
|
47
55
|
* Restart one Home-initiated tunnel now: stop its child, forget the backoff or failed state,
|
|
48
56
|
* and spawn a fresh attempt. False when the link is not a Home-initiated link of this store.
|
|
57
|
+
* "suspended" when an unsettled revoke keeps the link down: nothing was stopped or spawned.
|
|
49
58
|
*/
|
|
50
|
-
reconnect(linkId: string): Promise<boolean>;
|
|
59
|
+
reconnect(linkId: string): Promise<boolean | "suspended">;
|
|
51
60
|
status(): readonly LinkTunnelStatus[];
|
|
52
61
|
/** `<runtime instance>:<generation>` of a Home-initiated link's current tunnel; every spawn changes it. */
|
|
53
62
|
connectionGeneration?(linkId: string): string | undefined;
|
|
@@ -55,11 +64,35 @@ export interface LinkSupervisor {
|
|
|
55
64
|
stop(): Promise<void>;
|
|
56
65
|
}
|
|
57
66
|
|
|
67
|
+
interface WatchedProcess {
|
|
68
|
+
pid: number;
|
|
69
|
+
argv: readonly string[];
|
|
70
|
+
startTime?: number;
|
|
71
|
+
/** Set while this supervisor is stopping the process it identified. */
|
|
72
|
+
signal?: { at: number; killed: boolean };
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
interface OrphanWatch {
|
|
76
|
+
/**
|
|
77
|
+
* The link record instance it belongs to; a removed or one-route record retires it. A reload that
|
|
78
|
+
* replaces a flagged multi-route record re-keys it to the new instance as observe-only.
|
|
79
|
+
*/
|
|
80
|
+
instance: string;
|
|
81
|
+
/** The pidfile content the watch last saw; only this file is ever removed by it. */
|
|
82
|
+
file: { pid: number; argv: readonly string[]; startTime?: number };
|
|
83
|
+
/** Every process this pidfile has named since the watch began; the link stays blocked until all are gone. */
|
|
84
|
+
processes: WatchedProcess[];
|
|
85
|
+
/** Set once its record was replaced: it never signals or removes the pidfile, also when it ends. */
|
|
86
|
+
observeOnly?: true;
|
|
87
|
+
}
|
|
88
|
+
|
|
58
89
|
interface Pidfile {
|
|
59
90
|
version: 1;
|
|
60
91
|
linkId: string;
|
|
61
92
|
pid: number;
|
|
62
93
|
argv: string[];
|
|
94
|
+
/** Multi-route tunnels only: the process start time (/proc/<pid>/stat field 22) written right after spawn. */
|
|
95
|
+
startTime?: number;
|
|
63
96
|
}
|
|
64
97
|
|
|
65
98
|
export interface LinkSupervisorDeps {
|
|
@@ -70,8 +103,13 @@ export interface LinkSupervisorDeps {
|
|
|
70
103
|
setTimer?: (callback: () => void, ms: number) => ReturnType<typeof setInterval>;
|
|
71
104
|
clearTimer?: (timer: ReturnType<typeof setInterval>) => void;
|
|
72
105
|
readProcessArgv?: (pid: number) => readonly string[] | null;
|
|
73
|
-
|
|
106
|
+
/** Start time of a live process (Linux /proc/<pid>/stat field 22), null when unreadable or gone. */
|
|
107
|
+
readProcessStartTime?: (pid: number) => number | null;
|
|
108
|
+
killProcess?: (pid: number, signal?: NodeJS.Signals) => void;
|
|
109
|
+
/** false when the pid does not exist, true when it does, null when that cannot be told. */
|
|
110
|
+
processAlive?: (pid: number) => boolean | null;
|
|
74
111
|
pidfileDir?: string;
|
|
112
|
+
/** The link's pidfile, null only when there is none; throws when one exists but cannot be read or parsed. */
|
|
75
113
|
readPidfile?: (path: string) => Pidfile | null;
|
|
76
114
|
writePidfile?: (path: string, pidfile: Pidfile) => void;
|
|
77
115
|
removePidfile?: (path: string) => void;
|
|
@@ -84,6 +122,12 @@ export interface LinkSupervisorDeps {
|
|
|
84
122
|
diagnostic?: LinkDiagnosticSink;
|
|
85
123
|
/** Bounded drain of `diagnostic` at stop. */
|
|
86
124
|
flushDiagnostics?: () => Promise<void>;
|
|
125
|
+
/** Opt-in SSH capture (src/link/ssh-capture.ts); defaults to the one for this link directory. */
|
|
126
|
+
sshCapture?: SshCaptureOpener;
|
|
127
|
+
/** Readiness proofs of multi-route candidates; without it no multi-route link can be proven. */
|
|
128
|
+
routeProof?: RouteProofRegistry;
|
|
129
|
+
/** The operation journal; a link with an unsettled revoke there is not reconnected (src/link/operations-journal.ts). */
|
|
130
|
+
operationsJournalPath?: string;
|
|
87
131
|
}
|
|
88
132
|
|
|
89
133
|
const TIMER_MS = 1_000;
|
|
@@ -114,8 +158,12 @@ function parsePidfile(text: string): Pidfile | null {
|
|
|
114
158
|
const body = raw as Record<string, unknown>;
|
|
115
159
|
if (body.version !== 1 || typeof body.linkId !== "string" || typeof body.pid !== "number"
|
|
116
160
|
|| !Number.isSafeInteger(body.pid) || body.pid < 1 || !Array.isArray(body.argv)
|
|
117
|
-
|| body.argv.some(value => typeof value !== "string")
|
|
118
|
-
|
|
161
|
+
|| body.argv.some(value => typeof value !== "string")
|
|
162
|
+
|| (body.startTime !== undefined && (typeof body.startTime !== "number" || !Number.isSafeInteger(body.startTime) || body.startTime < 0))) return null;
|
|
163
|
+
return {
|
|
164
|
+
version: 1, linkId: body.linkId, pid: body.pid, argv: body.argv as string[],
|
|
165
|
+
...(typeof body.startTime === "number" ? { startTime: body.startTime } : {}),
|
|
166
|
+
};
|
|
119
167
|
} catch {
|
|
120
168
|
return null;
|
|
121
169
|
}
|
|
@@ -131,12 +179,35 @@ function linuxProcessArgv(pid: number): readonly string[] | null {
|
|
|
131
179
|
}
|
|
132
180
|
}
|
|
133
181
|
|
|
134
|
-
|
|
135
|
-
|
|
182
|
+
/** Field 22 of /proc/<pid>/stat, counted after the command name's closing parenthesis. */
|
|
183
|
+
function linuxProcessStartTime(pid: number): number | null {
|
|
184
|
+
try {
|
|
185
|
+
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
186
|
+
const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
|
|
187
|
+
const value = Number(fields[19]);
|
|
188
|
+
return Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
189
|
+
} catch {
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
function defaultProcessAlive(pid: number): boolean | null {
|
|
195
|
+
try {
|
|
196
|
+
process.kill(pid, 0);
|
|
197
|
+
return true;
|
|
198
|
+
} catch (error) {
|
|
199
|
+
const code = (error as NodeJS.ErrnoException).code;
|
|
200
|
+
return code === "ESRCH" ? false : code === "EPERM" ? true : null;
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function defaultKillProcess(pid: number, signal: NodeJS.Signals = "SIGTERM"): void {
|
|
205
|
+
try { process.kill(pid, signal); } catch { /* the process may already have exited */ }
|
|
136
206
|
}
|
|
137
207
|
|
|
138
208
|
export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervisor {
|
|
139
209
|
const storePath = linkStorePath();
|
|
210
|
+
const journalPath = deps.operationsJournalPath ?? operationsJournalPath();
|
|
140
211
|
const storeDir = deps.pidfileDir ?? linkDir();
|
|
141
212
|
const readStore = deps.readStore ?? (() => readLinkStore(storePath));
|
|
142
213
|
const runner = deps.runner ?? createSshRunner();
|
|
@@ -145,10 +216,23 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
145
216
|
const clearTimer = deps.clearTimer ?? ((timer: ReturnType<typeof setInterval>) => clearInterval(timer));
|
|
146
217
|
const platform = deps.platform ?? process.platform;
|
|
147
218
|
const readProcessArgv = deps.readProcessArgv ?? (platform === "linux" ? linuxProcessArgv : () => null);
|
|
219
|
+
const readProcessStartTime = deps.readProcessStartTime ?? (platform === "linux" ? linuxProcessStartTime : () => null);
|
|
148
220
|
const killProcess = deps.killProcess ?? defaultKillProcess;
|
|
149
|
-
const
|
|
150
|
-
|
|
221
|
+
const processAlive = deps.processAlive ?? defaultProcessAlive;
|
|
222
|
+
const loadPidfile = deps.readPidfile ?? ((path: string): Pidfile | null => {
|
|
223
|
+
let text: string;
|
|
224
|
+
try { text = readFileSync(path, "utf8"); } catch (error) {
|
|
225
|
+
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
|
|
226
|
+
throw error;
|
|
227
|
+
}
|
|
228
|
+
const pidfile = parsePidfile(text);
|
|
229
|
+
if (pidfile === null) throw new Error("link pidfile unreadable");
|
|
230
|
+
return pidfile;
|
|
151
231
|
});
|
|
232
|
+
/** Every reader except the orphan-flag lift treats an unreadable pidfile as none, as before. */
|
|
233
|
+
const readPidfile = (path: string): Pidfile | null => {
|
|
234
|
+
try { return loadPidfile(path); } catch { return null; }
|
|
235
|
+
};
|
|
152
236
|
const writePidfile = deps.writePidfile ?? ((path: string, pidfile: Pidfile) => {
|
|
153
237
|
mkdirSync(storeDir, { recursive: true, mode: 0o700 });
|
|
154
238
|
atomicWriteFile(path, `${JSON.stringify(pidfile)}\n`);
|
|
@@ -159,6 +243,9 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
159
243
|
}
|
|
160
244
|
});
|
|
161
245
|
const warn = deps.warn ?? ((message: string) => console.warn(message));
|
|
246
|
+
const sshCapture = deps.sshCapture ?? defaultSshCaptureOpener(storeDir);
|
|
247
|
+
// Runs at server start even with no links, so an earlier process's capture files still expire.
|
|
248
|
+
if (!deps.sshCapture) retainSshCapture(storeDir);
|
|
162
249
|
|
|
163
250
|
let store: LinkStore = emptyLinkStore();
|
|
164
251
|
let started = false;
|
|
@@ -167,19 +254,32 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
167
254
|
let lifecycleRevision = 0;
|
|
168
255
|
let timer: ReturnType<typeof setInterval> | undefined;
|
|
169
256
|
const states = new Map<string, TunnelState>();
|
|
170
|
-
|
|
257
|
+
/** stopAt/killed/warned: a failed link's child the tick stopped, kept until its exit is seen. */
|
|
258
|
+
const children = new Map<string, { child: SshChild; argv: readonly string[]; spawnedAt: number; stopAt?: number; killed?: boolean; warned?: boolean }>();
|
|
259
|
+
/**
|
|
260
|
+
* Tunnel children (one-route tunnels, multi-route candidates and proven tunnels) a stop sent SIGTERM
|
|
261
|
+
* and is awaiting; the tick sends SIGKILL after the grace.
|
|
262
|
+
*/
|
|
263
|
+
const draining = new Set<{ linkId: string; child: SshChild; stopAt: number; killed: boolean; warned: boolean }>();
|
|
171
264
|
/** Per-link generation, advanced by spawns and explicit stop/removal reservations. */
|
|
172
265
|
const generations = new Map<string, number>();
|
|
173
266
|
/** The process-unique connection name of each link's current ssh child, minted at spawn. */
|
|
174
267
|
const connectionNames = new Map<string, string>();
|
|
175
268
|
const linkRevisions = new Map<string, number>();
|
|
176
269
|
/** Reconnects in flight, so overlapping requests for one link restart it once. */
|
|
177
|
-
const reconnects = new Map<string, Promise<boolean>>();
|
|
270
|
+
const reconnects = new Map<string, Promise<boolean | "suspended">>();
|
|
178
271
|
const holds = new Map<string, number>();
|
|
179
272
|
const orphanUnverified = new Set<string>();
|
|
180
273
|
/** The generation of the child whose exit is still being classified, per link. */
|
|
181
274
|
const exiting = new Map<string, number>();
|
|
182
275
|
const recordInstances = new Map<string, string>();
|
|
276
|
+
/** Multi-route links blocked by an earlier tunnel, watched until it is gone (src/link/route-rounds.ts relies on it). */
|
|
277
|
+
const orphanWatches = new Map<string, OrphanWatch>();
|
|
278
|
+
/**
|
|
279
|
+
* Multi-route links blocked by a pidfile that exists but cannot be read or parsed, read again every
|
|
280
|
+
* tick; false once a reload replaced the record, so the watch that file may become only observes.
|
|
281
|
+
*/
|
|
282
|
+
const unreadablePidfiles = new Map<string, boolean>();
|
|
183
283
|
const history = createTunnelHistoryBook({
|
|
184
284
|
diagnostic: deps.diagnostic,
|
|
185
285
|
tunnelPort: linkId => store.links.find(link => link.id === linkId)?.tunnelPort,
|
|
@@ -210,16 +310,23 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
210
310
|
record.apiKeyId,
|
|
211
311
|
record.createdAt,
|
|
212
312
|
listenerPort,
|
|
313
|
+
// How ssh reaches the alias: an edit restarts the tunnel on the new route instead of keeping the old one. exclusive
|
|
314
|
+
// alone changes no argv (it only locks the route list), so it is not part of the instance.
|
|
315
|
+
record.explicitRoute ?? null,
|
|
213
316
|
]);
|
|
214
317
|
|
|
215
318
|
const conditionalRemovePidfile = (linkId: string, pid: number): void => {
|
|
216
319
|
const path = pidfilePath(linkId);
|
|
217
320
|
const current = readPidfile(path);
|
|
218
|
-
if (current?.linkId === linkId && current.pid === pid)
|
|
321
|
+
if (current?.linkId === linkId && current.pid === pid) dropStalePidfile(linkId);
|
|
219
322
|
};
|
|
220
323
|
|
|
221
324
|
const reapOrphans = (): void => {
|
|
222
325
|
for (const record of store.links) {
|
|
326
|
+
if (rounds.isMultiRoute(record)) {
|
|
327
|
+
recheckPidfile(record);
|
|
328
|
+
continue;
|
|
329
|
+
}
|
|
223
330
|
const pidfile = readPidfile(pidfilePath(record.id));
|
|
224
331
|
if (!pidfile || pidfile.linkId !== record.id) continue;
|
|
225
332
|
// macOS deliberately has no exact argv source here. A ps rendering is not an identity proof.
|
|
@@ -229,13 +336,162 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
229
336
|
}
|
|
230
337
|
const actualArgv = readProcessArgv(pidfile.pid);
|
|
231
338
|
if (actualArgv && sameArgv(actualArgv, pidfile.argv)) {
|
|
232
|
-
try { killProcess(pidfile.pid); } finally {
|
|
339
|
+
try { killProcess(pidfile.pid); } finally { dropStalePidfile(record.id); }
|
|
233
340
|
} else {
|
|
234
341
|
orphanUnverified.add(record.id);
|
|
235
342
|
}
|
|
236
343
|
}
|
|
237
344
|
};
|
|
238
345
|
|
|
346
|
+
/**
|
|
347
|
+
* The pidfile's own process, identified by pid, exact argv and start time. "gone" and "different"
|
|
348
|
+
* both mean the recorded process has exited (a different one may reuse the pid); "unreadable" means
|
|
349
|
+
* a live pid whose identity could not be read, which never counts as gone.
|
|
350
|
+
*/
|
|
351
|
+
const processIdentity = (pid: number, argv: readonly string[], startTime: number): "same" | "gone" | "different" | "unreadable" => {
|
|
352
|
+
const actualArgv = readProcessArgv(pid);
|
|
353
|
+
const actualStart = readProcessStartTime(pid);
|
|
354
|
+
if (actualArgv === null || actualStart === null) return processAlive(pid) === false ? "gone" : "unreadable";
|
|
355
|
+
return sameArgv(actualArgv, argv) && actualStart === startTime ? "same" : "different";
|
|
356
|
+
};
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* Remove a link's pidfile once its process is gone, or has been sent SIGTERM (an earlier one-route
|
|
360
|
+
* ssh matched at startup), or once this process's own child has exited or been stopped. A removal
|
|
361
|
+
* that fails (a read-only directory, a sharing lock) is warned and never stops the caller: a tick goes on
|
|
362
|
+
* to every other link, an exit is still classified, a stop or reload completes, and the block it
|
|
363
|
+
* was lifting stays lifted. The file left behind is mostly harmless: on Linux with a start time the
|
|
364
|
+
* next start sees its process gone; without a start time, a pid reused by another process blocks a
|
|
365
|
+
* multi-route link at the next start until that process exits.
|
|
366
|
+
*/
|
|
367
|
+
const dropStalePidfile = (linkId: string): void => {
|
|
368
|
+
try { removePidfile(pidfilePath(linkId)); } catch { warn(`[link] stale pidfile could not be removed link=${linkId}`); }
|
|
369
|
+
};
|
|
370
|
+
|
|
371
|
+
/**
|
|
372
|
+
* A multi-route link never starts a candidate while an earlier tunnel of it may live: the readiness
|
|
373
|
+
* proof's attribution relies on that. Only a process proven to be the pidfile's own (pid, argv and
|
|
374
|
+
* the start time written at spawn) is signalled, and its identity is re-read before every signal.
|
|
375
|
+
* Every blocked pidfile stays watched without signals, and the block lifts once the process is shown
|
|
376
|
+
* gone (or its pid now names another process); the link's first round then starts.
|
|
377
|
+
* With act false (a reload that replaced the record) nothing is signalled or removed: the new
|
|
378
|
+
* instance's watch only observes, as no watch acts for a replaced instance.
|
|
379
|
+
*/
|
|
380
|
+
const reapMultiRouteOrphan = (record: LinkRecord, pidfile: Pidfile, act = true): void => {
|
|
381
|
+
const watched: WatchedProcess = { pid: pidfile.pid, argv: pidfile.argv, ...(pidfile.startTime === undefined ? {} : { startTime: pidfile.startTime }) };
|
|
382
|
+
if (processGone(watched)) { if (act) dropStalePidfile(record.id); return; }
|
|
383
|
+
orphanUnverified.add(record.id);
|
|
384
|
+
const identity = platform === "linux" && pidfile.startTime !== undefined ? processIdentity(pidfile.pid, pidfile.argv, pidfile.startTime) : "unreadable";
|
|
385
|
+
if (act && identity === "same") {
|
|
386
|
+
killProcess(pidfile.pid, "SIGTERM");
|
|
387
|
+
watched.signal = { at: now(), killed: false };
|
|
388
|
+
}
|
|
389
|
+
orphanWatches.set(record.id, {
|
|
390
|
+
instance: recordInstance(record, store.listenerPort), file: fileIdentity(pidfile), processes: [watched], ...(act ? {} : { observeOnly: true as const }),
|
|
391
|
+
});
|
|
392
|
+
};
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* A multi-route link's block, settled from its pidfile: the earlier tunnel it names is watched, no
|
|
396
|
+
* pidfile (ENOENT) lifts the block, and one that exists but cannot be read or parsed keeps it, since
|
|
397
|
+
* it may name the ssh still holding the port. That file is read again every tick, so making it
|
|
398
|
+
* readable or deleting it settles the block without a restart.
|
|
399
|
+
*/
|
|
400
|
+
const recheckPidfile = (record: LinkRecord, act = true): void => {
|
|
401
|
+
let file: Pidfile | null;
|
|
402
|
+
try { file = loadPidfile(pidfilePath(record.id)); } catch {
|
|
403
|
+
if (!unreadablePidfiles.has(record.id)) warn(`[link] pidfile unreadable; link stays blocked link=${record.id}`);
|
|
404
|
+
unreadablePidfiles.set(record.id, (unreadablePidfiles.get(record.id) ?? true) && act);
|
|
405
|
+
orphanUnverified.add(record.id);
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
408
|
+
unreadablePidfiles.delete(record.id);
|
|
409
|
+
if (file !== null && file.linkId === record.id) reapMultiRouteOrphan(record, file, act);
|
|
410
|
+
if (!orphanWatches.has(record.id)) orphanUnverified.delete(record.id);
|
|
411
|
+
};
|
|
412
|
+
|
|
413
|
+
const fileIdentity = (pidfile: Pidfile): OrphanWatch["file"] => ({ pid: pidfile.pid, argv: pidfile.argv, ...(pidfile.startTime === undefined ? {} : { startTime: pidfile.startTime }) });
|
|
414
|
+
|
|
415
|
+
/** True when the process has exited (or, with a recorded start time on Linux, its pid names another one). */
|
|
416
|
+
const processGone = (watched: WatchedProcess): boolean => {
|
|
417
|
+
if (watched.startTime !== undefined && platform === "linux") {
|
|
418
|
+
const identity = processIdentity(watched.pid, watched.argv, watched.startTime);
|
|
419
|
+
return identity === "gone" || identity === "different";
|
|
420
|
+
}
|
|
421
|
+
return processAlive(watched.pid) === false;
|
|
422
|
+
};
|
|
423
|
+
|
|
424
|
+
const sameFile = (file: Pidfile, seen: OrphanWatch["file"]): boolean =>
|
|
425
|
+
file.pid === seen.pid && sameArgv(file.argv, seen.argv) && file.startTime === seen.startTime;
|
|
426
|
+
|
|
427
|
+
/** A watch belongs to one record instance; it never acts for a removed, replaced or one-route link. */
|
|
428
|
+
const watchCurrent = (linkId: string, watch: OrphanWatch): boolean => {
|
|
429
|
+
const record = store.links.find(link => link.id === linkId);
|
|
430
|
+
return record !== undefined && rounds.isMultiRoute(record) && recordInstance(record, store.listenerPort) === watch.instance;
|
|
431
|
+
};
|
|
432
|
+
|
|
433
|
+
const tickOrphanWatches = (current: number): void => {
|
|
434
|
+
for (const [linkId, watch] of [...orphanWatches]) {
|
|
435
|
+
if (!watchCurrent(linkId, watch)) { orphanWatches.delete(linkId); continue; }
|
|
436
|
+
// A pidfile that exists but cannot be read may name another tunnel now: it never ends the watch.
|
|
437
|
+
let file: Pidfile | null = null;
|
|
438
|
+
let unreadable = false;
|
|
439
|
+
try { file = loadPidfile(pidfilePath(linkId)); } catch { unreadable = true; }
|
|
440
|
+
if (file !== null && file.linkId === linkId && !sameFile(file, watch.file)) {
|
|
441
|
+
// Another owner replaced the pidfile: nothing is signalled any more, and the replacement joins
|
|
442
|
+
// the watched processes; the link stays blocked until every one of them is gone.
|
|
443
|
+
for (const watched of watch.processes) delete watched.signal;
|
|
444
|
+
watch.file = fileIdentity(file);
|
|
445
|
+
// Identity is pid, argv and start time together: a replacement that reuses an old pid is a new process.
|
|
446
|
+
if (!watch.processes.some(watched => watched.pid === file.pid && sameArgv(watched.argv, file.argv) && watched.startTime === file.startTime)) {
|
|
447
|
+
watch.processes.push({ pid: file.pid, argv: file.argv, ...(file.startTime === undefined ? {} : { startTime: file.startTime }) });
|
|
448
|
+
}
|
|
449
|
+
}
|
|
450
|
+
watch.processes = watch.processes.filter(watched => !processGone(watched));
|
|
451
|
+
if (watch.processes.length === 0) {
|
|
452
|
+
orphanWatches.delete(linkId);
|
|
453
|
+
if (unreadable) {
|
|
454
|
+
// Every watched process is gone, but the file cannot say what it names: stay blocked and read
|
|
455
|
+
// it again every tick. Whatever it names once readable is a replacement to this watch, so it
|
|
456
|
+
// is only observed: no signal, no removal.
|
|
457
|
+
if (!unreadablePidfiles.has(linkId)) warn(`[link] pidfile unreadable; link stays blocked link=${linkId}`);
|
|
458
|
+
unreadablePidfiles.set(linkId, false);
|
|
459
|
+
continue;
|
|
460
|
+
}
|
|
461
|
+
orphanUnverified.delete(linkId);
|
|
462
|
+
if (!watch.observeOnly && file !== null && file.linkId === linkId && sameFile(file, watch.file)) dropStalePidfile(linkId);
|
|
463
|
+
// The link was waiting for this: start its first round now.
|
|
464
|
+
const record = store.links.find(link => link.id === linkId);
|
|
465
|
+
if (record && !children.has(linkId) && (states.get(linkId) ?? IDLE).kind === "idle") spawnFor(record);
|
|
466
|
+
continue;
|
|
467
|
+
}
|
|
468
|
+
for (const watched of watch.processes) {
|
|
469
|
+
const signal = watched.signal;
|
|
470
|
+
if (!signal || watched.startTime === undefined || watch.observeOnly) continue;
|
|
471
|
+
if (file === null || processIdentity(watched.pid, watched.argv, watched.startTime) !== "same") {
|
|
472
|
+
// Cannot tell it is ours any more: no further signal; the watch goes on.
|
|
473
|
+
delete watched.signal;
|
|
474
|
+
} else if (!signal.killed && current - signal.at >= 2_000) {
|
|
475
|
+
signal.killed = true;
|
|
476
|
+
killProcess(watched.pid, "SIGKILL");
|
|
477
|
+
} else if (current - signal.at >= 4_000) {
|
|
478
|
+
// Still there: nothing is signalled again; the link stays blocked until the watch sees it gone.
|
|
479
|
+
delete watched.signal;
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
for (const [linkId, act] of [...unreadablePidfiles]) {
|
|
484
|
+
const record = store.links.find(link => link.id === linkId);
|
|
485
|
+
if (!record || !rounds.isMultiRoute(record) || !orphanUnverified.has(linkId) || orphanWatches.has(linkId)) {
|
|
486
|
+
unreadablePidfiles.delete(linkId);
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
489
|
+
recheckPidfile(record, act);
|
|
490
|
+
// Nothing is left to wait for: start the link's first round now, as a watch that ends does.
|
|
491
|
+
if (!orphanUnverified.has(linkId) && !children.has(linkId) && (states.get(linkId) ?? IDLE).kind === "idle") spawnFor(record);
|
|
492
|
+
}
|
|
493
|
+
};
|
|
494
|
+
|
|
239
495
|
/** Every state change goes through here, so the history sees each transition and its cause. */
|
|
240
496
|
const put = (linkId: string, next: TunnelState, cause: TunnelCause | null = null): TunnelState => {
|
|
241
497
|
const before = states.get(linkId) ?? IDLE;
|
|
@@ -252,8 +508,98 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
252
508
|
put(linkId, failedSpawnTunnel(prior, reason, now(), HOME_TUNNEL_RETRY_POLICY), cause);
|
|
253
509
|
};
|
|
254
510
|
|
|
511
|
+
/** Every tunnel child's exit, one-route or multi-route, goes through here. */
|
|
512
|
+
const watchExit = (linkId: string, child: SshChild, generation: number): void => {
|
|
513
|
+
void child.exited.then(async () => {
|
|
514
|
+
if (children.get(linkId)?.child !== child) return;
|
|
515
|
+
children.delete(linkId);
|
|
516
|
+
conditionalRemovePidfile(linkId, child.pid);
|
|
517
|
+
if (stopping) return;
|
|
518
|
+
exiting.set(linkId, generation);
|
|
519
|
+
try {
|
|
520
|
+
let stderr = "";
|
|
521
|
+
// An unreadable stderr (the runner rejects it past its size cap) still reports the exit.
|
|
522
|
+
try { stderr = child.stderr ? await child.stderr : ""; } catch { /* classified as unknown */ }
|
|
523
|
+
// A reconnect may have started a fresh child meanwhile; its own lifecycle owns the state.
|
|
524
|
+
if (generations.get(linkId) !== generation) return;
|
|
525
|
+
if (rounds.exited(linkId, child, stderr)) return;
|
|
526
|
+
const stderrClass = classifySshStderr(stderr);
|
|
527
|
+
setEvent(linkId, { type: "exit", now: now(), stderrClass }, exitCause(stderrClass));
|
|
528
|
+
} finally {
|
|
529
|
+
if (exiting.get(linkId) === generation) exiting.delete(linkId);
|
|
530
|
+
}
|
|
531
|
+
}).catch(() => {
|
|
532
|
+
if (children.get(linkId)?.child !== child) return;
|
|
533
|
+
children.delete(linkId);
|
|
534
|
+
conditionalRemovePidfile(linkId, child.pid);
|
|
535
|
+
if (stopping) return;
|
|
536
|
+
if (rounds.exited(linkId, child, "")) return;
|
|
537
|
+
setEvent(linkId, { type: "exit", now: now(), stderrClass: "network" }, "network");
|
|
538
|
+
});
|
|
539
|
+
};
|
|
540
|
+
|
|
541
|
+
const rounds = createRouteRounds({
|
|
542
|
+
runner,
|
|
543
|
+
now,
|
|
544
|
+
random: () => (deps.random ?? Math.random)(),
|
|
545
|
+
knownHostsFile: () => linkKnownHostsPath(),
|
|
546
|
+
listenerPort: () => store.listenerPort,
|
|
547
|
+
record: linkId => store.links.find(link => link.id === linkId),
|
|
548
|
+
blocked: linkId => stopping || holds.has(linkId) || orphanUnverified.has(linkId),
|
|
549
|
+
busy: () => lifecycleFlight !== undefined,
|
|
550
|
+
ownsChild: (linkId, child) => children.get(linkId)?.child === child,
|
|
551
|
+
spawnTunnel(record, destination, stderrLine) {
|
|
552
|
+
if (store.listenerPort === null) throw new Error("link listener unavailable");
|
|
553
|
+
const argv = buildTunnelArgv({
|
|
554
|
+
alias: record.alias,
|
|
555
|
+
destination,
|
|
556
|
+
direction: "R",
|
|
557
|
+
bindPort: record.tunnelPort,
|
|
558
|
+
targetPort: store.listenerPort,
|
|
559
|
+
knownHostsFile: linkKnownHostsPath(),
|
|
560
|
+
// OpenSSH confirms the granted remote listener at DEBUG1; the tap keeps today's classifier lines only.
|
|
561
|
+
logLevel: "DEBUG1",
|
|
562
|
+
// A record with a route list cannot carry an explicit route (store.ts rejects it). If one ever arrives here,
|
|
563
|
+
// buildTunnelArgv refuses an explicit route for any destination but the alias instead of ignoring it.
|
|
564
|
+
route: record.explicitRoute ?? null,
|
|
565
|
+
});
|
|
566
|
+
const child = runner.spawnTunnel(argv, { stderrLine });
|
|
567
|
+
const generation = (generations.get(record.id) ?? 0) + 1;
|
|
568
|
+
generations.set(record.id, generation);
|
|
569
|
+
const connection = nextLinkConnectionGeneration();
|
|
570
|
+
connectionNames.set(record.id, connection);
|
|
571
|
+
exiting.delete(record.id);
|
|
572
|
+
children.set(record.id, { child, argv, spawnedAt: now() });
|
|
573
|
+
// Owned and watched before anything else can fail, so a failed pidfile write never leaves an untracked tunnel.
|
|
574
|
+
watchExit(record.id, child, generation);
|
|
575
|
+
try {
|
|
576
|
+
const startTime = readProcessStartTime(child.pid);
|
|
577
|
+
writePidfile(pidfilePath(record.id), { version: 1, linkId: record.id, pid: child.pid, argv, ...(startTime === null ? {} : { startTime }) });
|
|
578
|
+
} catch {
|
|
579
|
+
return { child, generation: connection, setupFailed: true };
|
|
580
|
+
}
|
|
581
|
+
return { child, generation: connection };
|
|
582
|
+
},
|
|
583
|
+
captureTap(linkId) {
|
|
584
|
+
const capture = openSshCapture(sshCapture, "home", linkId);
|
|
585
|
+
return capture ? line => capture.stderrLine(line) : undefined;
|
|
586
|
+
},
|
|
587
|
+
setEvent: (linkId, event, cause) => setEvent(linkId, event, cause ?? null),
|
|
588
|
+
deferRetry(linkId, at) {
|
|
589
|
+
const state = states.get(linkId);
|
|
590
|
+
if (state && (state.kind === "reconnecting" || state.kind === "failed") && state.retryAt !== undefined && state.retryAt < at) {
|
|
591
|
+
states.set(linkId, { ...state, retryAt: at });
|
|
592
|
+
}
|
|
593
|
+
},
|
|
594
|
+
attempted: linkId => history.spawned(linkId),
|
|
595
|
+
...(deps.routeProof ? { routeProof: deps.routeProof } : {}),
|
|
596
|
+
warn,
|
|
597
|
+
});
|
|
598
|
+
|
|
255
599
|
const spawnFor = (record: LinkRecord): void => {
|
|
256
600
|
if (stopping || holds.has(record.id) || record.direction !== "hub-initiated" || children.has(record.id)) return;
|
|
601
|
+
// A revoke that reached the Child's cleanup and stopped unknown may have purged it: no reconnect until resolved.
|
|
602
|
+
if (linkSuspendedByRevoke(journalPath, record.id)) return;
|
|
257
603
|
// Only a fresh link or a due attempt spawns. A reload must neither skip the backoff nor start
|
|
258
604
|
// a second child while an exited one is still reporting its stderr.
|
|
259
605
|
const prior = states.get(record.id);
|
|
@@ -263,16 +609,22 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
263
609
|
failAttempt(record.id, prior, "forward", "listener_unavailable");
|
|
264
610
|
return;
|
|
265
611
|
}
|
|
612
|
+
// Several routes: an ordered, proven selection round (src/link/route-rounds.ts) instead.
|
|
613
|
+
if (rounds.begin(record)) return;
|
|
266
614
|
let argv: string[];
|
|
267
615
|
try {
|
|
616
|
+
// Off unless the operator opened a capture window on this machine; then only the level changes.
|
|
617
|
+
const capture = openSshCapture(sshCapture, "home", record.id);
|
|
268
618
|
argv = buildTunnelArgv({
|
|
269
619
|
alias: record.alias,
|
|
270
620
|
direction: "R",
|
|
271
621
|
bindPort: record.tunnelPort,
|
|
272
622
|
targetPort: store.listenerPort,
|
|
273
623
|
knownHostsFile: linkKnownHostsPath(),
|
|
624
|
+
...(capture ? { logLevel: capture.logLevel } : {}),
|
|
625
|
+
route: record.explicitRoute ?? null,
|
|
274
626
|
});
|
|
275
|
-
const child = runner.spawnTunnel(argv);
|
|
627
|
+
const child = capture ? runner.spawnTunnel(argv, { stderrLine: capture.stderrLine }) : runner.spawnTunnel(argv);
|
|
276
628
|
const generation = (generations.get(record.id) ?? 0) + 1;
|
|
277
629
|
generations.set(record.id, generation);
|
|
278
630
|
connectionNames.set(record.id, nextLinkConnectionGeneration());
|
|
@@ -282,38 +634,36 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
282
634
|
orphanUnverified.delete(record.id);
|
|
283
635
|
history.spawned(record.id);
|
|
284
636
|
setEvent(record.id, { type: "spawn", now: now() });
|
|
637
|
+
// Watched before the pidfile write can fail, so a child whose write failed still has its exit seen.
|
|
638
|
+
watchExit(record.id, child, generation);
|
|
285
639
|
writePidfile(pidfilePath(record.id), { version: 1, linkId: record.id, pid: child.pid, argv });
|
|
286
|
-
void child.exited.then(async () => {
|
|
287
|
-
if (children.get(record.id)?.child !== child) return;
|
|
288
|
-
children.delete(record.id);
|
|
289
|
-
conditionalRemovePidfile(record.id, child.pid);
|
|
290
|
-
if (stopping) return;
|
|
291
|
-
exiting.set(record.id, generation);
|
|
292
|
-
try {
|
|
293
|
-
let stderr = "";
|
|
294
|
-
// An unreadable stderr (the runner rejects it past its size cap) still reports the exit.
|
|
295
|
-
try { stderr = child.stderr ? await child.stderr : ""; } catch { /* classified as unknown */ }
|
|
296
|
-
// A reconnect may have started a fresh child meanwhile; its own lifecycle owns the state.
|
|
297
|
-
if (generations.get(record.id) !== generation) return;
|
|
298
|
-
const stderrClass = classifySshStderr(stderr);
|
|
299
|
-
setEvent(record.id, { type: "exit", now: now(), stderrClass }, exitCause(stderrClass));
|
|
300
|
-
} finally {
|
|
301
|
-
if (exiting.get(record.id) === generation) exiting.delete(record.id);
|
|
302
|
-
}
|
|
303
|
-
}).catch(() => {
|
|
304
|
-
if (children.get(record.id)?.child !== child) return;
|
|
305
|
-
children.delete(record.id);
|
|
306
|
-
conditionalRemovePidfile(record.id, child.pid);
|
|
307
|
-
if (!stopping) setEvent(record.id, { type: "exit", now: now(), stderrClass: "network" }, "network");
|
|
308
|
-
});
|
|
309
640
|
} catch {
|
|
310
641
|
failAttempt(record.id, prior, "forward", "spawn_failed");
|
|
311
642
|
}
|
|
312
643
|
};
|
|
313
644
|
|
|
314
645
|
const tick = (): void => {
|
|
315
|
-
|
|
646
|
+
// The SIGKILL for any tunnel child a stop is draining runs even while lifecycle work is pending or the
|
|
647
|
+
// supervisor shuts down: no reload, removal, reconnect or shutdown waits on an ssh that ignores SIGTERM.
|
|
648
|
+
// (A candidate route-rounds already stopped gets a fresh stop time from stopChild; one it already halted
|
|
649
|
+
// with SIGKILL gets a second, harmless one on the same unreaped process.)
|
|
650
|
+
for (const drain of draining) {
|
|
651
|
+
const age = now() - drain.stopAt;
|
|
652
|
+
if (!drain.killed && age >= CANDIDATE_KILL_GRACE_MS) {
|
|
653
|
+
drain.killed = true;
|
|
654
|
+
try { drain.child.kill("SIGKILL"); } catch { /* already gone */ }
|
|
655
|
+
} else if (drain.killed && !drain.warned && age >= 2 * CANDIDATE_KILL_GRACE_MS) {
|
|
656
|
+
// Its exit is still unseen, so the stop holds the lifecycle queue: say so once.
|
|
657
|
+
drain.warned = true;
|
|
658
|
+
warn(`[link] tunnel exit unconfirmed link=${drain.linkId}`);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
if (stopping) return;
|
|
662
|
+
// Route deadlines run even while lifecycle work is pending, so a stuck reload cannot stretch a probe.
|
|
663
|
+
for (const record of store.links) if (rounds.isMultiRoute(record) && !holds.has(record.id)) rounds.tick(record.id);
|
|
664
|
+
if (lifecycleFlight) return;
|
|
316
665
|
const current = now();
|
|
666
|
+
tickOrphanWatches(current);
|
|
317
667
|
for (const record of store.links) {
|
|
318
668
|
if (record.direction !== "hub-initiated") continue;
|
|
319
669
|
if (holds.has(record.id)) continue;
|
|
@@ -322,6 +672,16 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
322
672
|
if (exiting.has(record.id)) continue;
|
|
323
673
|
const before = states.get(record.id) ?? IDLE;
|
|
324
674
|
const child = children.get(record.id);
|
|
675
|
+
if (rounds.isMultiRoute(record)) {
|
|
676
|
+
// Promotion comes only from a proven candidate; the outage clock still runs across rounds.
|
|
677
|
+
const next = setEvent(record.id, { type: "tick", now: current }, "timeout");
|
|
678
|
+
if (next.kind === "failed" && !next.inFlight && (rounds.inRound(record.id) || child)) {
|
|
679
|
+
states.set(record.id, { ...next, inFlight: true });
|
|
680
|
+
} else if (dueForSpawn(next, current)) {
|
|
681
|
+
spawnFor(record);
|
|
682
|
+
}
|
|
683
|
+
continue;
|
|
684
|
+
}
|
|
325
685
|
// Promotion is checked before the outage limit, so a child that has outlived its grace by the
|
|
326
686
|
// time this tick runs (at the limit, or late after a sleep) is promoted rather than killed.
|
|
327
687
|
// An idle Child may send no catalog request for hours; for a retry, staying up past the
|
|
@@ -337,9 +697,20 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
337
697
|
// the failed state's retry instead of being killed with a link that may already be back.
|
|
338
698
|
states.set(record.id, { ...next, inFlight: true });
|
|
339
699
|
} else if (next.kind === "failed" && !next.inFlight && child) {
|
|
340
|
-
child
|
|
341
|
-
|
|
342
|
-
|
|
700
|
+
// A failed link with a running child (its pidfile write failed): stop it, SIGKILL after the grace,
|
|
701
|
+
// and keep it until its exit is seen, as a stopped candidate is. watchExit then forgets it and
|
|
702
|
+
// removes the pidfile naming it; until then no retry and no round starts past it.
|
|
703
|
+
if (child.stopAt === undefined) {
|
|
704
|
+
child.stopAt = current;
|
|
705
|
+
try { child.child.kill("SIGTERM"); } catch { /* already gone */ }
|
|
706
|
+
} else if (!child.killed && current - child.stopAt >= CANDIDATE_KILL_GRACE_MS) {
|
|
707
|
+
child.killed = true;
|
|
708
|
+
try { child.child.kill("SIGKILL"); } catch { /* already gone */ }
|
|
709
|
+
} else if (child.killed && !child.warned && current - child.stopAt >= 2 * CANDIDATE_KILL_GRACE_MS) {
|
|
710
|
+
// Its exit was never seen: the link stays failed and owned, as a halted candidate does, and says so once.
|
|
711
|
+
child.warned = true;
|
|
712
|
+
warn(`[link] tunnel exit unconfirmed link=${record.id}`);
|
|
713
|
+
}
|
|
343
714
|
} else if (dueForSpawn(next, current)) {
|
|
344
715
|
spawnFor(record);
|
|
345
716
|
}
|
|
@@ -378,6 +749,7 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
378
749
|
};
|
|
379
750
|
|
|
380
751
|
const stopChild = async (linkId: string): Promise<void> => {
|
|
752
|
+
rounds.cancel(linkId);
|
|
381
753
|
const current = children.get(linkId);
|
|
382
754
|
if (!current) {
|
|
383
755
|
put(linkId, IDLE);
|
|
@@ -386,7 +758,10 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
386
758
|
const generation = generations.get(linkId);
|
|
387
759
|
children.delete(linkId);
|
|
388
760
|
current.child.kill("SIGTERM");
|
|
389
|
-
|
|
761
|
+
// A child the tick already stopped keeps its stop time and its SIGKILL.
|
|
762
|
+
const drain = { linkId, child: current.child, stopAt: current.stopAt ?? now(), killed: current.killed === true, warned: current.warned === true };
|
|
763
|
+
draining.add(drain);
|
|
764
|
+
try { await current.child.exited; } finally { draining.delete(drain); }
|
|
390
765
|
conditionalRemovePidfile(linkId, current.child.pid);
|
|
391
766
|
// A stop or removal reservation made while this child exited owns the next state.
|
|
392
767
|
if (generations.get(linkId) === generation) put(linkId, IDLE);
|
|
@@ -395,18 +770,21 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
395
770
|
const invalidateLink = (linkId: string): void => {
|
|
396
771
|
linkRevisions.set(linkId, (linkRevisions.get(linkId) ?? 0) + 1);
|
|
397
772
|
generations.set(linkId, (generations.get(linkId) ?? 0) + 1);
|
|
773
|
+
rounds.cancel(linkId);
|
|
398
774
|
};
|
|
399
775
|
|
|
400
776
|
const holdLink = (linkId: string): (() => void) => {
|
|
401
777
|
holds.set(linkId, (holds.get(linkId) ?? 0) + 1);
|
|
402
|
-
invalidateLink(linkId);
|
|
403
778
|
let released = false;
|
|
404
|
-
|
|
779
|
+
const release = (): void => {
|
|
405
780
|
if (released) return;
|
|
406
781
|
released = true;
|
|
407
782
|
const remaining = (holds.get(linkId) ?? 1) - 1;
|
|
408
783
|
if (remaining) holds.set(linkId, remaining); else holds.delete(linkId);
|
|
409
784
|
};
|
|
785
|
+
// A cancellation that throws must not leave a hold nobody can release.
|
|
786
|
+
try { invalidateLink(linkId); } catch (error) { release(); throw error; }
|
|
787
|
+
return release;
|
|
410
788
|
};
|
|
411
789
|
|
|
412
790
|
return {
|
|
@@ -430,13 +808,81 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
430
808
|
}
|
|
431
809
|
const nextStore = readStore();
|
|
432
810
|
const nextInstances = new Map(nextStore.links.map(record => [record.id, recordInstance(record, nextStore.listenerPort)]));
|
|
811
|
+
const knownBefore = new Set(recordInstances.keys());
|
|
433
812
|
const changed = new Set<string>();
|
|
813
|
+
const routesEdited: LinkRecord[] = [];
|
|
814
|
+
const gainedRoutes: LinkRecord[] = [];
|
|
815
|
+
const gainedOnChange: string[] = [];
|
|
434
816
|
for (const record of nextStore.links) {
|
|
435
817
|
if (recordInstances.get(record.id) !== undefined && recordInstances.get(record.id) !== nextInstances.get(record.id)) {
|
|
436
818
|
changed.add(record.id);
|
|
819
|
+
// A changed record that also goes from one route to several never reaches the gained-routes
|
|
820
|
+
// check below; it settles from its pidfile with the flagged changed links, after the reload
|
|
821
|
+
// stopped its child (whose pidfile may not name it: a failed write keeps an older one).
|
|
822
|
+
const before = store.links.find(link => link.id === record.id);
|
|
823
|
+
if (before && !rounds.isMultiRoute(before) && rounds.isMultiRoute(record)) gainedOnChange.push(record.id);
|
|
824
|
+
continue;
|
|
825
|
+
}
|
|
826
|
+
const before = store.links.find(link => link.id === record.id);
|
|
827
|
+
if (!before || sameArgv(effectiveLinkRoutes(before), effectiveLinkRoutes(record))) continue;
|
|
828
|
+
// A route edit restarts nothing unless the tunnel's own route is gone, or the link changes
|
|
829
|
+
// between one route and several while no adoptable tunnel runs.
|
|
830
|
+
const wasMulti = rounds.isMultiRoute(before);
|
|
831
|
+
const isMulti = rounds.isMultiRoute(record);
|
|
832
|
+
const live = wasMulti ? rounds.liveRoute(record.id) : children.has(record.id) ? before.alias : undefined;
|
|
833
|
+
if (live !== undefined && !effectiveLinkRoutes(record).includes(live)) { changed.add(record.id); continue; }
|
|
834
|
+
if (wasMulti && !isMulti) {
|
|
835
|
+
if (rounds.inRound(record.id) && live !== record.alias) { changed.add(record.id); continue; }
|
|
836
|
+
rounds.forget(record.id);
|
|
837
|
+
} else if (!wasMulti && isMulti) {
|
|
838
|
+
const child = children.get(record.id);
|
|
839
|
+
const state = states.get(record.id);
|
|
840
|
+
// An unproven child is stopped by this reload; its link then settles from its pidfile as above.
|
|
841
|
+
if (child && state?.kind !== "connected") { changed.add(record.id); gainedOnChange.push(record.id); continue; }
|
|
842
|
+
if (child && state?.kind === "connected") rounds.adopt(record, child.child, state.since);
|
|
843
|
+
gainedRoutes.push(record);
|
|
437
844
|
}
|
|
845
|
+
routesEdited.push(record);
|
|
438
846
|
}
|
|
847
|
+
// Nothing of a changed or removed link's old round may advance while the stops below await,
|
|
848
|
+
// and no orphan watch acts for a record instance that is gone or no longer has several routes.
|
|
849
|
+
// A changed record does not end an earlier tunnel that may still hold the Child's port (a new
|
|
850
|
+
// listener port changes every instance and keeps every tunnel port). A flagged link that stays
|
|
851
|
+
// multi-route keeps its watch, re-keyed to the new instance and observe-only (it never acts for
|
|
852
|
+
// a replaced instance); one with nothing watched is settled again from its pidfile below.
|
|
853
|
+
const flaggedChanged = [...changed].filter(linkId => orphanUnverified.has(linkId));
|
|
854
|
+
for (const linkId of changed) {
|
|
855
|
+
rounds.cancel(linkId);
|
|
856
|
+
const watch = orphanWatches.get(linkId);
|
|
857
|
+
const next = nextStore.links.find(link => link.id === linkId);
|
|
858
|
+
if (watch && orphanUnverified.has(linkId) && next && rounds.isMultiRoute(next)) {
|
|
859
|
+
watch.instance = recordInstance(next, nextStore.listenerPort);
|
|
860
|
+
watch.observeOnly = true;
|
|
861
|
+
for (const watched of watch.processes) delete watched.signal;
|
|
862
|
+
} else {
|
|
863
|
+
orphanWatches.delete(linkId);
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
// A removed link takes its block with it; a link added again later is settled from its pidfile below.
|
|
867
|
+
for (const linkId of recordInstances.keys()) {
|
|
868
|
+
if (nextInstances.has(linkId)) continue;
|
|
869
|
+
rounds.cancel(linkId);
|
|
870
|
+
orphanWatches.delete(linkId);
|
|
871
|
+
orphanUnverified.delete(linkId);
|
|
872
|
+
unreadablePidfiles.delete(linkId);
|
|
873
|
+
}
|
|
874
|
+
for (const record of nextStore.links) if (!rounds.isMultiRoute(record)) orphanWatches.delete(record.id);
|
|
439
875
|
store = nextStore;
|
|
876
|
+
// A one-route link that gains routes with no running child (its spawn failed: ssh missing, say)
|
|
877
|
+
// settles from its pidfile, flagged or not: removing a link drops its flag, so a re-added link
|
|
878
|
+
// may carry none. Watch an earlier process, or lift the flag when there is none (recheckPidfile:
|
|
879
|
+
// an unreadable pidfile keeps the flag).
|
|
880
|
+
for (const record of gainedRoutes) {
|
|
881
|
+
if (orphanWatches.has(record.id) || children.has(record.id)) continue;
|
|
882
|
+
recheckPidfile(record);
|
|
883
|
+
}
|
|
884
|
+
if (routesEdited.some(record => rounds.isMultiRoute(record))) rounds.recheckClient();
|
|
885
|
+
for (const record of routesEdited) if (rounds.isMultiRoute(record)) rounds.routesChanged(record);
|
|
440
886
|
for (const [linkId] of [...children]) {
|
|
441
887
|
if (nextInstances.has(linkId) && !changed.has(linkId)) continue;
|
|
442
888
|
await stopChild(linkId);
|
|
@@ -448,13 +894,28 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
448
894
|
}
|
|
449
895
|
for (const record of nextStore.links) {
|
|
450
896
|
if (changed.has(record.id)) {
|
|
897
|
+
rounds.cancel(record.id);
|
|
451
898
|
states.delete(record.id);
|
|
452
899
|
history.forget(record.id);
|
|
453
900
|
orphanUnverified.delete(record.id);
|
|
454
901
|
}
|
|
455
902
|
}
|
|
903
|
+
for (const linkId of flaggedChanged) {
|
|
904
|
+
const record = nextStore.links.find(link => link.id === linkId);
|
|
905
|
+
if (!record || !rounds.isMultiRoute(record)) continue;
|
|
906
|
+
if (orphanWatches.has(linkId)) orphanUnverified.add(linkId);
|
|
907
|
+
else recheckPidfile(record, false);
|
|
908
|
+
}
|
|
909
|
+
for (const linkId of gainedOnChange) {
|
|
910
|
+
const record = nextStore.links.find(link => link.id === linkId);
|
|
911
|
+
if (!record || flaggedChanged.includes(linkId) || orphanWatches.has(linkId)) continue;
|
|
912
|
+
// Observe-only, as for a changed record (a route edit that restarts the child keeps its instance).
|
|
913
|
+
recheckPidfile(record, false);
|
|
914
|
+
}
|
|
915
|
+
// A link new to this process settles from its pidfile as it would at startup.
|
|
916
|
+
for (const record of nextStore.links) if (!knownBefore.has(record.id) && rounds.isMultiRoute(record)) recheckPidfile(record);
|
|
456
917
|
for (const linkId of [...states.keys()]) {
|
|
457
|
-
if (!nextInstances.has(linkId)) history.forget(linkId);
|
|
918
|
+
if (!nextInstances.has(linkId)) { history.forget(linkId); rounds.forget(linkId); }
|
|
458
919
|
}
|
|
459
920
|
recordInstances.clear();
|
|
460
921
|
for (const [linkId, instance] of nextInstances) recordInstances.set(linkId, instance);
|
|
@@ -474,11 +935,18 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
474
935
|
const inFlight = reconnects.get(linkId);
|
|
475
936
|
if (inFlight) return inFlight;
|
|
476
937
|
const linkRevision = linkRevisions.get(linkId);
|
|
477
|
-
let restarted = false;
|
|
938
|
+
let restarted: boolean | "suspended" = false;
|
|
478
939
|
const flight = runLifecycle(async () => {
|
|
479
940
|
if (stopping || !started || holds.has(linkId) || linkRevisions.get(linkId) !== linkRevision) return;
|
|
480
941
|
const record = store.links.find(link => link.id === linkId && link.direction === "hub-initiated");
|
|
481
942
|
if (!record) return;
|
|
943
|
+
// spawnFor would decline it after the stop; answer that instead of stopping a child and claiming a restart.
|
|
944
|
+
if (linkSuspendedByRevoke(journalPath, linkId)) {
|
|
945
|
+
restarted = "suspended";
|
|
946
|
+
return;
|
|
947
|
+
}
|
|
948
|
+
// An earlier tunnel of a multi-route link that may still live is never bypassed by a reconnect.
|
|
949
|
+
if (rounds.isMultiRoute(record) && orphanUnverified.has(linkId)) return;
|
|
482
950
|
// A preceding reload may have spawned a new generation; it is the current one we drain.
|
|
483
951
|
const generation = generations.get(linkId);
|
|
484
952
|
const revision = lifecycleRevision;
|
|
@@ -488,6 +956,10 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
488
956
|
const fresh = latest.links.find(link => link.id === linkId && link.direction === "hub-initiated");
|
|
489
957
|
if (stopping || holds.has(linkId) || generations.get(linkId) !== generation
|
|
490
958
|
|| lifecycleRevision !== revision || !fresh || recordInstance(fresh, latest.listenerPort) !== instance) return;
|
|
959
|
+
if (linkSuspendedByRevoke(journalPath, linkId)) {
|
|
960
|
+
restarted = "suspended";
|
|
961
|
+
return;
|
|
962
|
+
}
|
|
491
963
|
spawnFor(fresh);
|
|
492
964
|
restarted = true;
|
|
493
965
|
}).then(() => restarted).finally(() => { reconnects.delete(linkId); });
|
|
@@ -496,6 +968,8 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
496
968
|
},
|
|
497
969
|
notifyAuthenticatedRequest(apiKeyId: string) {
|
|
498
970
|
for (const record of store.links) {
|
|
971
|
+
// A multi-route link is promoted only by its candidate's correlated readiness proof.
|
|
972
|
+
if (rounds.isMultiRoute(record)) continue;
|
|
499
973
|
if (record.direction === "hub-initiated" && record.apiKeyId === apiKeyId && children.has(record.id)) {
|
|
500
974
|
const state = states.get(record.id);
|
|
501
975
|
if (state?.kind === "connecting" || state?.kind === "reconnecting" || (state && retryInFlight(state))) {
|
|
@@ -505,12 +979,15 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
505
979
|
}
|
|
506
980
|
},
|
|
507
981
|
status() {
|
|
982
|
+
const suspensionOf = revokeSuspensions(journalPath);
|
|
508
983
|
return store.links.map(record => {
|
|
509
984
|
if (record.direction === "client-initiated") {
|
|
510
985
|
return { linkId: record.id, direction: record.direction, state: "client-owned", pid: null };
|
|
511
986
|
}
|
|
512
987
|
const child = children.get(record.id);
|
|
513
988
|
const recorded = history.get(record.id);
|
|
989
|
+
const routes = rounds.isMultiRoute(record) ? rounds.status(record.id) : undefined;
|
|
990
|
+
const suspension = suspensionOf(record.id);
|
|
514
991
|
return {
|
|
515
992
|
linkId: record.id,
|
|
516
993
|
direction: record.direction,
|
|
@@ -518,21 +995,32 @@ export function createLinkSupervisor(deps: LinkSupervisorDeps = {}): LinkSupervi
|
|
|
518
995
|
pid: child?.child.pid ?? null,
|
|
519
996
|
...(orphanUnverified.has(record.id) ? { orphan: "orphan-unverified" as const } : {}),
|
|
520
997
|
...(recorded ? { history: recorded } : {}),
|
|
998
|
+
...(routes ? { routes } : {}),
|
|
999
|
+
...(suspension ? { suspended: { operationId: suspension.operationId, since: suspension.since } } : {}),
|
|
521
1000
|
};
|
|
522
1001
|
});
|
|
523
1002
|
},
|
|
524
1003
|
async stop() {
|
|
525
1004
|
stopping = true;
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
1005
|
+
rounds.stop();
|
|
1006
|
+
try {
|
|
1007
|
+
await runLifecycle(async () => {
|
|
1008
|
+
// Every stop runs to its end, so one that fails cannot cut short another child's drain.
|
|
1009
|
+
const results = await Promise.allSettled([...children.keys()].map(stopChild));
|
|
1010
|
+
const failed = results.find(result => result.status === "rejected");
|
|
1011
|
+
if (failed) throw failed.reason;
|
|
1012
|
+
for (const record of store.links) {
|
|
1013
|
+
if (record.direction === "hub-initiated") put(record.id, IDLE);
|
|
1014
|
+
}
|
|
1015
|
+
});
|
|
1016
|
+
} finally {
|
|
1017
|
+
// The timer runs until the stops above are done: it alone sends SIGKILL to a child that ignores
|
|
1018
|
+
// SIGTERM. It goes even when a stop failed, so it never keeps the process alive.
|
|
1019
|
+
if (timer !== undefined) {
|
|
1020
|
+
clearTimer(timer);
|
|
1021
|
+
timer = undefined;
|
|
534
1022
|
}
|
|
535
|
-
}
|
|
1023
|
+
}
|
|
536
1024
|
try { await deps.flushDiagnostics?.(); } catch { /* best effort */ }
|
|
537
1025
|
},
|
|
538
1026
|
};
|