botmux 3.15.0 → 3.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.en.md +2 -2
- package/README.md +2 -2
- package/dist/.runtime-build-id +1 -1
- package/dist/adapters/backend/destroy-result.d.ts +159 -0
- package/dist/adapters/backend/destroy-result.d.ts.map +1 -0
- package/dist/adapters/backend/destroy-result.js +223 -0
- package/dist/adapters/backend/destroy-result.js.map +1 -0
- package/dist/adapters/backend/herdr-backend.d.ts +4 -0
- package/dist/adapters/backend/herdr-backend.d.ts.map +1 -1
- package/dist/adapters/backend/herdr-backend.js +23 -1
- package/dist/adapters/backend/herdr-backend.js.map +1 -1
- package/dist/adapters/backend/mojo-backend.d.ts +577 -0
- package/dist/adapters/backend/mojo-backend.d.ts.map +1 -0
- package/dist/adapters/backend/mojo-backend.js +2471 -0
- package/dist/adapters/backend/mojo-backend.js.map +1 -0
- package/dist/adapters/backend/mojo-budgets.d.ts +21 -0
- package/dist/adapters/backend/mojo-budgets.d.ts.map +1 -0
- package/dist/adapters/backend/mojo-budgets.js +22 -0
- package/dist/adapters/backend/mojo-budgets.js.map +1 -0
- package/dist/adapters/backend/mojo-isolated-workspace.d.ts +69 -0
- package/dist/adapters/backend/mojo-isolated-workspace.d.ts.map +1 -0
- package/dist/adapters/backend/mojo-isolated-workspace.js +213 -0
- package/dist/adapters/backend/mojo-isolated-workspace.js.map +1 -0
- package/dist/adapters/backend/mojo-process-tree.d.ts +292 -0
- package/dist/adapters/backend/mojo-process-tree.d.ts.map +1 -0
- package/dist/adapters/backend/mojo-process-tree.js +486 -0
- package/dist/adapters/backend/mojo-process-tree.js.map +1 -0
- package/dist/adapters/backend/mojo-types.d.ts +542 -0
- package/dist/adapters/backend/mojo-types.d.ts.map +1 -0
- package/dist/adapters/backend/mojo-types.js +682 -0
- package/dist/adapters/backend/mojo-types.js.map +1 -0
- package/dist/adapters/backend/reproduce-command.js +3 -3
- package/dist/adapters/backend/reproduce-command.js.map +1 -1
- package/dist/adapters/backend/riff-backend.js +1 -1
- package/dist/adapters/backend/riff-backend.js.map +1 -1
- package/dist/adapters/backend/sandbox.d.ts +44 -7
- package/dist/adapters/backend/sandbox.d.ts.map +1 -1
- package/dist/adapters/backend/sandbox.js +30 -7
- package/dist/adapters/backend/sandbox.js.map +1 -1
- package/dist/adapters/backend/session-backend-selector.d.ts +14 -1
- package/dist/adapters/backend/session-backend-selector.d.ts.map +1 -1
- package/dist/adapters/backend/session-backend-selector.js +82 -10
- package/dist/adapters/backend/session-backend-selector.js.map +1 -1
- package/dist/adapters/backend/types.d.ts +101 -6
- package/dist/adapters/backend/types.d.ts.map +1 -1
- package/dist/adapters/backend/types.js.map +1 -1
- package/dist/adapters/backend/zmx-backend.d.ts +11 -0
- package/dist/adapters/backend/zmx-backend.d.ts.map +1 -1
- package/dist/adapters/backend/zmx-backend.js +16 -1
- package/dist/adapters/backend/zmx-backend.js.map +1 -1
- package/dist/adapters/cli/codex-app.d.ts.map +1 -1
- package/dist/adapters/cli/codex-app.js +2 -2
- package/dist/adapters/cli/codex-app.js.map +1 -1
- package/dist/adapters/cli/codex.d.ts.map +1 -1
- package/dist/adapters/cli/codex.js +7 -0
- package/dist/adapters/cli/codex.js.map +1 -1
- package/dist/adapters/cli/copilot.d.ts +3 -1
- package/dist/adapters/cli/copilot.d.ts.map +1 -1
- package/dist/adapters/cli/copilot.js +23 -5
- package/dist/adapters/cli/copilot.js.map +1 -1
- package/dist/adapters/cli/cursor.d.ts.map +1 -1
- package/dist/adapters/cli/cursor.js +13 -3
- package/dist/adapters/cli/cursor.js.map +1 -1
- package/dist/adapters/cli/grok.d.ts.map +1 -1
- package/dist/adapters/cli/grok.js +24 -5
- package/dist/adapters/cli/grok.js.map +1 -1
- package/dist/adapters/cli/kimi.d.ts.map +1 -1
- package/dist/adapters/cli/kimi.js +19 -1
- package/dist/adapters/cli/kimi.js.map +1 -1
- package/dist/adapters/cli/mojo.d.ts +28 -0
- package/dist/adapters/cli/mojo.d.ts.map +1 -0
- package/dist/adapters/cli/mojo.js +58 -0
- package/dist/adapters/cli/mojo.js.map +1 -0
- package/dist/adapters/cli/pi.d.ts +6 -4
- package/dist/adapters/cli/pi.d.ts.map +1 -1
- package/dist/adapters/cli/pi.js +6 -4
- package/dist/adapters/cli/pi.js.map +1 -1
- package/dist/adapters/cli/read-isolation.d.ts +251 -1
- package/dist/adapters/cli/read-isolation.d.ts.map +1 -1
- package/dist/adapters/cli/read-isolation.js +254 -1
- package/dist/adapters/cli/read-isolation.js.map +1 -1
- package/dist/adapters/cli/registry.d.ts +13 -1
- package/dist/adapters/cli/registry.d.ts.map +1 -1
- package/dist/adapters/cli/registry.js +19 -1
- package/dist/adapters/cli/registry.js.map +1 -1
- package/dist/adapters/cli/runner-input.d.ts +3 -3
- package/dist/adapters/cli/runner-input.d.ts.map +1 -1
- package/dist/adapters/cli/runner-input.js +4 -3
- package/dist/adapters/cli/runner-input.js.map +1 -1
- package/dist/adapters/cli/shared-hints.d.ts +3 -1
- package/dist/adapters/cli/shared-hints.d.ts.map +1 -1
- package/dist/adapters/cli/shared-hints.js +22 -6
- package/dist/adapters/cli/shared-hints.js.map +1 -1
- package/dist/adapters/cli/types.d.ts +29 -4
- package/dist/adapters/cli/types.d.ts.map +1 -1
- package/dist/bot-registry.d.ts +20 -0
- package/dist/bot-registry.d.ts.map +1 -1
- package/dist/bot-registry.js +30 -4
- package/dist/bot-registry.js.map +1 -1
- package/dist/cli/arg-utils.d.ts +5 -0
- package/dist/cli/arg-utils.d.ts.map +1 -1
- package/dist/cli/arg-utils.js +7 -0
- package/dist/cli/arg-utils.js.map +1 -1
- package/dist/cli/bots-list-output.d.ts +130 -2
- package/dist/cli/bots-list-output.d.ts.map +1 -1
- package/dist/cli/bots-list-output.js +173 -27
- package/dist/cli/bots-list-output.js.map +1 -1
- package/dist/cli/daemon-lifecycle-env.d.ts +10 -3
- package/dist/cli/daemon-lifecycle-env.d.ts.map +1 -1
- package/dist/cli/daemon-lifecycle-env.js +38 -11
- package/dist/cli/daemon-lifecycle-env.js.map +1 -1
- package/dist/cli/dashboard-command.d.ts +15 -0
- package/dist/cli/dashboard-command.d.ts.map +1 -1
- package/dist/cli/dashboard-command.js +22 -0
- package/dist/cli/dashboard-command.js.map +1 -1
- package/dist/cli/log-tail.d.ts +54 -0
- package/dist/cli/log-tail.d.ts.map +1 -0
- package/dist/cli/log-tail.js +167 -0
- package/dist/cli/log-tail.js.map +1 -0
- package/dist/cli/pm2-env.d.ts +16 -0
- package/dist/cli/pm2-env.d.ts.map +1 -0
- package/dist/cli/pm2-env.js +63 -0
- package/dist/cli/pm2-env.js.map +1 -0
- package/dist/cli/pm2-fleet-lock.d.ts +26 -0
- package/dist/cli/pm2-fleet-lock.d.ts.map +1 -0
- package/dist/cli/pm2-fleet-lock.js +50 -0
- package/dist/cli/pm2-fleet-lock.js.map +1 -0
- package/dist/cli/pm2-god-retirement.d.ts +59 -0
- package/dist/cli/pm2-god-retirement.d.ts.map +1 -0
- package/dist/cli/pm2-god-retirement.js +95 -0
- package/dist/cli/pm2-god-retirement.js.map +1 -0
- package/dist/cli/session-list-wake.d.ts +41 -0
- package/dist/cli/session-list-wake.d.ts.map +1 -0
- package/dist/cli/session-list-wake.js +102 -0
- package/dist/cli/session-list-wake.js.map +1 -0
- package/dist/cli/workflow.d.ts.map +1 -1
- package/dist/cli/workflow.js +26 -0
- package/dist/cli/workflow.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +933 -115
- package/dist/cli.js.map +1 -1
- package/dist/codex-app-runner.js +261 -49
- package/dist/codex-app-runner.js.map +1 -1
- package/dist/core/cli-runtime-update.d.ts +8 -0
- package/dist/core/cli-runtime-update.d.ts.map +1 -1
- package/dist/core/cli-runtime-update.js +127 -8
- package/dist/core/cli-runtime-update.js.map +1 -1
- package/dist/core/close-residual.d.ts +64 -0
- package/dist/core/close-residual.d.ts.map +1 -0
- package/dist/core/close-residual.js +93 -0
- package/dist/core/close-residual.js.map +1 -0
- package/dist/core/closed-session-card.d.ts.map +1 -1
- package/dist/core/closed-session-card.js +11 -3
- package/dist/core/closed-session-card.js.map +1 -1
- package/dist/core/command-handler.d.ts.map +1 -1
- package/dist/core/command-handler.js +129 -27
- package/dist/core/command-handler.js.map +1 -1
- package/dist/core/cost-calculator.d.ts +3 -1
- package/dist/core/cost-calculator.d.ts.map +1 -1
- package/dist/core/cost-calculator.js +230 -1
- package/dist/core/cost-calculator.js.map +1 -1
- package/dist/core/daemon-background-close.d.ts +19 -0
- package/dist/core/daemon-background-close.d.ts.map +1 -0
- package/dist/core/daemon-background-close.js +39 -0
- package/dist/core/daemon-background-close.js.map +1 -0
- package/dist/core/dashboard-command/overview.d.ts +4 -0
- package/dist/core/dashboard-command/overview.d.ts.map +1 -1
- package/dist/core/dashboard-command/overview.js +8 -1
- package/dist/core/dashboard-command/overview.js.map +1 -1
- package/dist/core/dashboard-ipc-server.d.ts.map +1 -1
- package/dist/core/dashboard-ipc-server.js +373 -41
- package/dist/core/dashboard-ipc-server.js.map +1 -1
- package/dist/core/dashboard-rows.d.ts +4 -0
- package/dist/core/dashboard-rows.d.ts.map +1 -1
- package/dist/core/dashboard-rows.js +3 -0
- package/dist/core/dashboard-rows.js.map +1 -1
- package/dist/core/dashboard-url.d.ts +37 -0
- package/dist/core/dashboard-url.d.ts.map +1 -1
- package/dist/core/dashboard-url.js +71 -0
- package/dist/core/dashboard-url.js.map +1 -1
- package/dist/core/deferred-schedule-settlement.d.ts +15 -1
- package/dist/core/deferred-schedule-settlement.d.ts.map +1 -1
- package/dist/core/deferred-schedule-settlement.js +4 -1
- package/dist/core/deferred-schedule-settlement.js.map +1 -1
- package/dist/core/device-isolation-daemon.d.ts +91 -1
- package/dist/core/device-isolation-daemon.d.ts.map +1 -1
- package/dist/core/device-isolation-daemon.js +346 -2
- package/dist/core/device-isolation-daemon.js.map +1 -1
- package/dist/core/local-terminal-opener.d.ts.map +1 -1
- package/dist/core/local-terminal-opener.js +5 -1
- package/dist/core/local-terminal-opener.js.map +1 -1
- package/dist/core/loopback-target.d.ts +14 -0
- package/dist/core/loopback-target.d.ts.map +1 -0
- package/dist/core/loopback-target.js +20 -0
- package/dist/core/loopback-target.js.map +1 -0
- package/dist/core/maintenance.d.ts.map +1 -1
- package/dist/core/maintenance.js +16 -2
- package/dist/core/maintenance.js.map +1 -1
- package/dist/core/mojo-containment-command.d.ts +20 -0
- package/dist/core/mojo-containment-command.d.ts.map +1 -0
- package/dist/core/mojo-containment-command.js +266 -0
- package/dist/core/mojo-containment-command.js.map +1 -0
- package/dist/core/mojo-containment.d.ts +575 -0
- package/dist/core/mojo-containment.d.ts.map +1 -0
- package/dist/core/mojo-containment.js +1158 -0
- package/dist/core/mojo-containment.js.map +1 -0
- package/dist/core/mojo-launcher-env-quarantine.d.ts +30 -0
- package/dist/core/mojo-launcher-env-quarantine.d.ts.map +1 -0
- package/dist/core/mojo-launcher-env-quarantine.js +175 -0
- package/dist/core/mojo-launcher-env-quarantine.js.map +1 -0
- package/dist/core/mojo-session-identity.d.ts +3 -0
- package/dist/core/mojo-session-identity.d.ts.map +1 -0
- package/dist/core/mojo-session-identity.js +126 -0
- package/dist/core/mojo-session-identity.js.map +1 -0
- package/dist/core/persistent-backend.d.ts +32 -8
- package/dist/core/persistent-backend.d.ts.map +1 -1
- package/dist/core/persistent-backend.js +55 -17
- package/dist/core/persistent-backend.js.map +1 -1
- package/dist/core/plugins/mcp/gateway.d.ts +14 -1
- package/dist/core/plugins/mcp/gateway.d.ts.map +1 -1
- package/dist/core/plugins/mcp/gateway.js +70 -3
- package/dist/core/plugins/mcp/gateway.js.map +1 -1
- package/dist/core/plugins/mcp/host.d.ts +2 -0
- package/dist/core/plugins/mcp/host.d.ts.map +1 -1
- package/dist/core/plugins/mcp/host.js +2 -0
- package/dist/core/plugins/mcp/host.js.map +1 -1
- package/dist/core/plugins/pm2.d.ts.map +1 -1
- package/dist/core/plugins/pm2.js +16 -3
- package/dist/core/plugins/pm2.js.map +1 -1
- package/dist/core/plugins/service-manager.d.ts +9 -0
- package/dist/core/plugins/service-manager.d.ts.map +1 -1
- package/dist/core/plugins/service-manager.js +33 -16
- package/dist/core/plugins/service-manager.js.map +1 -1
- package/dist/core/preview-port-owner.d.ts +109 -0
- package/dist/core/preview-port-owner.d.ts.map +1 -0
- package/dist/core/preview-port-owner.js +320 -0
- package/dist/core/preview-port-owner.js.map +1 -0
- package/dist/core/remote-cli-ids.d.ts +5 -0
- package/dist/core/remote-cli-ids.d.ts.map +1 -0
- package/dist/core/remote-cli-ids.js +24 -0
- package/dist/core/remote-cli-ids.js.map +1 -0
- package/dist/core/{riff-shutdown-detach.d.ts → remote-shutdown-detach.d.ts} +40 -39
- package/dist/core/remote-shutdown-detach.d.ts.map +1 -0
- package/dist/core/{riff-shutdown-detach.js → remote-shutdown-detach.js} +85 -83
- package/dist/core/remote-shutdown-detach.js.map +1 -0
- package/dist/core/{riff-worker-shutdown-readiness.d.ts → remote-worker-shutdown-readiness.d.ts} +5 -4
- package/dist/core/remote-worker-shutdown-readiness.d.ts.map +1 -0
- package/dist/core/{riff-worker-shutdown-readiness.js → remote-worker-shutdown-readiness.js} +4 -3
- package/dist/core/remote-worker-shutdown-readiness.js.map +1 -0
- package/dist/core/reply-target.d.ts +16 -1
- package/dist/core/reply-target.d.ts.map +1 -1
- package/dist/core/reply-target.js +61 -0
- package/dist/core/reply-target.js.map +1 -1
- package/dist/core/session-group-birth.d.ts +26 -0
- package/dist/core/session-group-birth.d.ts.map +1 -1
- package/dist/core/session-group-birth.js +81 -4
- package/dist/core/session-group-birth.js.map +1 -1
- package/dist/core/session-manager.d.ts +15 -2
- package/dist/core/session-manager.d.ts.map +1 -1
- package/dist/core/session-manager.js +259 -5
- package/dist/core/session-manager.js.map +1 -1
- package/dist/core/session-model.d.ts +46 -0
- package/dist/core/session-model.d.ts.map +1 -0
- package/dist/core/session-model.js +42 -0
- package/dist/core/session-model.js.map +1 -0
- package/dist/core/session-owner-reminder.d.ts +63 -0
- package/dist/core/session-owner-reminder.d.ts.map +1 -0
- package/dist/core/session-owner-reminder.js +196 -0
- package/dist/core/session-owner-reminder.js.map +1 -0
- package/dist/core/session-preview-registry.d.ts +53 -0
- package/dist/core/session-preview-registry.d.ts.map +1 -0
- package/dist/core/session-preview-registry.js +82 -0
- package/dist/core/session-preview-registry.js.map +1 -0
- package/dist/core/session-preview.d.ts +114 -0
- package/dist/core/session-preview.d.ts.map +1 -0
- package/dist/core/session-preview.js +184 -0
- package/dist/core/session-preview.js.map +1 -0
- package/dist/core/session-rename.d.ts.map +1 -1
- package/dist/core/session-rename.js +5 -3
- package/dist/core/session-rename.js.map +1 -1
- package/dist/core/session-wake-deadline.d.ts +5 -0
- package/dist/core/session-wake-deadline.d.ts.map +1 -0
- package/dist/core/session-wake-deadline.js +13 -0
- package/dist/core/session-wake-deadline.js.map +1 -0
- package/dist/core/shutdown-budgets.d.ts +7 -6
- package/dist/core/shutdown-budgets.d.ts.map +1 -1
- package/dist/core/shutdown-budgets.js +11 -10
- package/dist/core/shutdown-budgets.js.map +1 -1
- package/dist/core/terminal-control-grant.d.ts +74 -0
- package/dist/core/terminal-control-grant.d.ts.map +1 -0
- package/dist/core/terminal-control-grant.js +183 -0
- package/dist/core/terminal-control-grant.js.map +1 -0
- package/dist/core/terminal-proxy.d.ts.map +1 -1
- package/dist/core/terminal-proxy.js +4 -5
- package/dist/core/terminal-proxy.js.map +1 -1
- package/dist/core/terminal-write-auth.d.ts +4 -11
- package/dist/core/terminal-write-auth.d.ts.map +1 -1
- package/dist/core/terminal-write-auth.js +19 -16
- package/dist/core/terminal-write-auth.js.map +1 -1
- package/dist/core/trigger-session.d.ts.map +1 -1
- package/dist/core/trigger-session.js +36 -29
- package/dist/core/trigger-session.js.map +1 -1
- package/dist/core/types.d.ts +88 -10
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js +9 -7
- package/dist/core/types.js.map +1 -1
- package/dist/core/workbench-link.d.ts +81 -0
- package/dist/core/workbench-link.d.ts.map +1 -0
- package/dist/core/workbench-link.js +109 -0
- package/dist/core/workbench-link.js.map +1 -0
- package/dist/core/worker-pool.d.ts +312 -11
- package/dist/core/worker-pool.d.ts.map +1 -1
- package/dist/core/worker-pool.js +2494 -209
- package/dist/core/worker-pool.js.map +1 -1
- package/dist/daemon.d.ts +6 -0
- package/dist/daemon.d.ts.map +1 -1
- package/dist/daemon.js +443 -106
- package/dist/daemon.js.map +1 -1
- package/dist/dashboard/aggregator.d.ts +33 -3
- package/dist/dashboard/aggregator.d.ts.map +1 -1
- package/dist/dashboard/aggregator.js +93 -17
- package/dist/dashboard/aggregator.js.map +1 -1
- package/dist/dashboard/auth-session-connections.d.ts +59 -0
- package/dist/dashboard/auth-session-connections.d.ts.map +1 -0
- package/dist/dashboard/auth-session-connections.js +109 -0
- package/dist/dashboard/auth-session-connections.js.map +1 -0
- package/dist/dashboard/auth.d.ts +99 -1
- package/dist/dashboard/auth.d.ts.map +1 -1
- package/dist/dashboard/auth.js +176 -64
- package/dist/dashboard/auth.js.map +1 -1
- package/dist/dashboard/bot-payload.d.ts +1 -0
- package/dist/dashboard/bot-payload.d.ts.map +1 -1
- package/dist/dashboard/bot-payload.js +3 -0
- package/dist/dashboard/bot-payload.js.map +1 -1
- package/dist/dashboard/control-audit.d.ts +112 -0
- package/dist/dashboard/control-audit.d.ts.map +1 -0
- package/dist/dashboard/control-audit.js +195 -0
- package/dist/dashboard/control-audit.js.map +1 -0
- package/dist/dashboard/control-csrf.d.ts +123 -0
- package/dist/dashboard/control-csrf.d.ts.map +1 -0
- package/dist/dashboard/control-csrf.js +392 -0
- package/dist/dashboard/control-csrf.js.map +1 -0
- package/dist/dashboard/daemon-reconcile.d.ts +26 -0
- package/dist/dashboard/daemon-reconcile.d.ts.map +1 -0
- package/dist/dashboard/daemon-reconcile.js +46 -0
- package/dist/dashboard/daemon-reconcile.js.map +1 -0
- package/dist/dashboard/debug-terminal.d.ts.map +1 -1
- package/dist/dashboard/debug-terminal.js +18 -4
- package/dist/dashboard/debug-terminal.js.map +1 -1
- package/dist/dashboard/events-sse.d.ts +60 -0
- package/dist/dashboard/events-sse.d.ts.map +1 -0
- package/dist/dashboard/events-sse.js +63 -0
- package/dist/dashboard/events-sse.js.map +1 -0
- package/dist/dashboard/h5-auth.d.ts +256 -0
- package/dist/dashboard/h5-auth.d.ts.map +1 -0
- package/dist/dashboard/h5-auth.js +825 -0
- package/dist/dashboard/h5-auth.js.map +1 -0
- package/dist/dashboard/managed-spawn.d.ts +36 -0
- package/dist/dashboard/managed-spawn.d.ts.map +1 -0
- package/dist/dashboard/managed-spawn.js +169 -0
- package/dist/dashboard/managed-spawn.js.map +1 -0
- package/dist/dashboard/preview-content-capability.d.ts +41 -0
- package/dist/dashboard/preview-content-capability.d.ts.map +1 -0
- package/dist/dashboard/preview-content-capability.js +127 -0
- package/dist/dashboard/preview-content-capability.js.map +1 -0
- package/dist/dashboard/preview-contract.d.ts +85 -0
- package/dist/dashboard/preview-contract.d.ts.map +1 -0
- package/dist/dashboard/preview-contract.js +198 -0
- package/dist/dashboard/preview-contract.js.map +1 -0
- package/dist/dashboard/preview-guard-page.d.ts +52 -0
- package/dist/dashboard/preview-guard-page.d.ts.map +1 -0
- package/dist/dashboard/preview-guard-page.js +162 -0
- package/dist/dashboard/preview-guard-page.js.map +1 -0
- package/dist/dashboard/preview-interaction.d.ts +57 -0
- package/dist/dashboard/preview-interaction.d.ts.map +1 -0
- package/dist/dashboard/preview-interaction.js +198 -0
- package/dist/dashboard/preview-interaction.js.map +1 -0
- package/dist/dashboard/preview-proxy.d.ts +113 -0
- package/dist/dashboard/preview-proxy.d.ts.map +1 -0
- package/dist/dashboard/preview-proxy.js +543 -0
- package/dist/dashboard/preview-proxy.js.map +1 -0
- package/dist/dashboard/public-redact.d.ts +23 -0
- package/dist/dashboard/public-redact.d.ts.map +1 -1
- package/dist/dashboard/public-redact.js +90 -21
- package/dist/dashboard/public-redact.js.map +1 -1
- package/dist/dashboard/request-identity.d.ts +77 -0
- package/dist/dashboard/request-identity.d.ts.map +1 -0
- package/dist/dashboard/request-identity.js +72 -0
- package/dist/dashboard/request-identity.js.map +1 -0
- package/dist/dashboard/session-cleanup.d.ts +5 -0
- package/dist/dashboard/session-cleanup.d.ts.map +1 -1
- package/dist/dashboard/session-cleanup.js +4 -0
- package/dist/dashboard/session-cleanup.js.map +1 -1
- package/dist/dashboard/settings-write-applier.d.ts +5 -1
- package/dist/dashboard/settings-write-applier.d.ts.map +1 -1
- package/dist/dashboard/settings-write-applier.js +14 -0
- package/dist/dashboard/settings-write-applier.js.map +1 -1
- package/dist/dashboard/standing-link.d.ts +76 -0
- package/dist/dashboard/standing-link.d.ts.map +1 -0
- package/dist/dashboard/standing-link.js +111 -0
- package/dist/dashboard/standing-link.js.map +1 -0
- package/dist/dashboard/terminal-control.d.ts +105 -0
- package/dist/dashboard/terminal-control.d.ts.map +1 -0
- package/dist/dashboard/terminal-control.js +296 -0
- package/dist/dashboard/terminal-control.js.map +1 -0
- package/dist/dashboard/terminal-front-proxy.d.ts +55 -0
- package/dist/dashboard/terminal-front-proxy.d.ts.map +1 -0
- package/dist/dashboard/terminal-front-proxy.js +397 -0
- package/dist/dashboard/terminal-front-proxy.js.map +1 -0
- package/dist/dashboard/terminal-view-capability.d.ts +97 -0
- package/dist/dashboard/terminal-view-capability.d.ts.map +1 -0
- package/dist/dashboard/terminal-view-capability.js +145 -0
- package/dist/dashboard/terminal-view-capability.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-api.d.ts +59 -0
- package/dist/dashboard/web/agent-workbench-api.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-api.js +227 -0
- package/dist/dashboard/web/agent-workbench-api.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-appearance-menu.d.ts +59 -0
- package/dist/dashboard/web/agent-workbench-appearance-menu.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-appearance-menu.js +195 -0
- package/dist/dashboard/web/agent-workbench-appearance-menu.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-appearance.d.ts +222 -0
- package/dist/dashboard/web/agent-workbench-appearance.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-appearance.js +499 -0
- package/dist/dashboard/web/agent-workbench-appearance.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-capabilities.d.ts +13 -0
- package/dist/dashboard/web/agent-workbench-capabilities.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-capabilities.js +24 -0
- package/dist/dashboard/web/agent-workbench-capabilities.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-chat.d.ts +90 -0
- package/dist/dashboard/web/agent-workbench-chat.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-chat.js +168 -0
- package/dist/dashboard/web/agent-workbench-chat.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-dock-page.d.ts +3 -0
- package/dist/dashboard/web/agent-workbench-dock-page.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-dock-page.js +23 -0
- package/dist/dashboard/web/agent-workbench-dock-page.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-dock-view.d.ts +22 -0
- package/dist/dashboard/web/agent-workbench-dock-view.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-dock-view.js +76 -0
- package/dist/dashboard/web/agent-workbench-dock-view.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-model.d.ts +196 -0
- package/dist/dashboard/web/agent-workbench-model.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-model.js +529 -0
- package/dist/dashboard/web/agent-workbench-model.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-page.d.ts +3 -0
- package/dist/dashboard/web/agent-workbench-page.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-page.js +27 -0
- package/dist/dashboard/web/agent-workbench-page.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-panes.d.ts +49 -0
- package/dist/dashboard/web/agent-workbench-panes.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-panes.js +468 -0
- package/dist/dashboard/web/agent-workbench-panes.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-session-list.d.ts +42 -0
- package/dist/dashboard/web/agent-workbench-session-list.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-session-list.js +353 -0
- package/dist/dashboard/web/agent-workbench-session-list.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-storage.d.ts +49 -0
- package/dist/dashboard/web/agent-workbench-storage.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-storage.js +197 -0
- package/dist/dashboard/web/agent-workbench-storage.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-touch.d.ts +46 -0
- package/dist/dashboard/web/agent-workbench-touch.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-touch.js +96 -0
- package/dist/dashboard/web/agent-workbench-touch.js.map +1 -0
- package/dist/dashboard/web/agent-workbench-view.d.ts +34 -0
- package/dist/dashboard/web/agent-workbench-view.d.ts.map +1 -0
- package/dist/dashboard/web/agent-workbench-view.js +439 -0
- package/dist/dashboard/web/agent-workbench-view.js.map +1 -0
- package/dist/dashboard/web/app.d.ts.map +1 -1
- package/dist/dashboard/web/app.js +63 -2
- package/dist/dashboard/web/app.js.map +1 -1
- package/dist/dashboard/web/bot-defaults-page.d.ts +11 -0
- package/dist/dashboard/web/bot-defaults-page.d.ts.map +1 -1
- package/dist/dashboard/web/bot-defaults-page.js +267 -36
- package/dist/dashboard/web/bot-defaults-page.js.map +1 -1
- package/dist/dashboard/web/bot-defaults.d.ts +6 -0
- package/dist/dashboard/web/bot-defaults.d.ts.map +1 -1
- package/dist/dashboard/web/bot-defaults.js.map +1 -1
- package/dist/dashboard/web/control-csrf.d.ts +4 -0
- package/dist/dashboard/web/control-csrf.d.ts.map +1 -0
- package/dist/dashboard/web/control-csrf.js +26 -0
- package/dist/dashboard/web/control-csrf.js.map +1 -0
- package/dist/dashboard/web/dashboard-routes.d.ts.map +1 -1
- package/dist/dashboard/web/dashboard-routes.js +2 -0
- package/dist/dashboard/web/dashboard-routes.js.map +1 -1
- package/dist/dashboard/web/groups-page.d.ts.map +1 -1
- package/dist/dashboard/web/groups-page.js +11 -2
- package/dist/dashboard/web/groups-page.js.map +1 -1
- package/dist/dashboard/web/i18n.d.ts.map +1 -1
- package/dist/dashboard/web/i18n.js +132 -6
- package/dist/dashboard/web/i18n.js.map +1 -1
- package/dist/dashboard/web/overview-page.d.ts.map +1 -1
- package/dist/dashboard/web/overview-page.js +3 -2
- package/dist/dashboard/web/overview-page.js.map +1 -1
- package/dist/dashboard/web/sessions-page.d.ts.map +1 -1
- package/dist/dashboard/web/sessions-page.js +41 -1
- package/dist/dashboard/web/sessions-page.js.map +1 -1
- package/dist/dashboard/web/sessions.d.ts.map +1 -1
- package/dist/dashboard/web/sessions.js +5 -1
- package/dist/dashboard/web/sessions.js.map +1 -1
- package/dist/dashboard/web/settings-page.d.ts +3 -0
- package/dist/dashboard/web/settings-page.d.ts.map +1 -1
- package/dist/dashboard/web/settings-page.js +56 -10
- package/dist/dashboard/web/settings-page.js.map +1 -1
- package/dist/dashboard/web/store.d.ts +27 -2
- package/dist/dashboard/web/store.d.ts.map +1 -1
- package/dist/dashboard/web/store.js +43 -4
- package/dist/dashboard/web/store.js.map +1 -1
- package/dist/dashboard/web/ui.d.ts +3 -0
- package/dist/dashboard/web/ui.d.ts.map +1 -1
- package/dist/dashboard/web/ui.js +11 -0
- package/dist/dashboard/web/ui.js.map +1 -1
- package/dist/dashboard/web/update-action.d.ts +12 -0
- package/dist/dashboard/web/update-action.d.ts.map +1 -1
- package/dist/dashboard/web/update-action.js +11 -0
- package/dist/dashboard/web/update-action.js.map +1 -1
- package/dist/dashboard/workbench-doctor.d.ts +114 -0
- package/dist/dashboard/workbench-doctor.d.ts.map +1 -0
- package/dist/dashboard/workbench-doctor.js +620 -0
- package/dist/dashboard/workbench-doctor.js.map +1 -0
- package/dist/dashboard/workbench-ticket.d.ts +165 -0
- package/dist/dashboard/workbench-ticket.d.ts.map +1 -0
- package/dist/dashboard/workbench-ticket.js +495 -0
- package/dist/dashboard/workbench-ticket.js.map +1 -0
- package/dist/dashboard-web/app.js +1 -1
- package/dist/dashboard-web/chunks/agent-workbench-dock-page-IKAC5CFL.js +1 -0
- package/dist/dashboard-web/chunks/agent-workbench-page-M53UN5EB.js +1 -0
- package/dist/dashboard-web/chunks/bot-defaults-page-YKE5SDZF.js +10 -0
- package/dist/dashboard-web/chunks/chunk-22WIADUT.js +2 -0
- package/dist/dashboard-web/chunks/chunk-24ONYQMR.js +1 -0
- package/dist/dashboard-web/chunks/{chunk-Z7B3E3US.js → chunk-2W6SCN3M.js} +1 -1
- package/dist/dashboard-web/chunks/chunk-4XNJX6B3.js +1 -0
- package/dist/dashboard-web/chunks/{chunk-VMWNR3SV.js → chunk-F6KNPF4R.js} +11 -9
- package/dist/dashboard-web/chunks/chunk-FXW3V4JR.js +1 -0
- package/dist/dashboard-web/chunks/chunk-G5ZS2YUS.js +1 -0
- package/dist/dashboard-web/chunks/chunk-KTQ7B6ZR.js +1 -0
- package/dist/dashboard-web/chunks/{chunk-Z7DO42GJ.js → chunk-NACP7BMR.js} +1 -1
- package/dist/dashboard-web/chunks/{connectors-page-CPH6VVFY.js → connectors-page-VOISXXII.js} +1 -1
- package/dist/dashboard-web/chunks/{feedback-page-J2FKQ3LK.js → feedback-page-RN6HRZMS.js} +1 -1
- package/dist/dashboard-web/chunks/groups-page-UQT6DCB7.js +8 -0
- package/dist/dashboard-web/chunks/{insights-page-A2LWTKJD.js → insights-page-HJFDYR5N.js} +3 -3
- package/dist/dashboard-web/chunks/{monitor-room-2YR752E3.js → monitor-room-V7PCP47V.js} +1 -1
- package/dist/dashboard-web/chunks/{monitoring-page-BGYSKZGM.js → monitoring-page-NKEKVQAP.js} +1 -1
- package/dist/dashboard-web/chunks/{office-page-ZZHTEAE4.js → office-page-GVJF73XW.js} +1 -1
- package/dist/dashboard-web/chunks/overview-page-KHLV6VTI.js +1 -0
- package/dist/dashboard-web/chunks/{plugin-page-VEGTCEAS.js → plugin-page-J3OF5IY4.js} +1 -1
- package/dist/dashboard-web/chunks/{roles-page-YUC4YLJG.js → roles-page-WQIRSOLZ.js} +1 -1
- package/dist/dashboard-web/chunks/{schedules-page-2L2NKTDJ.js → schedules-page-J2DEY2X6.js} +1 -1
- package/dist/dashboard-web/chunks/sessions-page-7BQFEA5P.js +8 -0
- package/dist/dashboard-web/chunks/settings-page-BJEIOFJH.js +2 -0
- package/dist/dashboard-web/chunks/{skills-page-DQPBBI3T.js → skills-page-HGIFR3IU.js} +1 -1
- package/dist/dashboard-web/chunks/{team-federation-page-LQKLDV4K.js → team-federation-page-MR5ZPL4B.js} +1 -1
- package/dist/dashboard-web/chunks/{v3-page-RVKQIMAP.js → v3-page-GBR5JWPE.js} +2 -2
- package/dist/dashboard-web/chunks/{whiteboards-page-S5K55HRZ.js → whiteboards-page-QAZM4BN3.js} +1 -1
- package/dist/dashboard-web/index.html +8 -1
- package/dist/dashboard-web/style.css +2006 -10
- package/dist/dashboard.js +1006 -263
- package/dist/dashboard.js.map +1 -1
- package/dist/global-config.d.ts +30 -0
- package/dist/global-config.d.ts.map +1 -1
- package/dist/global-config.js +33 -0
- package/dist/global-config.js.map +1 -1
- package/dist/i18n/en.d.ts.map +1 -1
- package/dist/i18n/en.js +31 -1
- package/dist/i18n/en.js.map +1 -1
- package/dist/i18n/zh.d.ts.map +1 -1
- package/dist/i18n/zh.js +28 -1
- package/dist/i18n/zh.js.map +1 -1
- package/dist/im/lark/card-builder.d.ts +1 -1
- package/dist/im/lark/card-builder.d.ts.map +1 -1
- package/dist/im/lark/card-builder.js +22 -6
- package/dist/im/lark/card-builder.js.map +1 -1
- package/dist/im/lark/card-handler.d.ts.map +1 -1
- package/dist/im/lark/card-handler.js +118 -24
- package/dist/im/lark/card-handler.js.map +1 -1
- package/dist/im/lark/client.d.ts +15 -0
- package/dist/im/lark/client.d.ts.map +1 -1
- package/dist/im/lark/client.js +30 -0
- package/dist/im/lark/client.js.map +1 -1
- package/dist/im/lark/event-dispatcher.d.ts +48 -0
- package/dist/im/lark/event-dispatcher.d.ts.map +1 -1
- package/dist/im/lark/event-dispatcher.js +133 -22
- package/dist/im/lark/event-dispatcher.js.map +1 -1
- package/dist/im/lark/identity-cache.d.ts +14 -6
- package/dist/im/lark/identity-cache.d.ts.map +1 -1
- package/dist/im/lark/identity-cache.js +69 -26
- package/dist/im/lark/identity-cache.js.map +1 -1
- package/dist/im/lark/lark-hosts.d.ts +13 -0
- package/dist/im/lark/lark-hosts.d.ts.map +1 -1
- package/dist/im/lark/lark-hosts.js +16 -0
- package/dist/im/lark/lark-hosts.js.map +1 -1
- package/dist/im/lark/overview-card.d.ts +22 -0
- package/dist/im/lark/overview-card.d.ts.map +1 -1
- package/dist/im/lark/overview-card.js +53 -3
- package/dist/im/lark/overview-card.js.map +1 -1
- package/dist/im/lark/sessions-card.d.ts.map +1 -1
- package/dist/im/lark/sessions-card.js +26 -3
- package/dist/im/lark/sessions-card.js.map +1 -1
- package/dist/im/lark/workflow-slash-command.d.ts +6 -0
- package/dist/im/lark/workflow-slash-command.d.ts.map +1 -1
- package/dist/im/lark/workflow-slash-command.js +7 -0
- package/dist/im/lark/workflow-slash-command.js.map +1 -1
- package/dist/index-daemon.js +27 -4
- package/dist/index-daemon.js.map +1 -1
- package/dist/index-dashboard.d.ts +3 -0
- package/dist/index-dashboard.d.ts.map +1 -0
- package/dist/index-dashboard.js +94 -0
- package/dist/index-dashboard.js.map +1 -0
- package/dist/platform/team-agents-client.d.ts +199 -0
- package/dist/platform/team-agents-client.d.ts.map +1 -0
- package/dist/platform/team-agents-client.js +263 -0
- package/dist/platform/team-agents-client.js.map +1 -0
- package/dist/platform/tunnel-client.d.ts +6 -0
- package/dist/platform/tunnel-client.d.ts.map +1 -1
- package/dist/platform/tunnel-client.js.map +1 -1
- package/dist/services/async-trigger-state.d.ts +7 -3
- package/dist/services/async-trigger-state.d.ts.map +1 -1
- package/dist/services/async-trigger-state.js +28 -0
- package/dist/services/async-trigger-state.js.map +1 -1
- package/dist/services/async-trigger-store.d.ts +13 -4
- package/dist/services/async-trigger-store.d.ts.map +1 -1
- package/dist/services/async-trigger-store.js +55 -8
- package/dist/services/async-trigger-store.js.map +1 -1
- package/dist/services/bot-config-store.d.ts +1 -11
- package/dist/services/bot-config-store.d.ts.map +1 -1
- package/dist/services/bot-config-store.js +27 -3
- package/dist/services/bot-config-store.js.map +1 -1
- package/dist/services/bot-profile-store.d.ts +17 -2
- package/dist/services/bot-profile-store.d.ts.map +1 -1
- package/dist/services/bot-profile-store.js +93 -12
- package/dist/services/bot-profile-store.js.map +1 -1
- package/dist/services/bridge-turn-queue.d.ts +10 -1
- package/dist/services/bridge-turn-queue.d.ts.map +1 -1
- package/dist/services/bridge-turn-queue.js +29 -9
- package/dist/services/bridge-turn-queue.js.map +1 -1
- package/dist/services/claude-transcript.d.ts +19 -0
- package/dist/services/claude-transcript.d.ts.map +1 -1
- package/dist/services/claude-transcript.js +57 -5
- package/dist/services/claude-transcript.js.map +1 -1
- package/dist/services/codex-app-threads.d.ts +2 -0
- package/dist/services/codex-app-threads.d.ts.map +1 -1
- package/dist/services/codex-app-threads.js +1 -1
- package/dist/services/codex-app-threads.js.map +1 -1
- package/dist/services/codex-reasoning-effort.d.ts +8 -0
- package/dist/services/codex-reasoning-effort.d.ts.map +1 -1
- package/dist/services/codex-reasoning-effort.js +24 -0
- package/dist/services/codex-reasoning-effort.js.map +1 -1
- package/dist/services/cursor-resume-policy.d.ts.map +1 -1
- package/dist/services/cursor-resume-policy.js +15 -3
- package/dist/services/cursor-resume-policy.js.map +1 -1
- package/dist/services/open-platform-rename.d.ts.map +1 -1
- package/dist/services/open-platform-rename.js +3 -2
- package/dist/services/open-platform-rename.js.map +1 -1
- package/dist/services/ordinary-turn-recovery.d.ts +76 -0
- package/dist/services/ordinary-turn-recovery.d.ts.map +1 -0
- package/dist/services/ordinary-turn-recovery.js +349 -0
- package/dist/services/ordinary-turn-recovery.js.map +1 -0
- package/dist/services/resume-fresh-policy.d.ts +17 -0
- package/dist/services/resume-fresh-policy.d.ts.map +1 -0
- package/dist/services/resume-fresh-policy.js +24 -0
- package/dist/services/resume-fresh-policy.js.map +1 -0
- package/dist/services/session-group-title.d.ts +1 -0
- package/dist/services/session-group-title.d.ts.map +1 -1
- package/dist/services/session-group-title.js +16 -1
- package/dist/services/session-group-title.js.map +1 -1
- package/dist/services/session-groups-store.d.ts +30 -0
- package/dist/services/session-groups-store.d.ts.map +1 -1
- package/dist/services/session-groups-store.js +26 -4
- package/dist/services/session-groups-store.js.map +1 -1
- package/dist/services/session-owner-notification.d.ts +9 -0
- package/dist/services/session-owner-notification.d.ts.map +1 -0
- package/dist/services/session-owner-notification.js +10 -0
- package/dist/services/session-owner-notification.js.map +1 -0
- package/dist/services/session-owner-reminder-config-store.d.ts +9 -0
- package/dist/services/session-owner-reminder-config-store.d.ts.map +1 -0
- package/dist/services/session-owner-reminder-config-store.js +24 -0
- package/dist/services/session-owner-reminder-config-store.js.map +1 -0
- package/dist/services/session-owner-reminder-store.d.ts +5 -0
- package/dist/services/session-owner-reminder-store.d.ts.map +1 -0
- package/dist/services/session-owner-reminder-store.js +54 -0
- package/dist/services/session-owner-reminder-store.js.map +1 -0
- package/dist/services/session-store.d.ts +98 -21
- package/dist/services/session-store.d.ts.map +1 -1
- package/dist/services/session-store.js +406 -48
- package/dist/services/session-store.js.map +1 -1
- package/dist/services/structured-bridge-clis.d.ts +9 -5
- package/dist/services/structured-bridge-clis.d.ts.map +1 -1
- package/dist/services/structured-bridge-clis.js +9 -4
- package/dist/services/structured-bridge-clis.js.map +1 -1
- package/dist/services/transcript-resolver.d.ts +1 -1
- package/dist/services/transcript-resolver.d.ts.map +1 -1
- package/dist/services/transcript-resolver.js +6 -1
- package/dist/services/transcript-resolver.js.map +1 -1
- package/dist/setup/bot-config-editor.d.ts.map +1 -1
- package/dist/setup/bot-config-editor.js +7 -2
- package/dist/setup/bot-config-editor.js.map +1 -1
- package/dist/setup/ensure-herdr-integrations.d.ts +13 -0
- package/dist/setup/ensure-herdr-integrations.d.ts.map +1 -1
- package/dist/setup/ensure-herdr-integrations.js +21 -3
- package/dist/setup/ensure-herdr-integrations.js.map +1 -1
- package/dist/setup/open-platform-automation.d.ts +1 -0
- package/dist/setup/open-platform-automation.d.ts.map +1 -1
- package/dist/setup/open-platform-automation.js +107 -7
- package/dist/setup/open-platform-automation.js.map +1 -1
- package/dist/skills/definitions.d.ts +17 -0
- package/dist/skills/definitions.d.ts.map +1 -1
- package/dist/skills/definitions.js +78 -8
- package/dist/skills/definitions.js.map +1 -1
- package/dist/skills/injection-mode.d.ts +24 -3
- package/dist/skills/injection-mode.d.ts.map +1 -1
- package/dist/skills/injection-mode.js +64 -15
- package/dist/skills/injection-mode.js.map +1 -1
- package/dist/skills/installer.d.ts +18 -0
- package/dist/skills/installer.d.ts.map +1 -1
- package/dist/skills/installer.js +45 -1
- package/dist/skills/installer.js.map +1 -1
- package/dist/types.d.ts +221 -10
- package/dist/types.d.ts.map +1 -1
- package/dist/utils/child-env.d.ts +92 -2
- package/dist/utils/child-env.d.ts.map +1 -1
- package/dist/utils/child-env.js +195 -1
- package/dist/utils/child-env.js.map +1 -1
- package/dist/utils/cli-usage-limit.d.ts +25 -14
- package/dist/utils/cli-usage-limit.d.ts.map +1 -1
- package/dist/utils/cli-usage-limit.js +25 -15
- package/dist/utils/cli-usage-limit.js.map +1 -1
- package/dist/utils/daemon-rejection-guard.d.ts +35 -0
- package/dist/utils/daemon-rejection-guard.d.ts.map +1 -0
- package/dist/utils/daemon-rejection-guard.js +101 -0
- package/dist/utils/daemon-rejection-guard.js.map +1 -0
- package/dist/utils/dashboard-env.d.ts +53 -0
- package/dist/utils/dashboard-env.d.ts.map +1 -0
- package/dist/utils/dashboard-env.js +116 -0
- package/dist/utils/dashboard-env.js.map +1 -0
- package/dist/utils/global-install.d.ts.map +1 -1
- package/dist/utils/global-install.js +61 -5
- package/dist/utils/global-install.js.map +1 -1
- package/dist/utils/install-diagnostics.d.ts +14 -7
- package/dist/utils/install-diagnostics.d.ts.map +1 -1
- package/dist/utils/install-diagnostics.js +20 -11
- package/dist/utils/install-diagnostics.js.map +1 -1
- package/dist/utils/local-dev-update.d.ts +132 -0
- package/dist/utils/local-dev-update.d.ts.map +1 -0
- package/dist/utils/local-dev-update.js +176 -0
- package/dist/utils/local-dev-update.js.map +1 -0
- package/dist/utils/pending-input-queue.d.ts +12 -1
- package/dist/utils/pending-input-queue.d.ts.map +1 -1
- package/dist/utils/pending-input-queue.js.map +1 -1
- package/dist/worker.js +1284 -225
- package/dist/worker.js.map +1 -1
- package/dist/workflows/definition.d.ts +68 -68
- package/dist/workflows/events/payloads.d.ts +14 -14
- package/dist/workflows/events/schema.d.ts +312 -312
- package/dist/workflows/migration/v2-run-archive-schema.d.ts +40 -40
- package/dist/workflows/v3/cli-run.d.ts.map +1 -1
- package/dist/workflows/v3/cli-run.js +7 -0
- package/dist/workflows/v3/cli-run.js.map +1 -1
- package/dist/workflows/v3/goal-cli.d.ts +3 -0
- package/dist/workflows/v3/goal-cli.d.ts.map +1 -1
- package/dist/workflows/v3/goal-cli.js +11 -0
- package/dist/workflows/v3/goal-cli.js.map +1 -1
- package/dist/workflows/v3/run-envelope.d.ts +144 -144
- package/package.json +3 -2
- package/dist/cli/pm2-god-admission.d.ts +0 -8
- package/dist/cli/pm2-god-admission.d.ts.map +0 -1
- package/dist/cli/pm2-god-admission.js +0 -25
- package/dist/cli/pm2-god-admission.js.map +0 -1
- package/dist/core/riff-shutdown-detach.d.ts.map +0 -1
- package/dist/core/riff-shutdown-detach.js.map +0 -1
- package/dist/core/riff-worker-shutdown-readiness.d.ts.map +0 -1
- package/dist/core/riff-worker-shutdown-readiness.js.map +0 -1
- package/dist/dashboard-web/chunks/bot-defaults-page-65SMZ4QY.js +0 -10
- package/dist/dashboard-web/chunks/chunk-G5XGBCM6.js +0 -1
- package/dist/dashboard-web/chunks/chunk-IKN6WURK.js +0 -1
- package/dist/dashboard-web/chunks/chunk-WLIVBPJC.js +0 -1
- package/dist/dashboard-web/chunks/groups-page-VDOOLS4M.js +0 -8
- package/dist/dashboard-web/chunks/overview-page-7NYPKQSB.js +0 -1
- package/dist/dashboard-web/chunks/sessions-page-DYZZMKFL.js +0 -8
- package/dist/dashboard-web/chunks/settings-page-LIULUSMH.js +0 -2
package/dist/core/worker-pool.js
CHANGED
|
@@ -8,25 +8,35 @@ import { homedir } from 'node:os';
|
|
|
8
8
|
import { readFileSync, readdirSync, mkdirSync, existsSync, realpathSync, unlinkSync } from 'node:fs';
|
|
9
9
|
import { atomicWriteFileSync } from '../utils/atomic-write.js';
|
|
10
10
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
11
|
-
import { ensureSkills, ensureAskSkill, ensurePluginSkills, ensureWhiteboardSkill, removeGlobalBotmuxSkills } from '../skills/installer.js';
|
|
11
|
+
import { ensureSkills, ensureAskSkill, ensurePluginSkills, ensureWhiteboardSkill, ensureWorkflowSkills, removeGlobalBotmuxSkills } from '../skills/installer.js';
|
|
12
12
|
import { shouldInstallGlobalSkills } from '../skills/injection-mode.js';
|
|
13
13
|
import { whiteboardEnabled } from '../services/whiteboard-store.js';
|
|
14
14
|
import { cliSupportsNativeUsage } from '../services/transcript-resolver.js';
|
|
15
15
|
import { cleanupTraexAskHooks, installHook } from '../adapters/hook-installer.js';
|
|
16
16
|
import { hookCommandFor } from '../adapters/hook-command.js';
|
|
17
|
-
import { randomBytes, randomUUID } from 'node:crypto';
|
|
17
|
+
import { createHash, randomBytes, randomUUID } from 'node:crypto';
|
|
18
|
+
import { mayRestoreWriteAdmission } from '../adapters/backend/destroy-result.js';
|
|
18
19
|
import { config } from '../config.js';
|
|
20
|
+
import { isWorkflowFeatureEnabled } from '../global-config.js';
|
|
19
21
|
import * as sessionStore from '../services/session-store.js';
|
|
20
22
|
import * as asyncTriggerStore from '../services/async-trigger-store.js';
|
|
21
23
|
import { markMessageListenerRunPreviewFailed, markMessageListenerRunPreviewReplied, markMessageListenerRunPreviewRunning, } from '../services/message-listener-run-preview-store.js';
|
|
22
24
|
import { persistStreamCardState, rememberLastCliInput } from './session-manager.js';
|
|
23
|
-
import {
|
|
25
|
+
import { resolveSessionLaunchModel } from './session-model.js';
|
|
26
|
+
import { fallbackTurnId, frozenReplyContextForTurn, isSubstituteTurn, rehomeReplyTargetState, replyTargetKey } from './reply-target.js';
|
|
24
27
|
import { updateMessage, deleteMessage, sendEphemeralCard, sendUserMessage, addReaction, removeReaction, getMessageChatId, MessageWithdrawnError } from '../im/lark/client.js';
|
|
25
28
|
import { buildStreamingCard, buildPrivateSnapshotCard, buildSessionCard, buildTuiPromptCard, buildTuiPromptResolvedCard, buildTuiPromptFailedCard, buildRelayedFrozenCard, getCliDisplayName } from '../im/lark/card-builder.js';
|
|
26
29
|
import { codexServiceTierBadge } from '../services/codex-service-tier.js';
|
|
27
|
-
import {
|
|
30
|
+
import { cliModelSupportsReasoningEffort, isConfigurableReasoningCliId } from '../services/codex-reasoning-effort.js';
|
|
28
31
|
import { loadFrozenCards, saveFrozenCards } from '../services/frozen-card-store.js';
|
|
29
32
|
import { hashUrlForLog } from '../adapters/backend/riff-backend.js';
|
|
33
|
+
import { cancelMojoSessionById } from '../adapters/backend/mojo-backend.js';
|
|
34
|
+
import { cleanupMojoIsolatedWorkspace } from '../adapters/backend/mojo-isolated-workspace.js';
|
|
35
|
+
import { hasUnprovenContainment } from './mojo-containment.js';
|
|
36
|
+
import { MOJO_EXPLICIT_CLOSE_RESULT_TIMEOUT_MS } from '../adapters/backend/mojo-budgets.js';
|
|
37
|
+
import { buildEffectiveMojoConfig, diffMojoSessionIdentity, MOJO_IDENTITY_KEYS, pickMojoLivePatch, isMojoRemoteGone, mojoUnprovableEnvKeys, pickMojoSessionIdentity, } from '../adapters/backend/mojo-types.js';
|
|
38
|
+
import { sanitizePerBotEnv } from './per-bot-env.js';
|
|
39
|
+
import { closeResidualClause } from './close-residual.js';
|
|
30
40
|
import { logger } from '../utils/logger.js';
|
|
31
41
|
import { createCliAdapterSync } from '../adapters/cli/registry.js';
|
|
32
42
|
import { resolveCliRuntime, runtimeInstallationKey, runtimePathOverride, snapshotCliRuntime, } from '../adapters/cli/runtime.js';
|
|
@@ -45,8 +55,10 @@ import { ZmxBackend } from '../adapters/backend/zmx-backend.js';
|
|
|
45
55
|
import { backendSupportsWebTerminal } from '../adapters/backend/capabilities.js';
|
|
46
56
|
import { sandboxEnabled } from '../adapters/backend/sandbox.js';
|
|
47
57
|
import { isStrongManagedHerdrAgentName, managedHerdrAgentName, } from '../adapters/backend/session-backend-selector.js';
|
|
48
|
-
import {
|
|
58
|
+
import { isRemoteBackendSession, isRemoteBackendType, isSuspendableBackendType, getSessionPersistentBackendType, persistentBackendTargetForSession, persistentSessionName, killPersistentBackendTarget, killPersistentSession, probePersistentBackendTarget, resolvePairedSpawnBackendType, resolvePersistentBackendTarget } from './persistent-backend.js';
|
|
49
59
|
import { withBotTurnMutation } from './bot-turn-mutation-gate.js';
|
|
60
|
+
import { recordQuarantinedLauncherEnvKeys } from './mojo-launcher-env-quarantine.js';
|
|
61
|
+
import { freezeMojoIdentityForSession } from './mojo-session-identity.js';
|
|
50
62
|
import { getBot, getAllBots, resolveBrandLabel, getLoadedConfigPath, getLoadedConfigProvenance, resolveUsageDisplay } from '../bot-registry.js';
|
|
51
63
|
import { RestartCoordinator } from './restart-coordinator.js';
|
|
52
64
|
import { runtimeBuildIdentity } from '../utils/runtime-build-id.js';
|
|
@@ -64,6 +76,12 @@ function workerForkExecArgv() {
|
|
|
64
76
|
const preloadPath = join(__dirname, '..', 'worker-ipc-preload.js');
|
|
65
77
|
return [...process.execArgv, '--import', pathToFileURL(preloadPath).href];
|
|
66
78
|
}
|
|
79
|
+
/** Secondary index of the same retirements by session id: the WeakMap above is
|
|
80
|
+
* keyed by ds OBJECT and not iterable, but the VC fence's producer-liveness
|
|
81
|
+
* gate must find a dying process after the registry entry (and possibly the
|
|
82
|
+
* ds reference itself) is gone. Entries release on worker exit, so the map
|
|
83
|
+
* stays bounded by in-flight retirements. */
|
|
84
|
+
const lifecycleRetiringBySessionId = new Map();
|
|
67
85
|
function trackLifecycleRetirement(ds, worker) {
|
|
68
86
|
let workers = lifecycleRetiringWorkers.get(ds);
|
|
69
87
|
if (!workers) {
|
|
@@ -73,19 +91,85 @@ function trackLifecycleRetirement(ds, worker) {
|
|
|
73
91
|
if (workers.has(worker))
|
|
74
92
|
return;
|
|
75
93
|
workers.add(worker);
|
|
94
|
+
const sessionId = ds.session.sessionId;
|
|
95
|
+
let byId = lifecycleRetiringBySessionId.get(sessionId);
|
|
96
|
+
if (!byId) {
|
|
97
|
+
byId = new Set();
|
|
98
|
+
lifecycleRetiringBySessionId.set(sessionId, byId);
|
|
99
|
+
}
|
|
100
|
+
byId.add(worker);
|
|
76
101
|
const release = () => {
|
|
77
102
|
const current = lifecycleRetiringWorkers.get(ds);
|
|
78
103
|
current?.delete(worker);
|
|
79
104
|
if (current?.size === 0)
|
|
80
105
|
lifecycleRetiringWorkers.delete(ds);
|
|
106
|
+
const currentById = lifecycleRetiringBySessionId.get(sessionId);
|
|
107
|
+
currentById?.delete(worker);
|
|
108
|
+
if (currentById?.size === 0)
|
|
109
|
+
lifecycleRetiringBySessionId.delete(sessionId);
|
|
81
110
|
};
|
|
82
111
|
worker.once('exit', release);
|
|
83
112
|
}
|
|
113
|
+
/**
|
|
114
|
+
* Retiring worker processes for a session id — for liveness gates that outlive
|
|
115
|
+
* the registry entry. A collision loser (or any process-only retirement) is
|
|
116
|
+
* deleted from the registry while its process is still dying; a VC fence armed
|
|
117
|
+
* in that window would otherwise find no ds, skip its producer-liveness gate,
|
|
118
|
+
* and clear on pane probes alone (fifth-round review, A1 residual).
|
|
119
|
+
*/
|
|
120
|
+
export function retiringWorkersForSession(sessionId) {
|
|
121
|
+
return [...(lifecycleRetiringBySessionId.get(sessionId) ?? [])];
|
|
122
|
+
}
|
|
84
123
|
function clearLifecycleRetirement(ds, worker) {
|
|
85
124
|
const workers = lifecycleRetiringWorkers.get(ds);
|
|
86
125
|
workers?.delete(worker);
|
|
87
126
|
if (workers?.size === 0)
|
|
88
127
|
lifecycleRetiringWorkers.delete(ds);
|
|
128
|
+
const byId = lifecycleRetiringBySessionId.get(ds.session.sessionId);
|
|
129
|
+
byId?.delete(worker);
|
|
130
|
+
if (byId?.size === 0)
|
|
131
|
+
lifecycleRetiringBySessionId.delete(ds.session.sessionId);
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Start a new worker generation's launcher-env ledger.
|
|
135
|
+
*
|
|
136
|
+
* The previous generation's keys are PARKED, never dropped — and deliberately
|
|
137
|
+
* WITHOUT consulting whether that worker has exited.
|
|
138
|
+
*
|
|
139
|
+
* A worker exit does not prove the dangerous process is gone, and the dangerous
|
|
140
|
+
* env acts on the mojo CLI *child*, not on the worker that parents it:
|
|
141
|
+
* - `MojoBackend.kill()` sends a bare `SIGTERM`, nulls its handle and reports
|
|
142
|
+
* `exitCb(0)` immediately — no SIGKILL escalation, no wait. A child that
|
|
143
|
+
* traps/ignores TERM, or is slow, survives it.
|
|
144
|
+
* - the worker's close path then runs `killCli()` and `process.exit(0)` without
|
|
145
|
+
* awaiting the child, so worker exit can precede child death.
|
|
146
|
+
* - a mojo child may also have detached descendants, so even a per-PID exit
|
|
147
|
+
* proof would not cover the process tree.
|
|
148
|
+
* `PATH`-substituted launchers and `LD_PRELOAD` hooks live in exactly that child,
|
|
149
|
+
* which is what holds the activated device credential.
|
|
150
|
+
*
|
|
151
|
+
* So the ledger is monotonic for the life of this DaemonSession: it disappears
|
|
152
|
+
* only when the session ends or the daemon restarts (it is in-memory), never on
|
|
153
|
+
* an exit signal we cannot verify. A stricter alternative — escalating to SIGKILL
|
|
154
|
+
* and waiting for process-GROUP quiescence before releasing — is the only way to
|
|
155
|
+
* make release sound, and is deliberately out of scope here: it needs new
|
|
156
|
+
* teardown machinery and would still have to cover descendants, whereas monotonic
|
|
157
|
+
* retention is fail-closed by construction.
|
|
158
|
+
*
|
|
159
|
+
* Cost: a session that was ever handed a dangerous launcher env stays
|
|
160
|
+
* unprovable until it ends. That is an availability trade, not a credential leak.
|
|
161
|
+
*
|
|
162
|
+
* Exported so the behavioural guards can assert this directly — deleting the
|
|
163
|
+
* three call sites used to leave every test green.
|
|
164
|
+
*/
|
|
165
|
+
export function startNewGenerationEnvLedger(ds, initEnv) {
|
|
166
|
+
const parked = new Set([
|
|
167
|
+
...(ds.mojoRetiringUnprovableEnvKeys ?? []),
|
|
168
|
+
...(ds.mojoAppliedUnprovableEnvKeys ?? []),
|
|
169
|
+
]);
|
|
170
|
+
ds.mojoRetiringUnprovableEnvKeys = parked.size > 0 ? [...parked] : undefined;
|
|
171
|
+
ds.mojoAppliedUnprovableEnvKeys = undefined;
|
|
172
|
+
rememberAppliedUnprovableEnvKeys(ds, initEnv);
|
|
89
173
|
}
|
|
90
174
|
/** Symmetric lifecycle fence: relay must not start while another operation is
|
|
91
175
|
* still restarting, spawning, suspending, or closing this worker generation. */
|
|
@@ -104,7 +188,7 @@ function daemonCardLocalHomeLinkMode(ds) {
|
|
|
104
188
|
// worker after riff reconciliation; fall back to persisted session metadata
|
|
105
189
|
// while restoring sessions that do not yet have an initConfig.
|
|
106
190
|
const backendType = ds.initConfig?.backendType ?? ds.session.backendType;
|
|
107
|
-
return backendType
|
|
191
|
+
return (backendType !== undefined && isRemoteBackendType(backendType))
|
|
108
192
|
|| ds.session.sandbox === true
|
|
109
193
|
|| ds.initConfig?.readIsolation === true
|
|
110
194
|
|| sandboxEnabled()
|
|
@@ -193,21 +277,25 @@ export function getDaemonStreamingCardUsageSnapshot(ds, effectiveCliId, opts) {
|
|
|
193
277
|
// Missing runtime config → default 'streaming' → show usage (best-effort).
|
|
194
278
|
}
|
|
195
279
|
const snapshot = getDaemonSessionUsageSnapshot(ds, effectiveCliId, { fresh: opts?.fresh ?? false });
|
|
196
|
-
// Model comes
|
|
197
|
-
// ds.activeModel
|
|
198
|
-
//
|
|
199
|
-
//
|
|
200
|
-
//
|
|
280
|
+
// Model normally comes from an explicitly-wired executor runtime (TRAE/Codex
|
|
281
|
+
// set ds.activeModel) or launch config. Grok's native snapshot is the one
|
|
282
|
+
// exception: its signals.json exposes a stable user-facing primaryModelId.
|
|
283
|
+
// We still never surface snapshot.tokens.model, which can be an internal
|
|
284
|
+
// routing code for Relay-family CLIs (e.g. `ark/relay-code`).
|
|
285
|
+
const grokModel = effectiveCliId === 'grok' ? snapshot.model?.trim() : undefined;
|
|
286
|
+
const grokReasoningEffort = effectiveCliId === 'grok' ? snapshot.reasoningEffort?.trim() : undefined;
|
|
201
287
|
return {
|
|
202
288
|
...snapshot,
|
|
203
|
-
...(runtimeModel ? { model: runtimeModel } : {}),
|
|
204
|
-
...(reasoningEffort ? { reasoningEffort } : {}),
|
|
289
|
+
...(runtimeModel ? { model: runtimeModel } : grokModel ? { model: grokModel } : {}),
|
|
290
|
+
...(reasoningEffort ? { reasoningEffort } : grokReasoningEffort ? { reasoningEffort: grokReasoningEffort } : {}),
|
|
205
291
|
};
|
|
206
292
|
}
|
|
207
293
|
import { normalizeBrand } from '../im/lark/lark-hosts.js';
|
|
208
294
|
import { dashboardEventBus } from './dashboard-events.js';
|
|
295
|
+
import { publishSessionPreviewCleared, takeSessionPreviewTarget, } from './session-preview-registry.js';
|
|
209
296
|
import { composeRowFromActive, composeRowFromClosed } from './dashboard-rows.js';
|
|
210
297
|
import { publishAttentionPatch, publishClosedSessionPatch } from './session-activity.js';
|
|
298
|
+
import { attachOrdinaryTurnRecovery, beginOrdinaryTurnRecovery, cancelOrdinaryTurnRecoveryForUserInput as cancelOrdinaryRecoveryForUserInput, disposeOrdinaryTurnRecovery, handleOrdinaryTurnRecoveryTerminal, ordinaryTurnRecoveryHandlesTerminal, requireOrdinaryTurnRecoveryAttention, } from '../services/ordinary-turn-recovery.js';
|
|
211
299
|
import { knownBotOpenIdsFromCrossRef } from '../utils/bot-routing.js';
|
|
212
300
|
import { emitSessionLifecycleHook, emitSessionStateTransitionHook } from '../services/session-lifecycle-hooks.js';
|
|
213
301
|
import { anchorUsageForDaemonSession, recordOwnershipForDaemonSession, recordUsageForDaemonSession, reconcileUsageForDaemonSession } from '../services/usage-ledger.js';
|
|
@@ -215,7 +303,7 @@ import { isStructuredBridgeAdoptCli } from '../services/structured-bridge-clis.j
|
|
|
215
303
|
import { ensureGatewayEntry } from './plugins/mcp/gateway-installer.js';
|
|
216
304
|
import { readPeerCrossRef } from '../services/peer-cross-ref-store.js';
|
|
217
305
|
import { appendAcceptedCodexAppDispatch, cancelCodexAppDispatch, committedCodexAppSequence, hasUnsettledCodexAppDispatch, prepareCodexAppDispatch, retryPreparedCodexAppDispatch, retainFreshCodexAppGeneration, settleCodexAppDispatch, } from '../utils/codex-app-dispatch-ledger.js';
|
|
218
|
-
import { activeSessionKey, sessionKey, sessionAnchorId, storedSessionAnchorId, isDocNativeSession, larkTransportEnabled,
|
|
306
|
+
import { activeSessionKey, sessionKey, sessionAnchorId, storedSessionAnchorId, isDocNativeSession, larkTransportEnabled, remoteRetirementAdmissionPhase, } from './types.js';
|
|
219
307
|
import { hasProtectedSessionMutationOwnership } from './session-mutation-guard.js';
|
|
220
308
|
import { DONE_REACTION_EMOJI_TYPE } from './pending-response.js';
|
|
221
309
|
import { buildTerminalUrl } from './terminal-url.js';
|
|
@@ -238,7 +326,7 @@ import { acknowledgeSessionReady } from './session-ready-handshake.js';
|
|
|
238
326
|
import { recordDispatchInputCommit } from './dispatch.js';
|
|
239
327
|
import { sendWorkerIpc } from './worker-ipc.js';
|
|
240
328
|
import { cleanupExplicitSessionBacking } from './explicit-session-backing-cleanup.js';
|
|
241
|
-
import {
|
|
329
|
+
import { REMOTE_ADMISSION_RESTORE_TIMEOUT_MS } from './shutdown-budgets.js';
|
|
242
330
|
import { managedOriginCapabilityPath, replaceManagedOriginCapabilityFile, } from './managed-origin-capability.js';
|
|
243
331
|
const __filename = fileURLToPath(import.meta.url);
|
|
244
332
|
const __dirname = dirname(__filename);
|
|
@@ -314,7 +402,7 @@ function requireCallbacks() {
|
|
|
314
402
|
// daemon registers its Map here at boot. Helpers below return a snapshot or
|
|
315
403
|
// linear-scan by sessionId.
|
|
316
404
|
let activeSessionsRegistry;
|
|
317
|
-
const
|
|
405
|
+
const pendingRemoteWorkerCloses = new Map();
|
|
318
406
|
export function setActiveSessionsRegistry(m) {
|
|
319
407
|
activeSessionsRegistry = m;
|
|
320
408
|
}
|
|
@@ -695,6 +783,99 @@ function tag(ds) {
|
|
|
695
783
|
function sessionCliId(ds, botCfg) {
|
|
696
784
|
return ds.session.cliId ?? botCfg.cliId;
|
|
697
785
|
}
|
|
786
|
+
function ordinaryTurnRecoveryEligible(ds, botCfg = getBot(ds.larkAppId).config) {
|
|
787
|
+
return sessionCliId(ds, botCfg) === 'claude-code'
|
|
788
|
+
&& ds.session.status === 'active'
|
|
789
|
+
&& !ds.adoptedFrom
|
|
790
|
+
&& !ds.session.adoptedFrom
|
|
791
|
+
&& ds.initConfig?.adoptMode !== true
|
|
792
|
+
&& !ds.session.vcMeetingReceiver
|
|
793
|
+
&& larkTransportEnabled({ chatId: ds.chatId, apiOnly: botCfg.apiOnly });
|
|
794
|
+
}
|
|
795
|
+
function ordinaryTurnRecoveryStillOwnsSession(ds) {
|
|
796
|
+
if (ds.session.status !== 'active')
|
|
797
|
+
return false;
|
|
798
|
+
if (!activeSessionsRegistry)
|
|
799
|
+
return true;
|
|
800
|
+
return activeSessionsRegistry.get(sessionKey(sessionAnchorId(ds), ds.larkAppId)) === ds;
|
|
801
|
+
}
|
|
802
|
+
function ordinaryTurnRecoveryWarning(state, locale) {
|
|
803
|
+
if (state.status === 'exhausted' && state.continuationsStarted >= 2) {
|
|
804
|
+
return tr('worker.ordinary_recovery_exhausted', undefined, locale);
|
|
805
|
+
}
|
|
806
|
+
if (state.lastErrorCode === 'recovery_enqueue_failed') {
|
|
807
|
+
return tr('worker.ordinary_recovery_enqueue_failed', undefined, locale);
|
|
808
|
+
}
|
|
809
|
+
if (state.lastErrorCode === 'recovery_delivery_failed') {
|
|
810
|
+
return tr('worker.ordinary_recovery_delivery_failed', undefined, locale);
|
|
811
|
+
}
|
|
812
|
+
if (state.lastErrorCode === 'recovery_dispatch_interrupted') {
|
|
813
|
+
return tr('worker.ordinary_recovery_dispatch_interrupted', undefined, locale);
|
|
814
|
+
}
|
|
815
|
+
return tr('worker.ordinary_recovery_non_retryable', undefined, locale);
|
|
816
|
+
}
|
|
817
|
+
/** Attach the crash-safe timer owner for one eligible ordinary Claude/Lark
|
|
818
|
+
* session. Session state is persisted; this runtime registry only owns timers. */
|
|
819
|
+
export function ensureOrdinaryTurnRecoveryAttached(ds, botCfg = getBot(ds.larkAppId).config) {
|
|
820
|
+
if (!ordinaryTurnRecoveryEligible(ds, botCfg)) {
|
|
821
|
+
disposeOrdinaryTurnRecovery(ds.session);
|
|
822
|
+
return false;
|
|
823
|
+
}
|
|
824
|
+
attachOrdinaryTurnRecovery(ds.session, {
|
|
825
|
+
schedule: (delayMs, run) => {
|
|
826
|
+
const timer = setTimeout(run, delayMs);
|
|
827
|
+
timer.unref?.();
|
|
828
|
+
return timer;
|
|
829
|
+
},
|
|
830
|
+
cancel: timer => clearTimeout(timer),
|
|
831
|
+
persist: () => sessionStore.updateSession(ds.session),
|
|
832
|
+
enqueue: (dispatch) => {
|
|
833
|
+
if (!ordinaryTurnRecoveryEligible(ds) || !ordinaryTurnRecoveryStillOwnsSession(ds))
|
|
834
|
+
return false;
|
|
835
|
+
try {
|
|
836
|
+
if (!ds.worker || ds.worker.killed || ds.worker.connected === false) {
|
|
837
|
+
return forkWorker(ds, dispatch.prompt, {
|
|
838
|
+
resume: true,
|
|
839
|
+
turnId: dispatch.turnId,
|
|
840
|
+
});
|
|
841
|
+
}
|
|
842
|
+
return sendWorkerInput(ds, dispatch.prompt, dispatch.turnId);
|
|
843
|
+
}
|
|
844
|
+
catch (err) {
|
|
845
|
+
logger.error(`[${tag(ds)}] Failed to enqueue ordinary-turn recovery `
|
|
846
|
+
+ `${dispatch.continuation}: ${err instanceof Error ? err.message : String(err)}`);
|
|
847
|
+
return false;
|
|
848
|
+
}
|
|
849
|
+
},
|
|
850
|
+
warn: state => {
|
|
851
|
+
const warning = ordinaryTurnRecoveryWarning(state, localeForBot(ds.larkAppId));
|
|
852
|
+
ds.agentAttention = {
|
|
853
|
+
kind: 'blocked',
|
|
854
|
+
reason: warning,
|
|
855
|
+
at: state.alertSentAt ?? Date.now(),
|
|
856
|
+
};
|
|
857
|
+
publishAttentionPatch(ds);
|
|
858
|
+
emitSessionLifecycleHook(ds, 'session.requires_attention', {
|
|
859
|
+
reason: 'ordinary_turn_recovery_exhausted',
|
|
860
|
+
errorCode: state.lastErrorCode,
|
|
861
|
+
logicalTurnId: state.logicalTurnId,
|
|
862
|
+
});
|
|
863
|
+
void requireCallbacks().sessionReply(sessionAnchorId(ds), warning, 'text', ds.larkAppId, state.logicalTurnId).catch(err => logger.error(`[${tag(ds)}] Failed to deliver ordinary-turn recovery warning: `
|
|
864
|
+
+ `${err instanceof Error ? err.message : String(err)}`));
|
|
865
|
+
},
|
|
866
|
+
});
|
|
867
|
+
const restored = ds.session.ordinaryTurnRecovery;
|
|
868
|
+
if (restored?.alertSentAt
|
|
869
|
+
&& (restored.status === 'exhausted' || restored.status === 'attention_required')) {
|
|
870
|
+
ds.agentAttention = {
|
|
871
|
+
kind: 'blocked',
|
|
872
|
+
reason: ordinaryTurnRecoveryWarning(restored, localeForBot(ds.larkAppId)),
|
|
873
|
+
at: restored.alertSentAt,
|
|
874
|
+
};
|
|
875
|
+
publishAttentionPatch(ds);
|
|
876
|
+
}
|
|
877
|
+
return true;
|
|
878
|
+
}
|
|
698
879
|
function sessionRuntimeDisplayName(ds, botCfg) {
|
|
699
880
|
const liveRuntime = botCfg
|
|
700
881
|
? botCfg.cliRuntime
|
|
@@ -713,13 +894,47 @@ function storedSessionCliDisplayName(ds) {
|
|
|
713
894
|
return getCliDisplayName((ds.session.cliId ?? ds.initConfig?.cliId ?? 'claude-code'));
|
|
714
895
|
}
|
|
715
896
|
}
|
|
897
|
+
/**
|
|
898
|
+
* Keep `session.model` — the record of what this session was last LAUNCHED with —
|
|
899
|
+
* in sync with the model just resolved for a (re)spawn.
|
|
900
|
+
*
|
|
901
|
+
* The record is never the launch source of truth (see resolveSessionLaunchModel);
|
|
902
|
+
* it exists so the CLI-mismatch fallback has something to fall back ON after the
|
|
903
|
+
* bot switches CLI: a session pinned to the old CLI would otherwise lose its
|
|
904
|
+
* model and drop to the CLI default. Not every entry point that hot-swaps
|
|
905
|
+
* `bot.config.cliId` (`/botconfig set cli`, the config card) runs the dashboard's
|
|
906
|
+
* closeCliMismatchedSessionsForBot sweep, so such sessions do survive to respawn.
|
|
907
|
+
*
|
|
908
|
+
* MUST be called from every path that resolves a launch model — daemon refork
|
|
909
|
+
* AND the in-worker respawns (restart IPC / crash-park retry) — or the record
|
|
910
|
+
* silently keeps an older model and a later CLI switch resurrects it.
|
|
911
|
+
*
|
|
912
|
+
* A per-trigger override is deliberately NOT recorded: it is one-shot by
|
|
913
|
+
* contract, and persisting it is exactly the mistake the frozen model was.
|
|
914
|
+
*/
|
|
915
|
+
function recordLaunchModel(ds, model) {
|
|
916
|
+
if (ds.spawnModelOverride)
|
|
917
|
+
return;
|
|
918
|
+
if (ds.session.model === model)
|
|
919
|
+
return;
|
|
920
|
+
ds.session.model = model; // undefined clears a stale record
|
|
921
|
+
sessionStore.updateSession(ds.session);
|
|
922
|
+
}
|
|
716
923
|
function sessionAgentConfig(ds, botCfg) {
|
|
717
|
-
// Freeze the agent launch config (cli / runtime / cliPath / wrapper
|
|
924
|
+
// Freeze the agent launch config (cli / runtime / cliPath / wrapper) onto the
|
|
718
925
|
// session the first time a worker forks, so later bot-level edits never
|
|
719
926
|
// retroactively change a live session — same discipline as `sandbox`.
|
|
720
927
|
//
|
|
928
|
+
// `model` is deliberately NOT part of the frozen set (see the return below):
|
|
929
|
+
// swapping the CLI *distribution* under a live conversation is what this
|
|
930
|
+
// freeze protects against (a `ttadk codex` wrapper degrading to bare `codex`
|
|
931
|
+
// loses its gateway), whereas the model is an explicit, human-chosen bot
|
|
932
|
+
// setting whose whole point is to take effect. Freezing it made a long
|
|
933
|
+
// session ignore the model configured in the dashboard forever, i.e. an
|
|
934
|
+
// inherited default silently outranked a later explicit choice.
|
|
935
|
+
//
|
|
721
936
|
// Gated on `agentFrozen`, NOT on `resume`: a session created before these
|
|
722
|
-
// fields existed has `cliId` stamped historically but no frozen wrapper
|
|
937
|
+
// fields existed has `cliId` stamped historically but no frozen wrapper,
|
|
723
938
|
// yet it was launching off the live bot config — so its first post-upgrade
|
|
724
939
|
// resume must back-fill the still-missing fields from botCfg to keep launching
|
|
725
940
|
// identically (e.g. a `ttadk codex` wrapper bot must not silently drop to bare
|
|
@@ -745,8 +960,7 @@ function sessionAgentConfig(ds, botCfg) {
|
|
|
745
960
|
?? runtimePathOverride(runtime)
|
|
746
961
|
?? botCfg.cliPathOverride;
|
|
747
962
|
ds.session.wrapperCli = ds.session.wrapperCli ?? botCfg.wrapperCli;
|
|
748
|
-
ds.session.
|
|
749
|
-
ds.session.reasoningEffort = isCodexReasoningCliId(ds.session.cliId)
|
|
963
|
+
ds.session.reasoningEffort = isConfigurableReasoningCliId(ds.session.cliId)
|
|
750
964
|
? ds.session.reasoningEffort ?? botCfg.reasoningEffort
|
|
751
965
|
: undefined;
|
|
752
966
|
ds.session.agentFrozen = true;
|
|
@@ -780,8 +994,13 @@ function sessionAgentConfig(ds, botCfg) {
|
|
|
780
994
|
if (repaired)
|
|
781
995
|
sessionStore.updateSession(ds.session);
|
|
782
996
|
}
|
|
997
|
+
// Resolved at EVERY spawn, resume included — see resolveSessionLaunchModel.
|
|
998
|
+
const model = resolveSessionLaunchModel(ds, botCfg);
|
|
999
|
+
recordLaunchModel(ds, model);
|
|
1000
|
+
// Effort is frozen per session, but whether it is *supported* depends on the
|
|
1001
|
+
// model this spawn actually uses — which is resolved live above, not frozen.
|
|
783
1002
|
if (ds.session.reasoningEffort
|
|
784
|
-
&& !
|
|
1003
|
+
&& !cliModelSupportsReasoningEffort(ds.session.cliId, model, ds.session.reasoningEffort)) {
|
|
785
1004
|
ds.session.reasoningEffort = undefined;
|
|
786
1005
|
sessionStore.updateSession(ds.session);
|
|
787
1006
|
}
|
|
@@ -790,10 +1009,206 @@ function sessionAgentConfig(ds, botCfg) {
|
|
|
790
1009
|
cliRuntime: ds.session.cliRuntime,
|
|
791
1010
|
cliPathOverride: ds.session.cliPathOverride,
|
|
792
1011
|
wrapperCli: ds.session.wrapperCli,
|
|
793
|
-
model
|
|
1012
|
+
model,
|
|
794
1013
|
reasoningEffort: ds.session.reasoningEffort,
|
|
795
1014
|
};
|
|
796
1015
|
}
|
|
1016
|
+
export { freezeMojoIdentityForSession } from './mojo-session-identity.js';
|
|
1017
|
+
/**
|
|
1018
|
+
* Freeze the control-plane identity of every restored mojo session at daemon
|
|
1019
|
+
* startup, instead of waiting for each one's next worker fork.
|
|
1020
|
+
*
|
|
1021
|
+
* The lazy path left a window: after a restart but before a session was next
|
|
1022
|
+
* woken, an operator could change the bot's endpoint/workspace, and the session
|
|
1023
|
+
* would then adopt THAT as its "original" identity — pairing a remote lineage
|
|
1024
|
+
* created on tenant A with tenant B. Migrating during restore closes it, because
|
|
1025
|
+
* restore runs before the dispatcher can deliver a message.
|
|
1026
|
+
*
|
|
1027
|
+
* A legacy row that already holds a remote lineage cannot be migrated safely (no
|
|
1028
|
+
* record of its original control plane), so sessionMojoConfig drops that lineage;
|
|
1029
|
+
* this pass reuses the same helper so both entry points behave identically.
|
|
1030
|
+
*/
|
|
1031
|
+
export function migrateMojoSessionIdentities(activeSessions) {
|
|
1032
|
+
let migrated = 0;
|
|
1033
|
+
for (const ds of activeSessions.values()) {
|
|
1034
|
+
const backendType = ds.initConfig?.backendType ?? ds.session.backendType;
|
|
1035
|
+
if (backendType !== 'mojo')
|
|
1036
|
+
continue;
|
|
1037
|
+
if (ds.session.mojoIdentity)
|
|
1038
|
+
continue;
|
|
1039
|
+
let botCfg;
|
|
1040
|
+
try {
|
|
1041
|
+
botCfg = getBot(ds.larkAppId).config;
|
|
1042
|
+
}
|
|
1043
|
+
catch {
|
|
1044
|
+
// Bot deregistered — no config to freeze from. Leave the row untouched so a
|
|
1045
|
+
// later re-registration can migrate it.
|
|
1046
|
+
continue;
|
|
1047
|
+
}
|
|
1048
|
+
void botCfg;
|
|
1049
|
+
freezeMojoIdentityForSession(ds.session, ds.larkAppId);
|
|
1050
|
+
migrated += 1;
|
|
1051
|
+
}
|
|
1052
|
+
if (migrated > 0) {
|
|
1053
|
+
logger.info(`[mojo] froze the control-plane identity of ${migrated} restored session(s)`);
|
|
1054
|
+
}
|
|
1055
|
+
}
|
|
1056
|
+
/**
|
|
1057
|
+
* READ-ONLY frozen launch identity, for teardown paths that must not mutate the
|
|
1058
|
+
* session (sessionAgentConfig freezes and writes to the session store).
|
|
1059
|
+
*
|
|
1060
|
+
* Reads the values frozen at session creation, exactly like a live spawn would.
|
|
1061
|
+
* Only a legacy session that was never frozen (`agentFrozen` unset) falls back to
|
|
1062
|
+
* the live bot config, mirroring sessionAgentConfig's back-fill.
|
|
1063
|
+
*
|
|
1064
|
+
* This matters for cancelling an orphaned remote session: the remote session was
|
|
1065
|
+
* created through THIS session's wrapper/binary, so cancelling it through a
|
|
1066
|
+
* wrapper the bot gained later can hit the wrong gateway or tenant. And because
|
|
1067
|
+
* `agentFrozen=true` with `wrapperCli=undefined` explicitly means "frozen as
|
|
1068
|
+
* no-wrapper", inheriting a later-added wrapper would break that contract.
|
|
1069
|
+
*/
|
|
1070
|
+
function frozenSessionLaunchIdentity(ds, botCfg) {
|
|
1071
|
+
if (ds.session.agentFrozen) {
|
|
1072
|
+
return {
|
|
1073
|
+
cliPathOverride: ds.session.cliPathOverride,
|
|
1074
|
+
wrapperCli: ds.session.wrapperCli,
|
|
1075
|
+
};
|
|
1076
|
+
}
|
|
1077
|
+
// Legacy, never frozen: it was launching off the live bot config, so that is
|
|
1078
|
+
// the faithful reconstruction.
|
|
1079
|
+
return {
|
|
1080
|
+
cliPathOverride: ds.session.cliPathOverride ?? botCfg.cliPathOverride,
|
|
1081
|
+
wrapperCli: ds.session.wrapperCli ?? botCfg.wrapperCli,
|
|
1082
|
+
};
|
|
1083
|
+
}
|
|
1084
|
+
/**
|
|
1085
|
+
* Resolve the mojo config a session must run on, freezing its control-plane
|
|
1086
|
+
* identity the first time and honouring that snapshot forever after.
|
|
1087
|
+
*
|
|
1088
|
+
* Split by purpose:
|
|
1089
|
+
* - identity (cloud / localDaemon / baseUrl / ppeEnv / workspaceId / agentId)
|
|
1090
|
+
* comes from the FROZEN snapshot. A live edit must not move an existing
|
|
1091
|
+
* session between execution modes or tenants — a cold resume would continue,
|
|
1092
|
+
* and `/close` would cancel, against an endpoint that never created it.
|
|
1093
|
+
* - credentials (jwt / jwtEnv / env) and behaviour knobs (stream /
|
|
1094
|
+
* systemPrompt / idleTimeoutSec) stay LIVE, so a rotated token takes effect
|
|
1095
|
+
* and no plaintext JWT is persisted into session state.
|
|
1096
|
+
*
|
|
1097
|
+
* `freeze` is false on read-only paths (workerless cancel) so teardown never
|
|
1098
|
+
* mutates session state; a session that predates this field then simply runs on
|
|
1099
|
+
* live config, exactly as it did before.
|
|
1100
|
+
*/
|
|
1101
|
+
/**
|
|
1102
|
+
* Resolve the mojo config a session must run on, plus whether its remote lineage
|
|
1103
|
+
* may still be used.
|
|
1104
|
+
*
|
|
1105
|
+
* Returns an explicit state rather than a bare config, because "we have a config"
|
|
1106
|
+
* and "this lineage is safe to resume/cancel" are different questions and
|
|
1107
|
+
* conflating them is what let a cancel reach the wrong tenant:
|
|
1108
|
+
* - `usable` — identity is frozen (or freshly frozen); lineage is trusted
|
|
1109
|
+
* - `quarantined` — legacy row with a remote lineage but no frozen identity.
|
|
1110
|
+
* Nothing records which control plane holds it, so it must not
|
|
1111
|
+
* be resumed or cancelled automatically.
|
|
1112
|
+
* - `none` — legacy row with no lineage at all; nothing at risk.
|
|
1113
|
+
*
|
|
1114
|
+
* Identity (cloud / localDaemon / baseUrl / ppeEnv / workspaceId / agentId) comes
|
|
1115
|
+
* from the FROZEN snapshot; credentials (jwt / jwtEnv / env) and behaviour knobs
|
|
1116
|
+
* stay LIVE so a rotated token takes effect and no plaintext JWT is persisted.
|
|
1117
|
+
*
|
|
1118
|
+
* `freeze` is false on read-only paths (workerless cancel) so teardown never
|
|
1119
|
+
* mutates session state.
|
|
1120
|
+
*/
|
|
1121
|
+
export function sessionMojoConfig(ds, botCfg, opts) {
|
|
1122
|
+
const live = botCfg.mojo ?? {};
|
|
1123
|
+
const liveIdentity = pickMojoSessionIdentity(live);
|
|
1124
|
+
if (!ds.session.mojoIdentity) {
|
|
1125
|
+
// Legacy row, never frozen. Delegate to the shared helper rather than
|
|
1126
|
+
// duplicating the freeze/quarantine rules — two copies of this had already
|
|
1127
|
+
// started to drift. `undefined` means "predates this field"; an empty object
|
|
1128
|
+
// means "frozen with nothing configured", which is why the helper persists
|
|
1129
|
+
// `{}` instead of skipping it.
|
|
1130
|
+
if (opts.freeze) {
|
|
1131
|
+
freezeMojoIdentityForSession(ds.session, ds.larkAppId);
|
|
1132
|
+
}
|
|
1133
|
+
else if (ds.session.riffParentTaskId) {
|
|
1134
|
+
// Read-only path (teardown): the row still has an unverifiable lineage and
|
|
1135
|
+
// we must not write, so report it without mutating.
|
|
1136
|
+
return { config: live, lineage: 'quarantined' };
|
|
1137
|
+
}
|
|
1138
|
+
// Fall through: the helper may have frozen an identity just now, so re-derive
|
|
1139
|
+
// the answer from the (possibly updated) row below.
|
|
1140
|
+
}
|
|
1141
|
+
if (!ds.session.mojoIdentity) {
|
|
1142
|
+
// Still unfrozen — the bot was deregistered, so there is nothing to freeze
|
|
1143
|
+
// from. Behave as before: usable only if there is no lineage at risk.
|
|
1144
|
+
return {
|
|
1145
|
+
config: live,
|
|
1146
|
+
lineage: ds.session.riffParentTaskId ? 'quarantined' : 'none',
|
|
1147
|
+
};
|
|
1148
|
+
}
|
|
1149
|
+
const frozen = ds.session.mojoIdentity;
|
|
1150
|
+
const drift = diffMojoSessionIdentity(frozen, liveIdentity);
|
|
1151
|
+
if (drift.length > 0) {
|
|
1152
|
+
// Loud on purpose: the operator changed the control plane and this session is
|
|
1153
|
+
// deliberately NOT following, which is otherwise invisible.
|
|
1154
|
+
logger.warn(`[${tag(ds)}] mojo control-plane config changed after session creation; `
|
|
1155
|
+
+ `keeping the frozen identity (${drift.join(', ')}). `
|
|
1156
|
+
+ 'Start a new session to use the new configuration.');
|
|
1157
|
+
}
|
|
1158
|
+
// Live credentials/behaviour, frozen identity.
|
|
1159
|
+
const merged = { ...live };
|
|
1160
|
+
for (const key of MOJO_IDENTITY_KEYS) {
|
|
1161
|
+
const value = frozen[key];
|
|
1162
|
+
if (value === undefined)
|
|
1163
|
+
delete merged[key];
|
|
1164
|
+
else
|
|
1165
|
+
merged[key] = value;
|
|
1166
|
+
}
|
|
1167
|
+
// Upgrade guard (review F1): an identity frozen by a pre-host-default build
|
|
1168
|
+
// recorded "localDaemon unset" while double-unset still MEANT the cloud
|
|
1169
|
+
// sandbox. Resuming it through the new default would silently flip the
|
|
1170
|
+
// session cloud→host — the exact transition the freeze exists to prevent,
|
|
1171
|
+
// and the drift log stays silent because frozen and live are both `{}`.
|
|
1172
|
+
// Pin those rows to the legacy behaviour; a new session adopts the new
|
|
1173
|
+
// default and gets stamped mojoIdentityHostDefault by the freeze helper.
|
|
1174
|
+
if (frozen.localDaemon === undefined
|
|
1175
|
+
&& ds.session.mojoIdentityHostDefault !== true) {
|
|
1176
|
+
merged.localDaemon = false;
|
|
1177
|
+
logger.warn(`[${tag(ds)}] mojo identity predates the host-execution default; pinning `
|
|
1178
|
+
+ 'localDaemon=false (legacy sandbox behaviour) for this session. '
|
|
1179
|
+
+ 'Close and reopen the session to adopt the new default.');
|
|
1180
|
+
// A daemon-log line alone leaves the USER staring at a session where tools
|
|
1181
|
+
// and replies silently fail (observed live: the operator retried a pinned
|
|
1182
|
+
// session twice believing the new build was broken). Queue the in-chat
|
|
1183
|
+
// notice; the next turn with a reply context delivers it once.
|
|
1184
|
+
if (ds.session.mojoLegacyPinNoticePending === undefined) {
|
|
1185
|
+
ds.session.mojoLegacyPinNoticePending = true;
|
|
1186
|
+
// Best-effort persistence: sessionMojoConfig sits on paths that MUST
|
|
1187
|
+
// survive an unwritable store (the workerless orphan-cancel chain is
|
|
1188
|
+
// explicitly tested for it). A failed save only costs notice dedupe
|
|
1189
|
+
// across a restart — the in-memory flag still delivers this run.
|
|
1190
|
+
try {
|
|
1191
|
+
sessionStore.updateSession(ds.session);
|
|
1192
|
+
}
|
|
1193
|
+
catch (err) {
|
|
1194
|
+
logger.warn(`[${tag(ds)}] could not persist the mojo legacy-pin notice flag `
|
|
1195
|
+
+ `(will still deliver this run): ${err instanceof Error ? err.message : String(err)}`);
|
|
1196
|
+
}
|
|
1197
|
+
}
|
|
1198
|
+
}
|
|
1199
|
+
// Quarantine is bound to a specific ID, not to the session.
|
|
1200
|
+
//
|
|
1201
|
+
// Once an identity is frozen, any lineage created AFTERWARDS was created on that
|
|
1202
|
+
// known control plane and is therefore trustworthy. Treating the session as
|
|
1203
|
+
// permanently quarantined meant a new session could never be cancelled after its
|
|
1204
|
+
// worker died — it kept consuming cloud sandbox time and holding credentials.
|
|
1205
|
+
// The parked id remains as an audit record only.
|
|
1206
|
+
const active = ds.session.riffParentTaskId;
|
|
1207
|
+
const lineage = active === undefined
|
|
1208
|
+
? 'none'
|
|
1209
|
+
: (active === ds.session.mojoQuarantinedLineage ? 'quarantined' : 'usable');
|
|
1210
|
+
return { config: merged, lineage };
|
|
1211
|
+
}
|
|
797
1212
|
function loadKnownBotOpenIdsForApp(larkAppId) {
|
|
798
1213
|
const dataDir = config.session.dataDir;
|
|
799
1214
|
const crossRef = readPeerCrossRef(dataDir, larkAppId);
|
|
@@ -928,6 +1343,20 @@ function logWorkerStderr(t, line) {
|
|
|
928
1343
|
// Prevents duplicate card POSTs when multiple screen_updates arrive before
|
|
929
1344
|
// the first POST returns a real message_id.
|
|
930
1345
|
export const CARD_POSTING_SENTINEL = '__posting__';
|
|
1346
|
+
/**
|
|
1347
|
+
* Freeze the destination used by one streaming-card POST before dispatch.
|
|
1348
|
+
* A POST may await the Lark API while another turn replaces
|
|
1349
|
+
* `currentReplyTarget`; callers must commit this captured key rather than
|
|
1350
|
+
* re-reading mutable session state after the request resolves.
|
|
1351
|
+
*/
|
|
1352
|
+
function captureStreamingCardReplyTarget(ds, turnId) {
|
|
1353
|
+
const effectiveTurnId = fallbackTurnId(ds, turnId);
|
|
1354
|
+
const replyContext = frozenReplyContextForTurn(ds, effectiveTurnId);
|
|
1355
|
+
return {
|
|
1356
|
+
turnId: effectiveTurnId,
|
|
1357
|
+
replyTargetKey: replyTargetKey(replyContext.target),
|
|
1358
|
+
};
|
|
1359
|
+
}
|
|
931
1360
|
/**
|
|
932
1361
|
* Move the current streaming card into `frozenCards` without freezing it
|
|
933
1362
|
* cosmetically. The next successful card POST will sweep it via
|
|
@@ -954,6 +1383,7 @@ export function parkStreamCard(ds) {
|
|
|
954
1383
|
ds.frozenCards = loadFrozenCards(ds.session.sessionId);
|
|
955
1384
|
ds.frozenCards.set(ds.streamCardNonce, {
|
|
956
1385
|
messageId: ds.streamCardId,
|
|
1386
|
+
replyTargetKey: ds.streamCardReplyTargetKey,
|
|
957
1387
|
content: ds.lastScreenContent ?? '',
|
|
958
1388
|
title: ds.currentTurnTitle ?? '',
|
|
959
1389
|
displayMode: ds.displayMode ?? 'hidden',
|
|
@@ -966,9 +1396,12 @@ export function parkStreamCard(ds) {
|
|
|
966
1396
|
saveFrozenCards(ds.session.sessionId, ds.frozenCards);
|
|
967
1397
|
}
|
|
968
1398
|
/**
|
|
969
|
-
* Delete previously-frozen streaming cards from
|
|
1399
|
+
* Delete previously-frozen streaming cards from the live card's visible Lark
|
|
1400
|
+
* destination and clear those entries from the cache.
|
|
970
1401
|
* Called whenever a new streaming card becomes the active one — old turns'
|
|
971
|
-
* cards just add visual clutter when scrolling
|
|
1402
|
+
* cards just add visual clutter when scrolling that destination's history.
|
|
1403
|
+
* A chat-scope session may answer in several Lark topics; cards from another
|
|
1404
|
+
* topic remain frozen until a successor is posted in that same topic.
|
|
972
1405
|
*
|
|
973
1406
|
* Lazy-loads `frozenCards` from disk if the in-memory Map is missing
|
|
974
1407
|
* (post daemon-restart). Best-effort delete; failures (already withdrawn,
|
|
@@ -988,10 +1421,17 @@ export function recallFrozenCards(ds) {
|
|
|
988
1421
|
const activeId = ds.streamCardId && ds.streamCardId !== CARD_POSTING_SENTINEL
|
|
989
1422
|
? ds.streamCardId
|
|
990
1423
|
: undefined;
|
|
1424
|
+
const activeReplyTargetKey = ds.streamCardReplyTargetKey;
|
|
991
1425
|
const targets = [];
|
|
992
1426
|
for (const [nonce, fc] of [...ds.frozenCards.entries()]) {
|
|
993
1427
|
if (activeId && fc.messageId === activeId)
|
|
994
1428
|
continue;
|
|
1429
|
+
// Legacy sessions have no destination key. Preserve the old sweep-all
|
|
1430
|
+
// behavior only while the live card is also legacy. Once a new card has an
|
|
1431
|
+
// exact destination, fail safe: never withdraw an unattributed card that
|
|
1432
|
+
// may be visible in another topic.
|
|
1433
|
+
if (activeReplyTargetKey && fc.replyTargetKey !== activeReplyTargetKey)
|
|
1434
|
+
continue;
|
|
995
1435
|
targets.push(fc.messageId);
|
|
996
1436
|
ds.frozenCards.delete(nonce);
|
|
997
1437
|
}
|
|
@@ -1003,6 +1443,41 @@ export function recallFrozenCards(ds) {
|
|
|
1003
1443
|
}
|
|
1004
1444
|
logger.info(`[${tag(ds)}] Recalled ${targets.length} previous streaming card(s)`);
|
|
1005
1445
|
}
|
|
1446
|
+
/** The first visible state for a newly accepted turn.
|
|
1447
|
+
*
|
|
1448
|
+
* `starting` is a process/session lifecycle state. A live Grok worker that
|
|
1449
|
+
* accepts another turn is already past startup, so surface the turn as working
|
|
1450
|
+
* immediately. The worker's structured lifecycle gate will later publish the
|
|
1451
|
+
* authoritative terminal state from updates.jsonl.
|
|
1452
|
+
*/
|
|
1453
|
+
export function turnStartingCardStatus(ds, effectiveCliId) {
|
|
1454
|
+
if (ds.usageLimit)
|
|
1455
|
+
return 'limited';
|
|
1456
|
+
if (effectiveCliId === 'grok' && workerHasInitialized(ds))
|
|
1457
|
+
return 'working';
|
|
1458
|
+
return 'starting';
|
|
1459
|
+
}
|
|
1460
|
+
/** Incremented for every worker status observation, including same-value edges.
|
|
1461
|
+
* A screen update can land while the Feishu starting-card POST is in flight;
|
|
1462
|
+
* the revision lets the POST completion distinguish that from the pre-turn
|
|
1463
|
+
* cached idle state and immediately reconcile the newly-created card. */
|
|
1464
|
+
function bumpStreamCardStatusRevision(ds) {
|
|
1465
|
+
ds.streamCardStatusRevision = (ds.streamCardStatusRevision ?? 0) + 1;
|
|
1466
|
+
}
|
|
1467
|
+
/** Re-render a just-created starting card when a newer worker status arrived
|
|
1468
|
+
* during its POST. The turn id is forwarded to scheduleCardPatch so a late
|
|
1469
|
+
* patch from turn N cannot overwrite turn N+1's card. */
|
|
1470
|
+
function reconcilePostedStartingCard(ds, turnId, revisionAtPost) {
|
|
1471
|
+
if ((ds.streamCardStatusRevision ?? 0) === revisionAtPost)
|
|
1472
|
+
return;
|
|
1473
|
+
if (!ds.streamCardId || ds.streamCardId === CARD_POSTING_SENTINEL || ds.streamCardPending)
|
|
1474
|
+
return;
|
|
1475
|
+
const botCfg = getBot(ds.larkAppId).config;
|
|
1476
|
+
const effectiveCliId = sessionCliId(ds, botCfg);
|
|
1477
|
+
const status = ds.usageLimit ? 'limited' : (ds.lastScreenStatus ?? turnStartingCardStatus(ds, effectiveCliId));
|
|
1478
|
+
const cardJson = buildStreamingCard(ds.session.sessionId, sessionAnchorId(ds), readableTerminalUrlFor(ds), ds.currentTurnTitle || ds.session.title || sessionCliDisplayName(ds, botCfg), ds.lastScreenContent ?? '', status, effectiveCliId, ds.displayMode ?? 'hidden', ds.streamCardNonce, ds.currentImageKey, !!ds.adoptedFrom, false, localeForBot(ds.larkAppId), status === 'limited' ? ds.usageLimit : undefined, writableTerminalLinkFor(ds), isLocalCliOpenReady(ds, { cliId: effectiveCliId }), getDaemonStreamingCardUsageSnapshot(ds, effectiveCliId, { fresh: status === 'idle' }), sessionRuntimeDisplayName(ds, botCfg), codexServiceTierBadge(effectiveCliId, ds.codexServiceTier));
|
|
1479
|
+
scheduleCardPatch(ds, cardJson, turnId);
|
|
1480
|
+
}
|
|
1006
1481
|
/**
|
|
1007
1482
|
* Post the current turn's starting card as soon as the daemon accepts the
|
|
1008
1483
|
* inbound message. Terminal redraw is deliberately not part of this trigger:
|
|
@@ -1016,7 +1491,7 @@ export function recallFrozenCards(ds) {
|
|
|
1016
1491
|
export async function postTurnStartingCard(ds, sessionReply, turnId) {
|
|
1017
1492
|
if (!ds.streamCardPending || ds.streamCardPendingTurnId !== turnId)
|
|
1018
1493
|
return false;
|
|
1019
|
-
if (
|
|
1494
|
+
if (remoteRetirementAdmissionPhase(ds))
|
|
1020
1495
|
return false;
|
|
1021
1496
|
if (ds.streamCardId === CARD_POSTING_SENTINEL)
|
|
1022
1497
|
return false;
|
|
@@ -1035,12 +1510,15 @@ export async function postTurnStartingCard(ds, sessionReply, turnId) {
|
|
|
1035
1510
|
const effectiveCliId = sessionCliId(ds, botCfg);
|
|
1036
1511
|
const previousCardId = ds.streamCardId;
|
|
1037
1512
|
const previousNonce = ds.streamCardNonce;
|
|
1513
|
+
const previousReplyTargetKey = ds.streamCardReplyTargetKey;
|
|
1514
|
+
const cardReplyTarget = captureStreamingCardReplyTarget(ds, turnId);
|
|
1038
1515
|
// A newer turn may have arrived while the previous turn's POST was still in
|
|
1039
1516
|
// flight, so beginNewTurn could not park that sentinel. Park the now-live
|
|
1040
1517
|
// predecessor here before replacing its identity.
|
|
1041
1518
|
parkStreamCard(ds);
|
|
1042
1519
|
const nonce = randomBytes(4).toString('hex');
|
|
1043
|
-
const status = ds
|
|
1520
|
+
const status = turnStartingCardStatus(ds, effectiveCliId);
|
|
1521
|
+
const statusRevisionAtPost = ds.streamCardStatusRevision ?? 0;
|
|
1044
1522
|
const cardJson = buildStreamingCard(ds.session.sessionId, sessionAnchorId(ds), readableTerminalUrlFor(ds), ds.currentTurnTitle || ds.session.title || sessionCliDisplayName(ds, botCfg), '', status, effectiveCliId, ds.displayMode ?? 'hidden', nonce, undefined, !!ds.adoptedFrom, false, localeForBot(ds.larkAppId), ds.usageLimit, writableTerminalLinkFor(ds), isLocalCliOpenReady(ds, { cliId: effectiveCliId }), getDaemonStreamingCardUsageSnapshot(ds, effectiveCliId), sessionRuntimeDisplayName(ds, botCfg), codexServiceTierBadge(effectiveCliId, ds.codexServiceTier));
|
|
1045
1523
|
ds.streamCardNonce = nonce;
|
|
1046
1524
|
ds.streamCardId = CARD_POSTING_SENTINEL;
|
|
@@ -1052,17 +1530,18 @@ export async function postTurnStartingCard(ds, sessionReply, turnId) {
|
|
|
1052
1530
|
&& ds.streamCardId === CARD_POSTING_SENTINEL
|
|
1053
1531
|
&& ds.streamCardNonce === nonce
|
|
1054
1532
|
&& (!activeSessionsRegistry || activeSessionsRegistry.get(registryKeyAtPost) === ds);
|
|
1055
|
-
const stillOwnsPost = () => ownsPostIdentity() &&
|
|
1533
|
+
const stillOwnsPost = () => ownsPostIdentity() && remoteRetirementAdmissionPhase(ds) === null;
|
|
1056
1534
|
const restorePrePostIdentityForRetirement = () => {
|
|
1057
|
-
if (
|
|
1535
|
+
if (remoteRetirementAdmissionPhase(ds) === null || !ownsPostIdentity())
|
|
1058
1536
|
return false;
|
|
1059
1537
|
ds.streamCardId = previousCardId;
|
|
1060
1538
|
ds.streamCardNonce = previousNonce;
|
|
1539
|
+
ds.streamCardReplyTargetKey = previousReplyTargetKey;
|
|
1061
1540
|
persistStreamCardState(ds);
|
|
1062
1541
|
return true;
|
|
1063
1542
|
};
|
|
1064
1543
|
try {
|
|
1065
|
-
const messageId = await sessionReply(anchorAtPost, cardJson, 'interactive', larkAppIdAtPost, turnId);
|
|
1544
|
+
const messageId = await sessionReply(anchorAtPost, cardJson, 'interactive', larkAppIdAtPost, cardReplyTarget.turnId);
|
|
1066
1545
|
if (!stillOwnsPost()) {
|
|
1067
1546
|
void deleteMessage(larkAppIdAtPost, messageId).catch(() => { });
|
|
1068
1547
|
restorePrePostIdentityForRetirement();
|
|
@@ -1070,6 +1549,7 @@ export async function postTurnStartingCard(ds, sessionReply, turnId) {
|
|
|
1070
1549
|
return false;
|
|
1071
1550
|
}
|
|
1072
1551
|
ds.streamCardId = messageId;
|
|
1552
|
+
ds.streamCardReplyTargetKey = cardReplyTarget.replyTargetKey;
|
|
1073
1553
|
ds.parkedStreamCardNonce = undefined;
|
|
1074
1554
|
const superseded = (ds.streamCardTurnGeneration ?? 0) !== generation;
|
|
1075
1555
|
if (!superseded) {
|
|
@@ -1083,6 +1563,7 @@ export async function postTurnStartingCard(ds, sessionReply, turnId) {
|
|
|
1083
1563
|
flushPendingActiveRuntimePatch(ds);
|
|
1084
1564
|
flushPendingCodexServiceTierPatch(ds);
|
|
1085
1565
|
syncUsageRefreshTimer(ds);
|
|
1566
|
+
reconcilePostedStartingCard(ds, turnId, statusRevisionAtPost);
|
|
1086
1567
|
logger.info(`[${tag(ds)}] Posted starting card for turn ${turnId.substring(0, 12)}`);
|
|
1087
1568
|
if (superseded && ds.streamCardPendingTurnId) {
|
|
1088
1569
|
void postTurnStartingCard(ds, sessionReply, ds.streamCardPendingTurnId);
|
|
@@ -1097,6 +1578,7 @@ export async function postTurnStartingCard(ds, sessionReply, turnId) {
|
|
|
1097
1578
|
}
|
|
1098
1579
|
ds.streamCardId = previousCardId;
|
|
1099
1580
|
ds.streamCardNonce = previousNonce;
|
|
1581
|
+
ds.streamCardReplyTargetKey = previousReplyTargetKey;
|
|
1100
1582
|
persistStreamCardState(ds);
|
|
1101
1583
|
logger.warn(`[${tag(ds)}] Failed to post starting card for turn ${turnId.substring(0, 12)}: ${err}`);
|
|
1102
1584
|
if ((ds.streamCardTurnGeneration ?? 0) !== generation && ds.streamCardPendingTurnId) {
|
|
@@ -1138,12 +1620,15 @@ export async function postFreshStreamingCard(ds, sessionReply) {
|
|
|
1138
1620
|
// together so a failed /card leaves no orphaned nonce/pending state).
|
|
1139
1621
|
const prevCardId = ds.streamCardId;
|
|
1140
1622
|
const prevNonce = ds.streamCardNonce;
|
|
1623
|
+
const prevReplyTargetKey = ds.streamCardReplyTargetKey;
|
|
1141
1624
|
const prevPending = ds.streamCardPending;
|
|
1625
|
+
const cardReplyTarget = captureStreamingCardReplyTarget(ds);
|
|
1142
1626
|
ds.streamCardNonce = randomBytes(4).toString('hex');
|
|
1143
1627
|
const cardJson = buildStreamingCard(ds.session.sessionId, sessionAnchorId(ds), readUrl, title, ds.lastScreenContent ?? '', status, effectiveCliId, ds.displayMode ?? 'hidden', ds.streamCardNonce, ds.currentImageKey, !!ds.adoptedFrom, false, localeForBot(ds.larkAppId), cardUsageLimit(ds), writableTerminalLinkFor(ds), isLocalCliOpenReady(ds, { cliId: effectiveCliId }), getDaemonStreamingCardUsageSnapshot(ds, effectiveCliId), sessionRuntimeDisplayName(ds, botCfg), codexServiceTierBadge(effectiveCliId, ds.codexServiceTier));
|
|
1144
1628
|
ds.streamCardId = CARD_POSTING_SENTINEL;
|
|
1145
1629
|
try {
|
|
1146
|
-
ds.streamCardId = await sessionReply(sessionAnchorId(ds), cardJson, 'interactive', ds.larkAppId,
|
|
1630
|
+
ds.streamCardId = await sessionReply(sessionAnchorId(ds), cardJson, 'interactive', ds.larkAppId, cardReplyTarget.turnId);
|
|
1631
|
+
ds.streamCardReplyTargetKey = cardReplyTarget.replyTargetKey;
|
|
1147
1632
|
// This card is now the live one for the current turn. Clear the new-turn
|
|
1148
1633
|
// pending flag so the next screen_update PATCHes it instead of POSTing a
|
|
1149
1634
|
// duplicate (the gate above only suppresses cards when disabled+unforced;
|
|
@@ -1166,6 +1651,7 @@ export async function postFreshStreamingCard(ds, sessionReply) {
|
|
|
1166
1651
|
catch (err) {
|
|
1167
1652
|
ds.streamCardId = prevCardId;
|
|
1168
1653
|
ds.streamCardNonce = prevNonce;
|
|
1654
|
+
ds.streamCardReplyTargetKey = prevReplyTargetKey;
|
|
1169
1655
|
ds.streamCardPending = prevPending;
|
|
1170
1656
|
flushPendingLocalCliOpenReadinessPatch(ds);
|
|
1171
1657
|
flushPendingRiffUrlPatch(ds);
|
|
@@ -1585,6 +2071,9 @@ export function ensureCliSkills(cliId, cliPathOverride) {
|
|
|
1585
2071
|
const pluginSkillsDir = join(adapter.pluginDir, 'skills');
|
|
1586
2072
|
// 白板 skill 每次 spawn 重新评估(跟随运行时开关),不进 once-cache。
|
|
1587
2073
|
ensureWhiteboardSkill(cliId, pluginSkillsDir, whiteboardEnabled());
|
|
2074
|
+
// Workflow skill 家族同理:跟随机器级 workflow 开关,每次 spawn 重新评估,
|
|
2075
|
+
// 关闭时删除已装的三个 skill,下一个会话即生效。
|
|
2076
|
+
ensureWorkflowSkills(cliId, pluginSkillsDir, isWorkflowFeatureEnabled());
|
|
1588
2077
|
if (skillsInstalledCliIds.has(cliId))
|
|
1589
2078
|
return;
|
|
1590
2079
|
ensurePluginSkills(cliId, adapter.pluginDir);
|
|
@@ -1624,6 +2113,9 @@ export function ensureCliSkills(cliId, cliPathOverride) {
|
|
|
1624
2113
|
// 立刻不再看到 botmux 技能,无需等 daemon 重启。只动 `botmux-` 命名空间,
|
|
1625
2114
|
// 绝不碰用户自定义 skill。
|
|
1626
2115
|
ensureWhiteboardSkill(cliId, skillsDir, globalInstall && whiteboardEnabled());
|
|
2116
|
+
// Workflow skill 家族:仅 global 模式落全局盘,且跟随机器级 workflow 开关。
|
|
2117
|
+
// 非 global(prompt/off)由上面的 removeGlobalBotmuxSkills 前缀清理兜底。
|
|
2118
|
+
ensureWorkflowSkills(cliId, skillsDir, globalInstall && isWorkflowFeatureEnabled());
|
|
1627
2119
|
if (!globalInstall && skillsDir)
|
|
1628
2120
|
removeGlobalBotmuxSkills(skillsDir);
|
|
1629
2121
|
if (skillsInstalledCliIds.has(cliId))
|
|
@@ -1744,15 +2236,39 @@ export function ensureClaudeFolderTrust(workingDir, stateJsonPath = join(homedir
|
|
|
1744
2236
|
}
|
|
1745
2237
|
}
|
|
1746
2238
|
// ─── Kill worker ────────────────────────────────────────────────────────────
|
|
2239
|
+
/**
|
|
2240
|
+
* Retire a session's worker (and, worker-less, its orphaned backing session).
|
|
2241
|
+
*
|
|
2242
|
+
* CONTRACT FOR REMOTE BACKENDS (riff / mojo): a LIVE remote worker is refused
|
|
2243
|
+
* unless the call carries the prepare/commit `remoteCloseCommitRequestId` —
|
|
2244
|
+
* the function RETURNS WITHOUT RETIRING and the worker keeps running. Callers
|
|
2245
|
+
* must not assume success. Today that refusal is absorbed differently per
|
|
2246
|
+
* call site:
|
|
2247
|
+
* - /cd (IM + dashboard) rejects remote sessions BEFORE repinning, so this
|
|
2248
|
+
* refusal is never reached from there;
|
|
2249
|
+
* - the explicit-close path always carries the commit requestId;
|
|
2250
|
+
* - crash-loop, collision-loser, VC restore/upgrade and device-isolation
|
|
2251
|
+
* sweeps still call this best-effort and, for a live remote worker, keep
|
|
2252
|
+
* the generation alive with its lineage intact (fail-safe: never a silent
|
|
2253
|
+
* remote cancel). A caller that needs the worker GONE must go through
|
|
2254
|
+
* prepare/commit or the shutdown-detach flow — not this function.
|
|
2255
|
+
*/
|
|
1747
2256
|
export function killWorker(ds, opts = {}) {
|
|
1748
2257
|
const closeFrozenType = ds.initConfig?.backendType ?? ds.session.backendType;
|
|
1749
2258
|
if (ds.worker && !ds.worker.killed
|
|
1750
|
-
&& closeFrozenType
|
|
1751
|
-
&&
|
|
1752
|
-
|
|
1753
|
-
//
|
|
1754
|
-
//
|
|
1755
|
-
|
|
2259
|
+
&& closeFrozenType !== undefined
|
|
2260
|
+
&& isRemoteBackendType(closeFrozenType)
|
|
2261
|
+
&& !opts.remoteCloseCommitRequestId) {
|
|
2262
|
+
// A generic synchronous retirement cannot safely detach ANY remote backend,
|
|
2263
|
+
// not just Riff. An accepted create/follow-up may still be waiting for the
|
|
2264
|
+
// task id that becomes the only durable lineage anchor — and for Mojo the
|
|
2265
|
+
// worker's legacy close path used to answer a request-less `close` with
|
|
2266
|
+
// `mojo session cancel`, so every generic retirement (/cd cold restart,
|
|
2267
|
+
// crash-loop, collision loser, restore/upgrade) silently and irreversibly
|
|
2268
|
+
// cancelled the remote session with no close_result to carry the outcome.
|
|
2269
|
+
// Only the prepare/commit flow (remoteCloseCommitRequestId) may retire a
|
|
2270
|
+
// live remote worker.
|
|
2271
|
+
logger.error(`[${tag(ds)}] Refused unprepared live ${closeFrozenType} worker retirement; `
|
|
1756
2272
|
+ 'preserving worker and remote-task lineage');
|
|
1757
2273
|
return;
|
|
1758
2274
|
}
|
|
@@ -1778,18 +2294,20 @@ export function killWorker(ds, opts = {}) {
|
|
|
1778
2294
|
// lazy-restore keep the CLI alive for later resume), so /close on such a
|
|
1779
2295
|
// session would leave an orphaned CLI running in tmux that still replies.
|
|
1780
2296
|
// Destroy the backing session directly here so /close always terminates it.
|
|
1781
|
-
destroyOrphanedBackingSession(ds
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
2297
|
+
destroyOrphanedBackingSession(ds, {
|
|
2298
|
+
...(opts.mojoCancelAlreadyProven ? { skipRemoteCancel: true } : {}),
|
|
2299
|
+
});
|
|
2300
|
+
if (opts.remoteCloseCommitRequestId
|
|
2301
|
+
&& ds.remoteCloseState?.requestId === opts.remoteCloseCommitRequestId) {
|
|
2302
|
+
ds.remoteCloseState = undefined;
|
|
1785
2303
|
}
|
|
1786
2304
|
return;
|
|
1787
2305
|
}
|
|
1788
2306
|
try {
|
|
1789
|
-
if (opts.
|
|
2307
|
+
if (opts.remoteCloseCommitRequestId) {
|
|
1790
2308
|
ds.worker.send({
|
|
1791
2309
|
type: 'close_commit',
|
|
1792
|
-
requestId: opts.
|
|
2310
|
+
requestId: opts.remoteCloseCommitRequestId,
|
|
1793
2311
|
});
|
|
1794
2312
|
}
|
|
1795
2313
|
else {
|
|
@@ -1797,9 +2315,9 @@ export function killWorker(ds, opts = {}) {
|
|
|
1797
2315
|
}
|
|
1798
2316
|
}
|
|
1799
2317
|
catch { /* IPC already closed */ }
|
|
1800
|
-
if (opts.
|
|
1801
|
-
&& ds.
|
|
1802
|
-
ds.
|
|
2318
|
+
if (opts.remoteCloseCommitRequestId
|
|
2319
|
+
&& ds.remoteCloseState?.requestId === opts.remoteCloseCommitRequestId) {
|
|
2320
|
+
ds.remoteCloseState = undefined;
|
|
1803
2321
|
}
|
|
1804
2322
|
const w = ds.worker;
|
|
1805
2323
|
trackLifecycleRetirement(ds, w);
|
|
@@ -1808,7 +2326,51 @@ export function killWorker(ds, opts = {}) {
|
|
|
1808
2326
|
// 外层 race 8s)。默认 2s SIGTERM backstop 会在取消发出前掐死进程,已关闭话题
|
|
1809
2327
|
// 的远端任务照跑——冻结为 riff 的会话放宽到 24s(层级:destroy 20s < worker 22s
|
|
1810
2328
|
// < SIGTERM 24s < SIGKILL 29s;正常路径 worker 自行 exit,不会等满)。
|
|
1811
|
-
|
|
2329
|
+
// mojo needs the same grace: destroySession() shells out to
|
|
2330
|
+
// `mojo session cancel` (CLI timeout up to 60s) and waits for an in-flight
|
|
2331
|
+
// turn to publish its lineage first, so 2s kills it before the cancel lands.
|
|
2332
|
+
const closeNeedsRemoteGrace = closeFrozenType !== undefined
|
|
2333
|
+
&& isRemoteBackendType(closeFrozenType);
|
|
2334
|
+
armWorkerKillBackstop(w, tag(ds), closeNeedsRemoteGrace ? 24_000 : WORKER_SIGTERM_BACKSTOP_MS);
|
|
2335
|
+
ds.worker = null;
|
|
2336
|
+
ds.workerPort = null;
|
|
2337
|
+
ds.workerToken = null;
|
|
2338
|
+
ds.workerViewToken = null;
|
|
2339
|
+
}
|
|
2340
|
+
/**
|
|
2341
|
+
* Retire the worker PROCESS only — no close semantics of any kind.
|
|
2342
|
+
*
|
|
2343
|
+
* No `close` IPC (a request-less remote close is refused by the worker, and a
|
|
2344
|
+
* local one would destroy the backing session), no remote cancel, lineage and
|
|
2345
|
+
* containment handles intact. SIGTERM is sent immediately (the worker's
|
|
2346
|
+
* handler runs killCli() — for mojo that is backend.kill(), which by contract
|
|
2347
|
+
* never cancels the remote session) and armWorkerKillBackstop guarantees
|
|
2348
|
+
* SIGKILL if it wedges.
|
|
2349
|
+
*
|
|
2350
|
+
* For callers that must make a worker generation DIE while its logical/remote
|
|
2351
|
+
* state lives on: the collision loser (its registry entry is deleted right
|
|
2352
|
+
* after, so a refused killWorker left a live credential-carrying worker
|
|
2353
|
+
* unreachable — the P0-new orphan shape) and the VC runtime-lease fence
|
|
2354
|
+
* (which must prove the local producer process dead before admitting replay).
|
|
2355
|
+
*/
|
|
2356
|
+
export function retireWorkerProcessOnly(ds, reason) {
|
|
2357
|
+
restartCoordinator.cancelSession(ds.session.sessionId);
|
|
2358
|
+
clearUsageLimitState(ds);
|
|
2359
|
+
ds.workerReady = false;
|
|
2360
|
+
clearUsageRefreshTimer(ds);
|
|
2361
|
+
ds.localProcessAttestation = undefined;
|
|
2362
|
+
ds.managedTurnOrigin = undefined;
|
|
2363
|
+
invalidateStuckWarning(ds, reason);
|
|
2364
|
+
invalidateTuiPrompt(ds, reason);
|
|
2365
|
+
const w = ds.worker;
|
|
2366
|
+
if (w && !w.killed) {
|
|
2367
|
+
trackLifecycleRetirement(ds, w);
|
|
2368
|
+
try {
|
|
2369
|
+
w.kill('SIGTERM');
|
|
2370
|
+
}
|
|
2371
|
+
catch { /* already gone */ }
|
|
2372
|
+
armWorkerKillBackstop(w, tag(ds));
|
|
2373
|
+
}
|
|
1812
2374
|
ds.worker = null;
|
|
1813
2375
|
ds.workerPort = null;
|
|
1814
2376
|
ds.workerToken = null;
|
|
@@ -2040,12 +2602,118 @@ function destroyLivePaneBeforeRestart(ds) {
|
|
|
2040
2602
|
* ANTHROPIC_BASE_URL/TOKEN)后 /restart 并不会生效(只有 refork 路径生效)。
|
|
2041
2603
|
* 三分态返回:对象 = 最新 env;null = 明确清空(dashboard 已删);
|
|
2042
2604
|
* undefined = 取不到(bot 已注销等异常),让 worker 保持快照不动(=旧行为)。
|
|
2043
|
-
* 只热更 env
|
|
2605
|
+
* 只热更 env / model 两个字段:sandbox/backendType 是刻意 freeze-once 的设计(见
|
|
2044
2606
|
* forkWorker init 注释),cliId 换 CLI 会踩 resume transcript 对齐,均不带。
|
|
2045
2607
|
*/
|
|
2046
2608
|
export function latestPerBotEnvForRestart(ds) {
|
|
2609
|
+
// Two failure modes that must NOT be conflated. The old single try/catch
|
|
2610
|
+
// swallowed both, so a quarantine write failure silently returned `undefined`
|
|
2611
|
+
// (restart proceeds, risk unrecorded) — the exact fail-open this ledger exists
|
|
2612
|
+
// to prevent.
|
|
2613
|
+
let env;
|
|
2047
2614
|
try {
|
|
2048
|
-
|
|
2615
|
+
env = getBot(ds.larkAppId).config.env ?? null;
|
|
2616
|
+
}
|
|
2617
|
+
catch {
|
|
2618
|
+
// Bot deregistered: nothing to hot-update. Keep the worker's snapshot (legacy
|
|
2619
|
+
// behaviour) — this is a benign, expected state.
|
|
2620
|
+
return undefined;
|
|
2621
|
+
}
|
|
2622
|
+
// Record what this generation is being handed BEFORE the restart is sent.
|
|
2623
|
+
//
|
|
2624
|
+
// This is the single choke point all four restart senders share (dashboard-ipc
|
|
2625
|
+
// operator + working-dir, requestSessionRestart, cli_crash auto-restart), so
|
|
2626
|
+
// recording here cannot be missed by adding a fifth caller — which is exactly
|
|
2627
|
+
// how the device-isolation proof lost track of the env the running child had
|
|
2628
|
+
// actually been given.
|
|
2629
|
+
//
|
|
2630
|
+
// Accumulate, never replace: the restart may fail or be coalesced, so a later
|
|
2631
|
+
// clean env is not evidence that the dangerous child is gone.
|
|
2632
|
+
//
|
|
2633
|
+
// Any throw here propagates on purpose: the caller must not send a restart while
|
|
2634
|
+
// believing a risk was recorded when it was not.
|
|
2635
|
+
rememberAppliedUnprovableEnvKeys(ds, env ?? undefined);
|
|
2636
|
+
return env;
|
|
2637
|
+
}
|
|
2638
|
+
/**
|
|
2639
|
+
* Fold the unprovable keys of an env payload into the generation's ledger.
|
|
2640
|
+
*
|
|
2641
|
+
* Names only — the values can be credentials, and the proof only inspects names.
|
|
2642
|
+
* Exported for the restart-timeline tests, which must be able to assert the
|
|
2643
|
+
* ledger itself rather than infer it from a classifier result.
|
|
2644
|
+
*/
|
|
2645
|
+
export function rememberAppliedUnprovableEnvKeys(ds, env) {
|
|
2646
|
+
// MOJO ONLY. This choke point is shared by every backend's restart, so an
|
|
2647
|
+
// ungated call quarantined codex/tmux sessions as "mojo" forever: a bot with
|
|
2648
|
+
// LD_PRELOAD in bots.json env poisoned unrelated backends (cross-backend
|
|
2649
|
+
// contamination + permanent unprovability). The ledger exists solely to answer
|
|
2650
|
+
// "could a hooked MOJO client still be running", so only mojo may write to it.
|
|
2651
|
+
if (!isMojoSession(ds))
|
|
2652
|
+
return;
|
|
2653
|
+
// BOTH env layers. `mojoUnprovableEnvKeys` is fed the top-level per-bot env AND
|
|
2654
|
+
// the mojo block's own env: they are peers (see the `init` message), the mojo
|
|
2655
|
+
// block wins the merge, and a dangerous key living only in `mojo.env` used to be
|
|
2656
|
+
// recorded nowhere — so it survived a daemon restart as "clean".
|
|
2657
|
+
const mojoBlockEnv = sessionMojoBlockEnv(ds);
|
|
2658
|
+
const keys = mojoUnprovableEnvKeys({ env: { ...(env ?? {}), ...mojoBlockEnv } });
|
|
2659
|
+
if (keys.length === 0)
|
|
2660
|
+
return;
|
|
2661
|
+
const merged = new Set([...(ds.mojoAppliedUnprovableEnvKeys ?? []), ...keys]);
|
|
2662
|
+
ds.mojoAppliedUnprovableEnvKeys = [...merged];
|
|
2663
|
+
// Also persist, because the in-memory ledger cannot answer the question after
|
|
2664
|
+
// a daemon restart or an explicit /close: the SIGTERM-ed mojo child (and any
|
|
2665
|
+
// detached descendants) can outlive both, while still holding an activated
|
|
2666
|
+
// credential. The durable record is keyed by session id and never retracted.
|
|
2667
|
+
const sessionId = ds.session?.sessionId;
|
|
2668
|
+
if (!sessionId) {
|
|
2669
|
+
// Cannot key the durable record without a session id. Never swallow this:
|
|
2670
|
+
// keeping the risk in memory only is the fail-open direction, and this branch
|
|
2671
|
+
// should be unreachable for a real DaemonSession.
|
|
2672
|
+
throw new Error('[mojo-quarantine] refusing to lose an unprovable launcher env: session id missing'
|
|
2673
|
+
+ ` (keys: ${keys.join(', ')})`);
|
|
2674
|
+
}
|
|
2675
|
+
// Deliberately NOT wrapped in try/catch: a failed persist means the daemon
|
|
2676
|
+
// cannot prove this host is clean, and swallowing it would hand back a false
|
|
2677
|
+
// "recorded" to every restart sender.
|
|
2678
|
+
recordQuarantinedLauncherEnvKeys(sessionId, keys);
|
|
2679
|
+
}
|
|
2680
|
+
/** Is this session frozen onto the mojo backend? Prefers the live worker stamp. */
|
|
2681
|
+
function isMojoSession(ds) {
|
|
2682
|
+
return (ds.initConfig?.backendType ?? ds.session?.backendType) === 'mojo';
|
|
2683
|
+
}
|
|
2684
|
+
/**
|
|
2685
|
+
* The mojo block's own `env`, from whichever layer the next turn will actually
|
|
2686
|
+
* use: the config frozen onto the live worker, else the live bot config.
|
|
2687
|
+
*/
|
|
2688
|
+
function sessionMojoBlockEnv(ds) {
|
|
2689
|
+
const fromInit = ds.initConfig?.backendConfig?.env;
|
|
2690
|
+
if (fromInit)
|
|
2691
|
+
return fromInit;
|
|
2692
|
+
try {
|
|
2693
|
+
return getBot(ds.larkAppId).config.mojo?.env ?? {};
|
|
2694
|
+
}
|
|
2695
|
+
catch {
|
|
2696
|
+
return {};
|
|
2697
|
+
}
|
|
2698
|
+
}
|
|
2699
|
+
/**
|
|
2700
|
+
* Live-worker /restart 携带的最新模型(与 env 同一条通道、同一套三分态)。
|
|
2701
|
+
*
|
|
2702
|
+
* model 不进冻结集合、每次 spawn 按当前 bot 配置解析(见 resolveSessionLaunchModel),
|
|
2703
|
+
* 但 `/restart`、dashboard 重启、CLI 崩溃自动重启这三条路**不 refork**:worker 收到
|
|
2704
|
+
* restart 后用旧 `lastInitConfig` 原地 respawn。不捎带的话,改完模型再重启起来的还是
|
|
2705
|
+
* 旧模型——正是「改配置对存量会话生效」要修的那件事。
|
|
2706
|
+
* 三分态:字符串 = 用它;null = 明确不传模型(bot 未配 / 已清空);
|
|
2707
|
+
* undefined = 取不到(bot 已注销等异常),让 worker 保持快照不动(=旧行为)。
|
|
2708
|
+
*/
|
|
2709
|
+
export function latestModelForRespawn(ds) {
|
|
2710
|
+
try {
|
|
2711
|
+
const model = resolveSessionLaunchModel(ds, getBot(ds.larkAppId).config);
|
|
2712
|
+
// An in-worker respawn IS a launch, so the rule-3 record must converge here
|
|
2713
|
+
// too — otherwise `/restart` starts B while the record still says A, and a
|
|
2714
|
+
// later CLI hot-switch pulls A back out of the record.
|
|
2715
|
+
recordLaunchModel(ds, model);
|
|
2716
|
+
return model ?? null;
|
|
2049
2717
|
}
|
|
2050
2718
|
catch {
|
|
2051
2719
|
return undefined;
|
|
@@ -2060,7 +2728,7 @@ export function requestSessionRestart(ds, observer) {
|
|
|
2060
2728
|
return restartCoordinator.request(ds.session.sessionId, observer, attemptId => {
|
|
2061
2729
|
if (ds.worker && !ds.worker.killed) {
|
|
2062
2730
|
ds.workerReady = false;
|
|
2063
|
-
ds.worker.send({ type: 'restart', attemptId, env: latestPerBotEnvForRestart(ds) });
|
|
2731
|
+
ds.worker.send({ type: 'restart', attemptId, env: latestPerBotEnvForRestart(ds), model: latestModelForRespawn(ds) });
|
|
2064
2732
|
return;
|
|
2065
2733
|
}
|
|
2066
2734
|
// No live worker but the persistent pane may still be alive (e.g. after a
|
|
@@ -2087,7 +2755,69 @@ export function __testOnly_resetRestartCoordinator() {
|
|
|
2087
2755
|
* false worker-side too), so killing it would violate the bridge invariant of
|
|
2088
2756
|
* leaving the user's own CLI untouched.
|
|
2089
2757
|
*/
|
|
2090
|
-
|
|
2758
|
+
/**
|
|
2759
|
+
* Build the launch config for cancelling a mojo session's remote lineage, or say
|
|
2760
|
+
* why it must not be attempted.
|
|
2761
|
+
*
|
|
2762
|
+
* Shared by the best-effort killWorker teardown and the awaited explicit-close
|
|
2763
|
+
* path so the two cannot cancel through DIFFERENT configs — the launch identity
|
|
2764
|
+
* and control plane decide which tenant the cancel reaches, so a second copy of
|
|
2765
|
+
* these rules drifting would silently cancel against the wrong endpoint.
|
|
2766
|
+
*/
|
|
2767
|
+
function resolveMojoCancelLaunch(ds, remoteId, label) {
|
|
2768
|
+
let botCfg;
|
|
2769
|
+
try {
|
|
2770
|
+
botCfg = getBot(ds.larkAppId).config;
|
|
2771
|
+
}
|
|
2772
|
+
catch {
|
|
2773
|
+
// Bot deregistered — nothing to cancel WITH. The caller decides what that
|
|
2774
|
+
// means; the explicit-close path treats it as a RETRYABLE refusal, because
|
|
2775
|
+
// re-registering the bot restores exactly the config this needs.
|
|
2776
|
+
logger.warn(`[${tag(ds)}] ${label}: cannot cancel mojo session ${remoteId} — bot `
|
|
2777
|
+
+ `${ds.larkAppId} is deregistered. Lineage retained for manual cleanup.`);
|
|
2778
|
+
return { ok: false, reason: 'bot_gone' };
|
|
2779
|
+
}
|
|
2780
|
+
// mojo needs no baseUrl gate — the CLI resolves its own endpoint, and an absent
|
|
2781
|
+
// `mojo` config block is a valid setup.
|
|
2782
|
+
//
|
|
2783
|
+
// Build the SAME effective config the worker would (shared helper): a bare
|
|
2784
|
+
// `botCfg.mojo` carries neither the pinned binary (cliPathOverride) nor the
|
|
2785
|
+
// per-bot env holding the JWT, so a bot that ran fine on a custom path/identity
|
|
2786
|
+
// would fail to cancel here — the remote session then keeps burning cloud
|
|
2787
|
+
// sandbox time while still holding injected credentials.
|
|
2788
|
+
//
|
|
2789
|
+
// The launch IDENTITY (binary + wrapper) must come from the values FROZEN on the
|
|
2790
|
+
// session, not from live bot config: the remote session was created through
|
|
2791
|
+
// those, so cancelling through a wrapper the bot gained afterwards can reach the
|
|
2792
|
+
// wrong gateway/tenant. Session cwd is not persisted for a dead worker, so that
|
|
2793
|
+
// one falls back to the bot's configured working dir.
|
|
2794
|
+
const frozenLaunch = frozenSessionLaunchIdentity(ds, botCfg);
|
|
2795
|
+
// freeze:false — teardown must not mutate session state. Cancel has to reach the
|
|
2796
|
+
// endpoint/tenant that CREATED the remote session, so the frozen control plane
|
|
2797
|
+
// matters here as much as the frozen binary.
|
|
2798
|
+
const frozenMojo = sessionMojoConfig(ds, botCfg, { freeze: false });
|
|
2799
|
+
// A quarantined lineage must NOT be cancelled: nothing records which control
|
|
2800
|
+
// plane holds it, so cancelling through the current config could reach a
|
|
2801
|
+
// different tenant. Callers capture `remoteId` BEFORE this call, so clearing the
|
|
2802
|
+
// field would not have stopped it — the explicit state is what makes the refusal
|
|
2803
|
+
// reliable.
|
|
2804
|
+
if (frozenMojo.lineage === 'quarantined') {
|
|
2805
|
+
logger.warn(`[${tag(ds)}] ${label}: NOT cancelling mojo session ${remoteId} — `
|
|
2806
|
+
+ 'its control plane is unverifiable (quarantined). It is preserved on the '
|
|
2807
|
+
+ 'session for manual cleanup.');
|
|
2808
|
+
return { ok: false, reason: 'quarantined' };
|
|
2809
|
+
}
|
|
2810
|
+
return {
|
|
2811
|
+
ok: true,
|
|
2812
|
+
launchCfg: buildEffectiveMojoConfig(frozenMojo.config, {
|
|
2813
|
+
cliPathOverride: frozenLaunch.cliPathOverride,
|
|
2814
|
+
workingDir: ds.session.workingDir ?? botCfg.defaultWorkingDir ?? botCfg.workingDir,
|
|
2815
|
+
env: botCfg.env ? sanitizePerBotEnv(botCfg.env) : undefined,
|
|
2816
|
+
wrapperCli: frozenLaunch.wrapperCli,
|
|
2817
|
+
}),
|
|
2818
|
+
};
|
|
2819
|
+
}
|
|
2820
|
+
function destroyOrphanedBackingSession(ds, opts = {}) {
|
|
2091
2821
|
if (ds.initConfig?.adoptMode || ds.adoptedFrom)
|
|
2092
2822
|
return;
|
|
2093
2823
|
reclaimParkedCrashDiagnostic(ds);
|
|
@@ -2103,6 +2833,54 @@ function destroyOrphanedBackingSession(ds) {
|
|
|
2103
2833
|
}
|
|
2104
2834
|
return;
|
|
2105
2835
|
}
|
|
2836
|
+
if (frozenType === 'mojo') {
|
|
2837
|
+
// Remote backends persist their lineage in the same field riff uses.
|
|
2838
|
+
const remoteId = ds.session.riffParentTaskId;
|
|
2839
|
+
if (!remoteId || opts.skipRemoteCancel)
|
|
2840
|
+
return;
|
|
2841
|
+
const resolved = resolveMojoCancelLaunch(ds, remoteId, 'killWorker');
|
|
2842
|
+
if (!resolved.ok)
|
|
2843
|
+
return;
|
|
2844
|
+
// Best-effort ONLY. The authoritative /close path (prepareMojoExplicitClose)
|
|
2845
|
+
// awaits the same cancellation before publishing the closed row; this branch
|
|
2846
|
+
// exists for the killWorker callers that are not an explicit close (crash
|
|
2847
|
+
// retirement, transfer teardown), where there is no result to report to.
|
|
2848
|
+
//
|
|
2849
|
+
// Clear the lineage ONLY once the remote session is really gone. This used to
|
|
2850
|
+
// clear unconditionally right after firing, so a failed cancel erased the only
|
|
2851
|
+
// id — the exact leak master's riff change above calls out.
|
|
2852
|
+
void cancelMojoSessionById(resolved.launchCfg, remoteId).then((outcome) => {
|
|
2853
|
+
// Structured outcome, not a boolean — `if (!outcome)` would be always-false
|
|
2854
|
+
// and silently treat every failure as a success (an object is truthy).
|
|
2855
|
+
if (!isMojoRemoteGone(outcome)) {
|
|
2856
|
+
logger.warn(`[${tag(ds)}] killWorker: orphan mojo session ${remoteId} NOT cancelled; `
|
|
2857
|
+
+ 'retaining the lineage — note nothing retries this automatically, so an '
|
|
2858
|
+
+ 'explicit /close (which DOES await the cancel) or manual cleanup is required');
|
|
2859
|
+
return;
|
|
2860
|
+
}
|
|
2861
|
+
logger.info(`[${tag(ds)}] killWorker: orphan mojo session ${remoteId} cancelled`);
|
|
2862
|
+
ds.session.riffParentTaskId = undefined;
|
|
2863
|
+
sessionStore.updateSession(ds.session);
|
|
2864
|
+
}).catch((err) => {
|
|
2865
|
+
// Defence in depth for a FIRE-AND-FORGET promise. This helper is
|
|
2866
|
+
// synchronous, so there is no caller to reject to: an escaping rejection
|
|
2867
|
+
// becomes an unhandledRejection, and the daemon installs no handler for
|
|
2868
|
+
// those (only worker.ts does), so Node v22 terminates the process --
|
|
2869
|
+
// taking down every OTHER session this daemon serves. The awaited call
|
|
2870
|
+
// site already guards itself the same way; tmux-pipe-backend.ts documents
|
|
2871
|
+
// a real incident of exactly this shape.
|
|
2872
|
+
//
|
|
2873
|
+
// Reachable, not theoretical: cancelMojoSessionById constructs a backend
|
|
2874
|
+
// before its own try block, so an unreadable containment store throws out
|
|
2875
|
+
// of it rather than returning the structured failure its type promises.
|
|
2876
|
+
// sessionStore.updateSession above can throw for the same reason (a failed
|
|
2877
|
+
// save). Both must degrade to a retained lineage, never to a crash.
|
|
2878
|
+
logger.error(`[${tag(ds)}] killWorker: orphan mojo cancel for ${remoteId} threw; `
|
|
2879
|
+
+ `retaining the lineage for explicit /close or manual cleanup: `
|
|
2880
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
2881
|
+
});
|
|
2882
|
+
return;
|
|
2883
|
+
}
|
|
2106
2884
|
const backendType = getSessionPersistentBackendType(ds);
|
|
2107
2885
|
if (!backendType)
|
|
2108
2886
|
return;
|
|
@@ -2114,16 +2892,16 @@ function destroyOrphanedBackingSession(ds) {
|
|
|
2114
2892
|
logger.warn(`[${tag(ds)}] killWorker: failed to destroy orphaned ${backendType} backing session: ${err}`);
|
|
2115
2893
|
}
|
|
2116
2894
|
}
|
|
2117
|
-
async function
|
|
2118
|
-
if (ds.
|
|
2895
|
+
async function abortLiveRemoteWorkerClose(ds, requestId, opts = {}) {
|
|
2896
|
+
if (ds.remoteCloseState?.requestId !== requestId)
|
|
2119
2897
|
return false;
|
|
2120
2898
|
const worker = ds.worker;
|
|
2121
2899
|
if (!worker || worker.killed) {
|
|
2122
2900
|
if (opts.allowAbsentAfterProvenRestore) {
|
|
2123
|
-
ds.
|
|
2901
|
+
ds.remoteCloseState = undefined;
|
|
2124
2902
|
return true;
|
|
2125
2903
|
}
|
|
2126
|
-
ds.
|
|
2904
|
+
ds.remoteCloseState = { ...ds.remoteCloseState, phase: 'uncertain' };
|
|
2127
2905
|
return false;
|
|
2128
2906
|
}
|
|
2129
2907
|
const restored = await new Promise(resolve => {
|
|
@@ -2145,13 +2923,23 @@ async function abortLiveRiffWorkerClose(ds, requestId, opts = {}) {
|
|
|
2145
2923
|
finish({ ok: false, error: 'stale_worker_generation' });
|
|
2146
2924
|
return;
|
|
2147
2925
|
}
|
|
2148
|
-
|
|
2926
|
+
// `ok` says the abort was HANDLED; `admissionRestored` says whether writes
|
|
2927
|
+
// actually came back. A backend holding a latched fence answers ok:true /
|
|
2928
|
+
// admissionRestored:false, and inferring restoration from `ok` journalled a
|
|
2929
|
+
// cleared fence while write() kept returning false. Absent means `ok`
|
|
2930
|
+
// (legacy backends, riff).
|
|
2931
|
+
finish({
|
|
2932
|
+
ok: msg.ok,
|
|
2933
|
+
admissionRestored: msg.admissionRestored ?? msg.ok,
|
|
2934
|
+
...(msg.fenceReason ? { fenceReason: msg.fenceReason } : {}),
|
|
2935
|
+
...(msg.error ? { error: msg.error } : {}),
|
|
2936
|
+
});
|
|
2149
2937
|
};
|
|
2150
2938
|
const onExit = () => finish({
|
|
2151
2939
|
ok: false,
|
|
2152
2940
|
error: 'worker_exited_before_close_abort_result',
|
|
2153
2941
|
});
|
|
2154
|
-
const timer = setTimeout(() => finish({ ok: false, error: 'close_abort_result_timeout' }),
|
|
2942
|
+
const timer = setTimeout(() => finish({ ok: false, error: 'close_abort_result_timeout' }), REMOTE_ADMISSION_RESTORE_TIMEOUT_MS);
|
|
2155
2943
|
timer.unref?.();
|
|
2156
2944
|
worker.on?.('message', onMessage);
|
|
2157
2945
|
worker.once?.('exit', onExit);
|
|
@@ -2162,39 +2950,192 @@ async function abortLiveRiffWorkerClose(ds, requestId, opts = {}) {
|
|
|
2162
2950
|
finish({ ok: false, error: err instanceof Error ? err.message : String(err) });
|
|
2163
2951
|
}
|
|
2164
2952
|
});
|
|
2165
|
-
|
|
2166
|
-
|
|
2953
|
+
// A handled-but-REFUSED abort must not clear the fence. `ok` alone did, which
|
|
2954
|
+
// is the same laundering as before, one path over.
|
|
2955
|
+
const admissionRestored = restored.ok && restored.admissionRestored !== false;
|
|
2956
|
+
if (admissionRestored && ds.remoteCloseState?.requestId === requestId) {
|
|
2957
|
+
ds.remoteCloseState = undefined;
|
|
2167
2958
|
return true;
|
|
2168
2959
|
}
|
|
2960
|
+
// A proven earlier restore excuses a MISSING ack, never a refused one: the
|
|
2961
|
+
// backend just told us writes are still fenced.
|
|
2169
2962
|
if (opts.allowAbsentAfterProvenRestore
|
|
2170
|
-
&&
|
|
2171
|
-
ds.
|
|
2963
|
+
&& restored.admissionRestored !== false
|
|
2964
|
+
&& ds.remoteCloseState?.requestId === requestId) {
|
|
2965
|
+
ds.remoteCloseState = undefined;
|
|
2172
2966
|
return true;
|
|
2173
2967
|
}
|
|
2174
|
-
if (ds.
|
|
2175
|
-
ds.
|
|
2968
|
+
if (ds.remoteCloseState?.requestId === requestId) {
|
|
2969
|
+
ds.remoteCloseState = { ...ds.remoteCloseState, phase: 'uncertain' };
|
|
2176
2970
|
}
|
|
2177
|
-
logger.warn(`[${tag(ds)}]
|
|
2971
|
+
logger.warn(`[${tag(ds)}] Remote close abort did not restore write admission `
|
|
2972
|
+
+ `(${restored.fenceReason ?? restored.error ?? 'unknown'}); `
|
|
2178
2973
|
+ 'retaining admission fence pending explicit lineage reconciliation');
|
|
2179
2974
|
return false;
|
|
2180
2975
|
}
|
|
2181
|
-
async function
|
|
2976
|
+
async function prepareLiveRemoteWorkerClose(ds, backendType) {
|
|
2182
2977
|
const worker = ds.worker;
|
|
2183
2978
|
if (!worker || worker.killed) {
|
|
2184
|
-
return {
|
|
2979
|
+
return {
|
|
2980
|
+
ok: false,
|
|
2981
|
+
error: backendType === 'riff' ? 'riff_worker_close_failed' : 'mojo_cancel_failed',
|
|
2982
|
+
retryable: true,
|
|
2983
|
+
};
|
|
2185
2984
|
}
|
|
2186
|
-
if (ds.
|
|
2985
|
+
if (backendType === 'mojo' && ds.remoteCloseState?.phase === 'abort_restored') {
|
|
2986
|
+
const restoredState = ds.remoteCloseState;
|
|
2987
|
+
try {
|
|
2988
|
+
const committed = sessionStore.finishMojoCloseAbort(ds.session.sessionId, restoredState.requestId, {
|
|
2989
|
+
admissionRestored: true,
|
|
2990
|
+
...(restoredState.taskId ? { taskId: restoredState.taskId } : {}),
|
|
2991
|
+
});
|
|
2992
|
+
ds.session.mojoCloseJournal = committed.mojoCloseJournal;
|
|
2993
|
+
ds.session.riffParentTaskId = committed.riffParentTaskId;
|
|
2994
|
+
ds.remoteCloseState = undefined;
|
|
2995
|
+
}
|
|
2996
|
+
catch (err) {
|
|
2997
|
+
logger.error(`[${tag(ds)}] Mojo admission restore proof could not clear the durable close journal: `
|
|
2998
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
2999
|
+
return {
|
|
3000
|
+
ok: false,
|
|
3001
|
+
error: 'mojo_durable_close_failed',
|
|
3002
|
+
retryable: true,
|
|
3003
|
+
...(restoredState.taskId ? { taskId: restoredState.taskId } : {}),
|
|
3004
|
+
};
|
|
3005
|
+
}
|
|
3006
|
+
}
|
|
3007
|
+
if (backendType === 'mojo' && ds.remoteCloseState?.phase === 'prepared') {
|
|
3008
|
+
try {
|
|
3009
|
+
const committed = sessionStore.markMojoClosePrepared(ds.session.sessionId, ds.remoteCloseState.requestId, ds.remoteCloseState.taskId,
|
|
3010
|
+
// The retry must republish the SAME proof, residual included — the
|
|
3011
|
+
// original prepare saw the local evidence grade; this replay did not.
|
|
3012
|
+
ds.remoteCloseState.localResidual);
|
|
3013
|
+
ds.session.mojoCloseJournal = committed.mojoCloseJournal;
|
|
3014
|
+
ds.session.riffParentTaskId = committed.riffParentTaskId;
|
|
3015
|
+
}
|
|
3016
|
+
catch (err) {
|
|
3017
|
+
logger.error(`[${tag(ds)}] Mojo close proof could not be persisted for durable commit: `
|
|
3018
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
3019
|
+
return {
|
|
3020
|
+
ok: false,
|
|
3021
|
+
error: 'mojo_durable_close_failed',
|
|
3022
|
+
retryable: true,
|
|
3023
|
+
...(ds.remoteCloseState.taskId ? { taskId: ds.remoteCloseState.taskId } : {}),
|
|
3024
|
+
};
|
|
3025
|
+
}
|
|
3026
|
+
return {
|
|
3027
|
+
ok: true,
|
|
3028
|
+
...(ds.remoteCloseState.taskId ? { taskId: ds.remoteCloseState.taskId } : {}),
|
|
3029
|
+
// Stored residual → derived at read time (see liveLocalResidual): the
|
|
3030
|
+
// journal write above still republishes the original proof verbatim.
|
|
3031
|
+
...(residualField(liveLocalResidual(ds.session.sessionId, ds.remoteCloseState.localResidual))),
|
|
3032
|
+
};
|
|
3033
|
+
}
|
|
3034
|
+
if (ds.remoteShutdownState) {
|
|
2187
3035
|
return {
|
|
2188
3036
|
ok: false,
|
|
2189
|
-
error: '
|
|
3037
|
+
error: 'remote_shutdown_fence_in_progress',
|
|
2190
3038
|
retryable: true,
|
|
2191
|
-
...(
|
|
2192
|
-
? { taskId:
|
|
3039
|
+
...(typeof ds.remoteShutdownState.taskId === 'string' && ds.remoteShutdownState.taskId
|
|
3040
|
+
? { taskId: ds.remoteShutdownState.taskId }
|
|
2193
3041
|
: {}),
|
|
2194
3042
|
};
|
|
2195
3043
|
}
|
|
3044
|
+
let takenOverRuntimeState;
|
|
3045
|
+
if (ds.remoteCloseState) {
|
|
3046
|
+
const runtimeUncertain = backendType === 'mojo'
|
|
3047
|
+
&& ds.remoteCloseState.phase === 'uncertain';
|
|
3048
|
+
// A runtime `uncertain` fence used to be a dead end: "a blind retry proves
|
|
3049
|
+
// nothing" was true when the journal refused every restart, so /close
|
|
3050
|
+
// failed with retryable:false for the daemon's whole lifetime and the only
|
|
3051
|
+
// exit was a daemon restart — a process-level soft deadlock (third-round
|
|
3052
|
+
// review, gate 1). The retry is no longer blind: an explicit close with a
|
|
3053
|
+
// LIVE worker re-runs the full destroySession, which can produce fresh
|
|
3054
|
+
// evidence. When the runtime fence and the DURABLE journal describe the
|
|
3055
|
+
// same failed attempt — same phase, same requestId, same lineage — the
|
|
3056
|
+
// fence is cleared and the close re-enters prepare under a new requestId
|
|
3057
|
+
// (beginMojoCloseJournal accepts the uncertain takeover). Anything less
|
|
3058
|
+
// than an exact match keeps the refusal: a mismatched pair means runtime
|
|
3059
|
+
// and disk disagree about which attempt happened, and clearing on that
|
|
3060
|
+
// would launder an unknown state.
|
|
3061
|
+
const durableForTakeover = ds.session.mojoCloseJournal;
|
|
3062
|
+
// Three legs, each covering a REAL production writer of this pair
|
|
3063
|
+
// (fourth-round review named the two the exact-match version missed):
|
|
3064
|
+
// phase — the runtime fence is `uncertain` for every non-irreversible
|
|
3065
|
+
// failed prepare, while the DURABLE phase differs by verdict:
|
|
3066
|
+
// `uncertain` for an uncertain one, `preparing` (with
|
|
3067
|
+
// recovery 'retryable') for a retryable-but-fenced one. Both
|
|
3068
|
+
// pairs describe the same failed attempt; matching only
|
|
3069
|
+
// uncertain/uncertain left retryable+fenced soft-deadlocked
|
|
3070
|
+
// with a durable row that itself promises a retry (gap A).
|
|
3071
|
+
// requestId — exact, always: both sides were written by that attempt.
|
|
3072
|
+
// lineage — equal, or the durable side is a superset (the worker may
|
|
3073
|
+
// have reported the pre-init lineage into the journal while
|
|
3074
|
+
// the runtime fence kept none, gap B). A CONFLICTING pair
|
|
3075
|
+
// still refuses: runtime and disk disagreeing about which
|
|
3076
|
+
// remote session was touched must never be laundered.
|
|
3077
|
+
const durablePhaseMatches = durableForTakeover?.phase === 'uncertain'
|
|
3078
|
+
|| (durableForTakeover?.phase === 'preparing'
|
|
3079
|
+
&& durableForTakeover.recovery === 'retryable');
|
|
3080
|
+
const runtimeTaskId = ds.remoteCloseState.taskId ?? undefined;
|
|
3081
|
+
const durableTaskId = durableForTakeover?.taskId ?? undefined;
|
|
3082
|
+
const takeoverEligible = runtimeUncertain
|
|
3083
|
+
&& durableForTakeover !== undefined
|
|
3084
|
+
&& durablePhaseMatches
|
|
3085
|
+
&& durableForTakeover.requestId === ds.remoteCloseState.requestId
|
|
3086
|
+
&& (durableTaskId === runtimeTaskId || runtimeTaskId === undefined);
|
|
3087
|
+
if (takeoverEligible) {
|
|
3088
|
+
logger.warn(`[${tag(ds)}] explicit close takes over ${durableForTakeover.phase} Mojo journal `
|
|
3089
|
+
+ `(${ds.remoteCloseState.requestId}); re-running the cancel through the live worker`);
|
|
3090
|
+
// Cleared only in memory here; restored if the fresh attempt's durable
|
|
3091
|
+
// write below fails, so the runtime fence (and everything keyed on it,
|
|
3092
|
+
// e.g. the shutdown-detach explicit-close guard) never vanishes without
|
|
3093
|
+
// a successor journal actually on disk.
|
|
3094
|
+
takenOverRuntimeState = ds.remoteCloseState;
|
|
3095
|
+
ds.remoteCloseState = undefined;
|
|
3096
|
+
}
|
|
3097
|
+
else if (runtimeUncertain) {
|
|
3098
|
+
return {
|
|
3099
|
+
ok: false,
|
|
3100
|
+
error: 'mojo_close_reconciliation_required',
|
|
3101
|
+
retryable: false,
|
|
3102
|
+
recovery: 'uncertain',
|
|
3103
|
+
...(ds.remoteCloseState.taskId ? { taskId: ds.remoteCloseState.taskId } : {}),
|
|
3104
|
+
};
|
|
3105
|
+
}
|
|
3106
|
+
else {
|
|
3107
|
+
return {
|
|
3108
|
+
ok: false,
|
|
3109
|
+
error: backendType === 'riff' ? 'riff_worker_close_failed' : 'mojo_cancel_failed',
|
|
3110
|
+
retryable: true,
|
|
3111
|
+
...(ds.remoteCloseState.taskId ? { taskId: ds.remoteCloseState.taskId } : {}),
|
|
3112
|
+
};
|
|
3113
|
+
}
|
|
3114
|
+
}
|
|
2196
3115
|
const requestId = randomUUID();
|
|
2197
|
-
|
|
3116
|
+
if (backendType === 'mojo') {
|
|
3117
|
+
try {
|
|
3118
|
+
const committed = sessionStore.beginMojoCloseJournal(ds.session.sessionId, requestId, ds.session.riffParentTaskId);
|
|
3119
|
+
ds.session.mojoCloseJournal = committed.mojoCloseJournal;
|
|
3120
|
+
}
|
|
3121
|
+
catch (err) {
|
|
3122
|
+
// Reinstate a runtime fence the takeover cleared: without this, a failed
|
|
3123
|
+
// durable write left the ds fence-less while disk still said uncertain,
|
|
3124
|
+
// and guards keyed on ds.remoteCloseState (shutdown-detach's
|
|
3125
|
+
// explicit-close check) silently lost their signal.
|
|
3126
|
+
if (takenOverRuntimeState)
|
|
3127
|
+
ds.remoteCloseState = takenOverRuntimeState;
|
|
3128
|
+
logger.error(`[${tag(ds)}] Mojo close journal could not be persisted before cancellation: `
|
|
3129
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
3130
|
+
return {
|
|
3131
|
+
ok: false,
|
|
3132
|
+
error: 'mojo_durable_close_failed',
|
|
3133
|
+
retryable: true,
|
|
3134
|
+
...(ds.session.riffParentTaskId ? { taskId: ds.session.riffParentTaskId } : {}),
|
|
3135
|
+
};
|
|
3136
|
+
}
|
|
3137
|
+
}
|
|
3138
|
+
ds.remoteCloseState = {
|
|
2198
3139
|
phase: 'preparing',
|
|
2199
3140
|
requestId,
|
|
2200
3141
|
...(ds.session.riffParentTaskId ? { taskId: ds.session.riffParentTaskId } : {}),
|
|
@@ -2208,17 +3149,29 @@ async function prepareLiveRiffWorkerClose(ds) {
|
|
|
2208
3149
|
settled = true;
|
|
2209
3150
|
clearTimeout(timer);
|
|
2210
3151
|
worker.removeListener?.('exit', onExit);
|
|
2211
|
-
|
|
3152
|
+
pendingRemoteWorkerCloses.delete(requestId);
|
|
2212
3153
|
resolve(value);
|
|
2213
3154
|
};
|
|
3155
|
+
// The worker died mid-prepare, so whatever destroySession() had already done
|
|
3156
|
+
// is unknown -- including a completed irreversible cancel. Fence.
|
|
2214
3157
|
const onExit = () => finish({
|
|
2215
3158
|
ok: false,
|
|
2216
3159
|
error: 'worker_exited_before_close_result',
|
|
3160
|
+
recovery: 'uncertain',
|
|
2217
3161
|
});
|
|
2218
|
-
|
|
3162
|
+
// Riff's adapter has its own 20s destroy deadline. Mojo may first spend 8s
|
|
3163
|
+
// waiting for system/init, then up to 60s in `mojo session cancel`; aborting
|
|
3164
|
+
// at the old 23s Riff budget re-opened admission while that irreversible
|
|
3165
|
+
// cancellation was still in flight. Stay above both bounded Mojo phases.
|
|
3166
|
+
const closeResultTimeoutMs = backendType === 'mojo'
|
|
3167
|
+
? MOJO_EXPLICIT_CLOSE_RESULT_TIMEOUT_MS
|
|
3168
|
+
: 23_000;
|
|
3169
|
+
const timer = setTimeout(
|
|
3170
|
+
// A timed-out prepare may still be running remotely; the outcome is unknown.
|
|
3171
|
+
() => finish({ ok: false, error: 'worker_close_result_timeout', recovery: 'uncertain' }), closeResultTimeoutMs);
|
|
2219
3172
|
timer.unref?.();
|
|
2220
3173
|
worker.once?.('exit', onExit);
|
|
2221
|
-
|
|
3174
|
+
pendingRemoteWorkerCloses.set(requestId, {
|
|
2222
3175
|
sessionId: ds.session.sessionId,
|
|
2223
3176
|
worker,
|
|
2224
3177
|
resolve: value => {
|
|
@@ -2234,14 +3187,223 @@ async function prepareLiveRiffWorkerClose(ds) {
|
|
|
2234
3187
|
}
|
|
2235
3188
|
});
|
|
2236
3189
|
const taskId = result.taskId ?? ds.session.riffParentTaskId;
|
|
3190
|
+
if (backendType === 'mojo') {
|
|
3191
|
+
if (result.ok) {
|
|
3192
|
+
try {
|
|
3193
|
+
const committed = sessionStore.markMojoClosePrepared(ds.session.sessionId, requestId, taskId,
|
|
3194
|
+
// Durable ON PURPOSE: the residual is part of the close outcome, and a
|
|
3195
|
+
// daemon restart replaying this prepared journal must still publish
|
|
3196
|
+
// `closed_with_residual` (only the worker saw the evidence grade).
|
|
3197
|
+
result.residual);
|
|
3198
|
+
ds.session.mojoCloseJournal = committed.mojoCloseJournal;
|
|
3199
|
+
ds.session.riffParentTaskId = committed.riffParentTaskId;
|
|
3200
|
+
}
|
|
3201
|
+
catch (err) {
|
|
3202
|
+
// Cancellation is irreversible. The runtime proof stays prepared and
|
|
3203
|
+
// admission remains fenced; a retry republishes that same proof without
|
|
3204
|
+
// asking the worker to cancel twice — residual included, or the retry
|
|
3205
|
+
// would silently downgrade a residual close to a plain one. The durable
|
|
3206
|
+
// `preparing` journal makes a daemon crash fail closed rather than
|
|
3207
|
+
// resume this generation.
|
|
3208
|
+
ds.remoteCloseState = {
|
|
3209
|
+
phase: 'prepared',
|
|
3210
|
+
requestId,
|
|
3211
|
+
...(taskId ? { taskId } : {}),
|
|
3212
|
+
...(result.residual ? { localResidual: result.residual } : {}),
|
|
3213
|
+
};
|
|
3214
|
+
logger.error(`[${tag(ds)}] Mojo close proof persistence failed after cancellation; `
|
|
3215
|
+
+ `retaining prepared commit: ${err instanceof Error ? err.message : String(err)}`);
|
|
3216
|
+
return {
|
|
3217
|
+
ok: false,
|
|
3218
|
+
error: 'mojo_durable_close_failed',
|
|
3219
|
+
retryable: true,
|
|
3220
|
+
...(taskId ? { taskId } : {}),
|
|
3221
|
+
};
|
|
3222
|
+
}
|
|
3223
|
+
ds.remoteCloseState = {
|
|
3224
|
+
phase: 'prepared',
|
|
3225
|
+
requestId,
|
|
3226
|
+
...(taskId ? { taskId } : {}),
|
|
3227
|
+
...(result.residual ? { localResidual: result.residual } : {}),
|
|
3228
|
+
};
|
|
3229
|
+
logger.info(`[${tag(ds)}] mojo worker close prepared and remote cancellation confirmed`);
|
|
3230
|
+
// The remote lineage is confirmed cancelled, but that says nothing about the
|
|
3231
|
+
// LOCAL process tree. When the backend could not prove its subtree gone it
|
|
3232
|
+
// kept the containment handle, so this row must NOT be published as an
|
|
3233
|
+
// ordinary closed one: the device-isolation blocker is still live and an
|
|
3234
|
+
// operator reading "closed" would believe a credentialed process was gone.
|
|
3235
|
+
//
|
|
3236
|
+
// Kept distinct from the remote lineage residual on purpose. That one names a
|
|
3237
|
+
// surviving remote task id and asks for remote cleanup; this one names a host
|
|
3238
|
+
// subtree and asks for local cleanup, and collapsing them would send the
|
|
3239
|
+
// operator after the wrong thing. Hence no taskId here even when one exists —
|
|
3240
|
+
// the remote side really was cancelled.
|
|
3241
|
+
if (result.residual) {
|
|
3242
|
+
logger.warn(`[${tag(ds)}] mojo remote cancellation confirmed, but the LOCAL subtree was not `
|
|
3243
|
+
+ `proven gone (${result.residual}); publishing a residual close so the `
|
|
3244
|
+
+ 'device-isolation blocker stays visible');
|
|
3245
|
+
return {
|
|
3246
|
+
ok: true,
|
|
3247
|
+
...(taskId ? { taskId } : {}),
|
|
3248
|
+
residual: { reason: result.residual },
|
|
3249
|
+
};
|
|
3250
|
+
}
|
|
3251
|
+
return { ok: true, ...(taskId ? { taskId } : {}) };
|
|
3252
|
+
}
|
|
3253
|
+
// Only a REVERSIBLE failure may be rolled back. Sending close_abort for every
|
|
3254
|
+
// ok:false re-opened write admission after an `uncertain` verdict (an unnamed
|
|
3255
|
+
// remote session may exist) and after an `irreversible` one (the remote side is
|
|
3256
|
+
// already gone), which laundered both back into `retryable` end to end.
|
|
3257
|
+
const recovery = result.recovery ?? 'retryable';
|
|
3258
|
+
// Whether writes may be admitted is a DIFFERENT question from whether the
|
|
3259
|
+
// close may be retried. Keying the rollback on `recovery` re-opened admission
|
|
3260
|
+
// for an unproven local child termination (retryable, yet a credentialed
|
|
3261
|
+
// process may still be alive); absent `admission` keeps the historical
|
|
3262
|
+
// derivation so riff and older workers are unaffected.
|
|
3263
|
+
// Delegated to the shared helper rather than re-implementing the derivation:
|
|
3264
|
+
// two copies of this rule is how the daemon drifted back to reading
|
|
3265
|
+
// `recovery` and re-opened admission on a latched fence.
|
|
3266
|
+
const mayRestore = mayRestoreWriteAdmission({
|
|
3267
|
+
ok: false,
|
|
3268
|
+
...(result.recovery ? { recovery: result.recovery } : {}),
|
|
3269
|
+
...(result.admission ? { admission: result.admission } : {}),
|
|
3270
|
+
});
|
|
3271
|
+
if (!mayRestore) {
|
|
3272
|
+
// No abort: the runtime fence stays up, so the session remains active and
|
|
3273
|
+
// keeps its device-isolation blocker. But the DURABLE row must also stop
|
|
3274
|
+
// saying `preparing` with the pre-prepare task id -- a crash here left a
|
|
3275
|
+
// journal that could not be told apart from an interrupted cancel, and
|
|
3276
|
+
// silently discarded the exact lineage the worker just reported.
|
|
3277
|
+
const irreversible = recovery === 'irreversible';
|
|
3278
|
+
let journalled = false;
|
|
3279
|
+
try {
|
|
3280
|
+
const committed = sessionStore.markMojoCloseUnresolved(ds.session.sessionId, requestId, {
|
|
3281
|
+
recovery,
|
|
3282
|
+
// The EXACT id from close_result (the pre-init window means the worker
|
|
3283
|
+
// may be the first to know it), not the pre-prepare guess.
|
|
3284
|
+
...(taskId ? { taskId } : {}),
|
|
3285
|
+
// Persisted verbatim and separately from `recovery`: no close_abort
|
|
3286
|
+
// was sent, so writes are still fenced even when the close itself is
|
|
3287
|
+
// retryable. Re-deriving this on restore is what re-opened admission.
|
|
3288
|
+
admission: 'fenced',
|
|
3289
|
+
});
|
|
3290
|
+
ds.session.mojoCloseJournal = committed.mojoCloseJournal;
|
|
3291
|
+
ds.session.riffParentTaskId = committed.riffParentTaskId;
|
|
3292
|
+
journalled = true;
|
|
3293
|
+
}
|
|
3294
|
+
catch (err) {
|
|
3295
|
+
// Fail closed: keep the runtime fence exactly as strict as the disk state
|
|
3296
|
+
// is ambiguous. Never fall back to a rollback.
|
|
3297
|
+
logger.error(`[${tag(ds)}] Mojo ${recovery} close verdict could not be journalled: `
|
|
3298
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
3299
|
+
}
|
|
3300
|
+
// An irreversible teardown leaves ONLY the local commit outstanding, so the
|
|
3301
|
+
// runtime state must be `prepared` -- parking it at `uncertain` made every
|
|
3302
|
+
// retry bounce off the fence and the row could never be closed at all.
|
|
3303
|
+
// NOTE: the runtime phase deliberately stays `uncertain` for everything
|
|
3304
|
+
// that is not a journalled irreversible teardown. A retryable-but-fenced
|
|
3305
|
+
// close is recorded as `preparing` in the DURABLE journal (a restart may
|
|
3306
|
+
// retry the cancel), but within this process the reconciliation gate
|
|
3307
|
+
// refuses a retry either way, so distinguishing the runtime phase here
|
|
3308
|
+
// would be an untestable claim.
|
|
3309
|
+
ds.remoteCloseState = {
|
|
3310
|
+
phase: irreversible && journalled ? 'prepared' : 'uncertain',
|
|
3311
|
+
requestId,
|
|
3312
|
+
...(taskId ? { taskId } : {}),
|
|
3313
|
+
};
|
|
3314
|
+
logger.error(`[${tag(ds)}] mojo worker close prepare failed as ${recovery} `
|
|
3315
|
+
+ `(${result.error ?? 'unknown'}); admission stays fenced, `
|
|
3316
|
+
+ `journal=${journalled ? ds.session.mojoCloseJournal?.phase ?? 'unknown' : 'unwritten'}`);
|
|
3317
|
+
if (!journalled) {
|
|
3318
|
+
return {
|
|
3319
|
+
ok: false,
|
|
3320
|
+
error: 'mojo_durable_close_failed',
|
|
3321
|
+
retryable: true,
|
|
3322
|
+
recovery,
|
|
3323
|
+
...(taskId ? { taskId } : {}),
|
|
3324
|
+
};
|
|
3325
|
+
}
|
|
3326
|
+
// Fenced but still retryable (an unproven LOCAL termination; the
|
|
3327
|
+
// irreversible remote cancel never ran). The close may be retried as a
|
|
3328
|
+
// close -- reporting it as `mojo_close_reconciliation_required` would
|
|
3329
|
+
// strand a session a plain retry can still finish.
|
|
3330
|
+
if (recovery === 'retryable') {
|
|
3331
|
+
return {
|
|
3332
|
+
ok: false,
|
|
3333
|
+
error: 'mojo_cancel_failed',
|
|
3334
|
+
retryable: true,
|
|
3335
|
+
recovery,
|
|
3336
|
+
...(taskId ? { taskId } : {}),
|
|
3337
|
+
};
|
|
3338
|
+
}
|
|
3339
|
+
return irreversible
|
|
3340
|
+
// Retryable, but only as a LOCAL commit: the durable journal is now
|
|
3341
|
+
// commit-only, so a retry can never issue a second cancel.
|
|
3342
|
+
? {
|
|
3343
|
+
ok: false,
|
|
3344
|
+
error: 'mojo_close_commit_required',
|
|
3345
|
+
retryable: true,
|
|
3346
|
+
recovery,
|
|
3347
|
+
...(taskId ? { taskId } : {}),
|
|
3348
|
+
}
|
|
3349
|
+
// Unknown outcome: a blind retry proves nothing. Demand reconciliation
|
|
3350
|
+
// instead of advertising a retry that cannot succeed.
|
|
3351
|
+
: {
|
|
3352
|
+
ok: false,
|
|
3353
|
+
error: 'mojo_close_reconciliation_required',
|
|
3354
|
+
retryable: false,
|
|
3355
|
+
recovery,
|
|
3356
|
+
...(taskId ? { taskId } : {}),
|
|
3357
|
+
};
|
|
3358
|
+
}
|
|
3359
|
+
const admissionRestored = await abortLiveRemoteWorkerClose(ds, requestId, {
|
|
3360
|
+
allowAbsentAfterProvenRestore: matchedCloseResult,
|
|
3361
|
+
});
|
|
3362
|
+
// NOTE: no separate "refused" branch here. finishMojoCloseAbort already keeps
|
|
3363
|
+
// a durable fenced row when admissionRestored is false; the bug was never its
|
|
3364
|
+
// behaviour but the CALLER inventing that flag from `close_abort_result.ok`.
|
|
3365
|
+
// An extra branch differing only in which non-clearing row it writes was not
|
|
3366
|
+
// distinguishable by any test, so it is not claimed as a fix.
|
|
3367
|
+
try {
|
|
3368
|
+
const committed = sessionStore.finishMojoCloseAbort(ds.session.sessionId, requestId, { admissionRestored, ...(taskId ? { taskId } : {}) });
|
|
3369
|
+
ds.session.mojoCloseJournal = committed.mojoCloseJournal;
|
|
3370
|
+
ds.session.riffParentTaskId = committed.riffParentTaskId;
|
|
3371
|
+
}
|
|
3372
|
+
catch (err) {
|
|
3373
|
+
// The worker may have restored admission, but without a durable journal
|
|
3374
|
+
// clear a crash would still be ambiguous. Keep the daemon fence aligned
|
|
3375
|
+
// with the disk-side uncertainty instead of accepting new input.
|
|
3376
|
+
ds.remoteCloseState = {
|
|
3377
|
+
phase: admissionRestored ? 'abort_restored' : 'uncertain',
|
|
3378
|
+
requestId,
|
|
3379
|
+
...(taskId ? { taskId } : {}),
|
|
3380
|
+
};
|
|
3381
|
+
logger.error(`[${tag(ds)}] Mojo close abort could not be committed durably: `
|
|
3382
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
3383
|
+
return {
|
|
3384
|
+
ok: false,
|
|
3385
|
+
error: 'mojo_durable_close_failed',
|
|
3386
|
+
retryable: true,
|
|
3387
|
+
...(taskId ? { taskId } : {}),
|
|
3388
|
+
};
|
|
3389
|
+
}
|
|
3390
|
+
logger.warn(`[${tag(ds)}] mojo worker close prepare failed: ${result.error ?? 'unknown'}; `
|
|
3391
|
+
+ `session remains active${taskId ? ` (task ${taskId})` : ''}`);
|
|
3392
|
+
return {
|
|
3393
|
+
ok: false,
|
|
3394
|
+
error: 'mojo_cancel_failed',
|
|
3395
|
+
retryable: true,
|
|
3396
|
+
...(taskId ? { taskId } : {}),
|
|
3397
|
+
};
|
|
3398
|
+
}
|
|
2237
3399
|
if (result.taskId) {
|
|
2238
3400
|
ds.session.riffParentTaskId = result.taskId;
|
|
2239
3401
|
try {
|
|
2240
3402
|
sessionStore.updateSession(ds.session);
|
|
2241
3403
|
}
|
|
2242
3404
|
catch (err) {
|
|
2243
|
-
await
|
|
2244
|
-
logger.error(`[${tag(ds)}]
|
|
3405
|
+
await abortLiveRemoteWorkerClose(ds, requestId);
|
|
3406
|
+
logger.error(`[${tag(ds)}] ${backendType} close lineage persistence failed; close aborted: `
|
|
2245
3407
|
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
2246
3408
|
return {
|
|
2247
3409
|
ok: false,
|
|
@@ -2252,10 +3414,10 @@ async function prepareLiveRiffWorkerClose(ds) {
|
|
|
2252
3414
|
}
|
|
2253
3415
|
}
|
|
2254
3416
|
if (!result.ok) {
|
|
2255
|
-
await
|
|
3417
|
+
await abortLiveRemoteWorkerClose(ds, requestId, {
|
|
2256
3418
|
allowAbsentAfterProvenRestore: matchedCloseResult,
|
|
2257
3419
|
});
|
|
2258
|
-
logger.warn(`[${tag(ds)}]
|
|
3420
|
+
logger.warn(`[${tag(ds)}] ${backendType} worker close prepare failed: ${result.error ?? 'unknown'}; `
|
|
2259
3421
|
+ `session remains active${taskId ? ` (task ${taskId})` : ''}`);
|
|
2260
3422
|
return {
|
|
2261
3423
|
ok: false,
|
|
@@ -2264,12 +3426,12 @@ async function prepareLiveRiffWorkerClose(ds) {
|
|
|
2264
3426
|
...(taskId ? { taskId } : {}),
|
|
2265
3427
|
};
|
|
2266
3428
|
}
|
|
2267
|
-
ds.
|
|
3429
|
+
ds.remoteCloseState = {
|
|
2268
3430
|
phase: 'prepared',
|
|
2269
3431
|
requestId,
|
|
2270
3432
|
...(taskId ? { taskId } : {}),
|
|
2271
3433
|
};
|
|
2272
|
-
logger.info(`[${tag(ds)}]
|
|
3434
|
+
logger.info(`[${tag(ds)}] ${backendType} worker close prepared and remote cancellation confirmed`);
|
|
2273
3435
|
return { ok: true, ...(taskId ? { taskId } : {}) };
|
|
2274
3436
|
}
|
|
2275
3437
|
/** Await remote cancellation for any Riff owner before its durable row is
|
|
@@ -2281,23 +3443,23 @@ async function prepareRiffExplicitClose(ds, stored) {
|
|
|
2281
3443
|
return { ok: true };
|
|
2282
3444
|
const backendType = ds?.initConfig?.backendType ?? session.backendType;
|
|
2283
3445
|
const taskId = session.riffParentTaskId;
|
|
2284
|
-
if (ds?.
|
|
2285
|
-
const fencedTaskId = Object.prototype.hasOwnProperty.call(ds.
|
|
2286
|
-
? ds.
|
|
3446
|
+
if (ds?.remoteShutdownState) {
|
|
3447
|
+
const fencedTaskId = Object.prototype.hasOwnProperty.call(ds.remoteShutdownState, 'taskId')
|
|
3448
|
+
? ds.remoteShutdownState.taskId
|
|
2287
3449
|
: taskId;
|
|
2288
3450
|
return {
|
|
2289
3451
|
ok: false,
|
|
2290
|
-
error: '
|
|
3452
|
+
error: 'remote_shutdown_fence_in_progress',
|
|
2291
3453
|
retryable: true,
|
|
2292
3454
|
...(typeof fencedTaskId === 'string' && fencedTaskId ? { taskId: fencedTaskId } : {}),
|
|
2293
3455
|
};
|
|
2294
3456
|
}
|
|
2295
|
-
if (ds?.
|
|
3457
|
+
if (ds?.remoteCloseState) {
|
|
2296
3458
|
return {
|
|
2297
3459
|
ok: false,
|
|
2298
3460
|
error: 'riff_close_reconciliation_required',
|
|
2299
3461
|
retryable: true,
|
|
2300
|
-
...(ds.
|
|
3462
|
+
...(ds.remoteCloseState.taskId ? { taskId: ds.remoteCloseState.taskId } : {}),
|
|
2301
3463
|
};
|
|
2302
3464
|
}
|
|
2303
3465
|
if (backendType !== 'riff')
|
|
@@ -2313,7 +3475,7 @@ async function prepareRiffExplicitClose(ds, stored) {
|
|
|
2313
3475
|
...(taskId ? { taskId } : {}),
|
|
2314
3476
|
};
|
|
2315
3477
|
}
|
|
2316
|
-
return
|
|
3478
|
+
return prepareLiveRemoteWorkerClose(ds, 'riff');
|
|
2317
3479
|
}
|
|
2318
3480
|
const durableBefore = sessionStore.getSession(session.sessionId);
|
|
2319
3481
|
if (ds && durableBefore
|
|
@@ -2322,57 +3484,511 @@ async function prepareRiffExplicitClose(ds, stored) {
|
|
|
2322
3484
|
logger.warn(`[${tag(ds)}] explicit close refused before cancellation: runtime/durable Riff lineage differs `
|
|
2323
3485
|
+ `(${ds.session.riffParentTaskId ?? 'none'}/${durableBefore.riffParentTaskId ?? 'none'})`);
|
|
2324
3486
|
return {
|
|
2325
|
-
ok: false,
|
|
2326
|
-
error: 'riff_task_changed',
|
|
2327
|
-
retryable: true,
|
|
2328
|
-
...(authoritativeTaskId ? { taskId: authoritativeTaskId } : {}),
|
|
3487
|
+
ok: false,
|
|
3488
|
+
error: 'riff_task_changed',
|
|
3489
|
+
retryable: true,
|
|
3490
|
+
...(authoritativeTaskId ? { taskId: authoritativeTaskId } : {}),
|
|
3491
|
+
};
|
|
3492
|
+
}
|
|
3493
|
+
if (!taskId)
|
|
3494
|
+
return { ok: true };
|
|
3495
|
+
let riffConfig;
|
|
3496
|
+
const larkAppId = ds?.larkAppId ?? session.larkAppId;
|
|
3497
|
+
try {
|
|
3498
|
+
if (larkAppId)
|
|
3499
|
+
riffConfig = getBot(larkAppId).config.riff;
|
|
3500
|
+
}
|
|
3501
|
+
catch { /* bot removed */ }
|
|
3502
|
+
const cleanup = await cleanupExplicitSessionBacking({
|
|
3503
|
+
sessionId: session.sessionId,
|
|
3504
|
+
backendType,
|
|
3505
|
+
riffParentTaskId: taskId,
|
|
3506
|
+
riffConfig,
|
|
3507
|
+
});
|
|
3508
|
+
const closeLabel = ds ? tag(ds) : session.sessionId.slice(0, 8);
|
|
3509
|
+
if (!cleanup.ok) {
|
|
3510
|
+
logger.warn(`[${closeLabel}] explicit close refused: ${cleanup.kind}; `
|
|
3511
|
+
+ `Riff task ${taskId} remains active and retryable`);
|
|
3512
|
+
return {
|
|
3513
|
+
ok: false,
|
|
3514
|
+
error: cleanup.kind === 'riff_config_missing' ? 'riff_config_missing' : 'riff_cancel_failed',
|
|
3515
|
+
retryable: true,
|
|
3516
|
+
taskId,
|
|
3517
|
+
};
|
|
3518
|
+
}
|
|
3519
|
+
if (cleanup.kind !== 'cancelled_riff')
|
|
3520
|
+
return { ok: true };
|
|
3521
|
+
const latest = sessionStore.getSession(session.sessionId);
|
|
3522
|
+
const latestTaskId = latest?.riffParentTaskId;
|
|
3523
|
+
const runtimeTaskId = ds?.session.riffParentTaskId;
|
|
3524
|
+
if (!latest || latest.status !== 'active' || latestTaskId !== cleanup.taskId
|
|
3525
|
+
|| (ds && runtimeTaskId !== cleanup.taskId)) {
|
|
3526
|
+
logger.warn(`[${closeLabel}] explicit close cancelled stale Riff task ${cleanup.taskId}, `
|
|
3527
|
+
+ `but runtime/durable lineage or status changed to `
|
|
3528
|
+
+ `${runtimeTaskId ?? 'none'}/${latestTaskId ?? 'none'}/${latest?.status ?? 'missing'}; retry required`);
|
|
3529
|
+
return {
|
|
3530
|
+
ok: false,
|
|
3531
|
+
error: 'riff_task_changed',
|
|
3532
|
+
retryable: true,
|
|
3533
|
+
taskId: latestTaskId ?? runtimeTaskId ?? cleanup.taskId,
|
|
3534
|
+
};
|
|
3535
|
+
}
|
|
3536
|
+
logger.info(`[${closeLabel}] Riff task ${cleanup.taskId} cancellation prepared for explicit close`);
|
|
3537
|
+
return { ok: true, taskId: cleanup.taskId };
|
|
3538
|
+
}
|
|
3539
|
+
/**
|
|
3540
|
+
* Prove a mojo session's remote lineage is cancelled BEFORE its row is published
|
|
3541
|
+
* as closed — the mojo half of what prepareRiffExplicitClose does.
|
|
3542
|
+
*
|
|
3543
|
+
* Without this, `/close` on a workerless mojo session marked the row closed and
|
|
3544
|
+
* returned success while the cancel was still in flight, so a failed cancel left
|
|
3545
|
+
* the operator believing the session was gone while the remote one kept running
|
|
3546
|
+
* and holding the injected credential. The retained lineage was not a recovery
|
|
3547
|
+
* path either: cancelMojoSessionById has exactly one call site (the best-effort
|
|
3548
|
+
* killWorker teardown), which a second `/close` can no longer reach because the
|
|
3549
|
+
* first close removed the session from the active registry.
|
|
3550
|
+
*
|
|
3551
|
+
* A live worker uses prepare/commit: MojoBackend.destroySession() awaits the
|
|
3552
|
+
* cancel (including the pre-init window), then the daemon durably closes the row
|
|
3553
|
+
* before committing worker retirement. A failed durable write keeps the exact
|
|
3554
|
+
* generation prepared so retry commits without cancelling twice.
|
|
3555
|
+
*
|
|
3556
|
+
* An open durable row with no entry in the active registry cannot be cancelled
|
|
3557
|
+
* from here at all (every helper the cancel needs is keyed on DaemonSession), so
|
|
3558
|
+
* it is refused as retryable rather than published as closed — restoring the owner
|
|
3559
|
+
* makes the retry work.
|
|
3560
|
+
*/
|
|
3561
|
+
async function prepareMojoExplicitClose(ds, stored) {
|
|
3562
|
+
const session = ds?.session ?? stored;
|
|
3563
|
+
if (!session)
|
|
3564
|
+
return { ok: true };
|
|
3565
|
+
if (ds?.initConfig?.adoptMode || ds?.adoptedFrom || session.adoptedFrom)
|
|
3566
|
+
return { ok: true };
|
|
3567
|
+
// A row whose lineage was PARKED (restore-time quarantine moves the id here and
|
|
3568
|
+
// clears the active slot) still has a remote session running. It is reported as
|
|
3569
|
+
// a residual on every close of this row, including replays of an already-closed
|
|
3570
|
+
// one — which is what makes the outcome idempotent instead of degrading to a
|
|
3571
|
+
// failure once the owner is gone.
|
|
3572
|
+
const parked = mojoCloseResidualForRow(stored ?? session);
|
|
3573
|
+
const remoteId = stored ? stored.riffParentTaskId : session.riffParentTaskId;
|
|
3574
|
+
const runtimeClose = ds?.session.mojoCloseJournal;
|
|
3575
|
+
const storedClose = stored?.mojoCloseJournal;
|
|
3576
|
+
if (ds && stored && ds.session !== stored && runtimeClose) {
|
|
3577
|
+
const runtimeMatchesDurable = !!storedClose
|
|
3578
|
+
&& runtimeClose.phase === storedClose.phase
|
|
3579
|
+
&& runtimeClose.requestId === storedClose.requestId
|
|
3580
|
+
&& runtimeClose.taskId === storedClose.taskId
|
|
3581
|
+
&& runtimeClose.updatedAt === storedClose.updatedAt;
|
|
3582
|
+
if (!runtimeMatchesDurable) {
|
|
3583
|
+
// A runtime copy is not durable authority. In particular, a stale/forged
|
|
3584
|
+
// in-memory `prepared` value must never bypass cancellation, while a real
|
|
3585
|
+
// prepared store row must not be re-cancelled because a detached runtime
|
|
3586
|
+
// copy missed the last sync.
|
|
3587
|
+
logger.error(`[${session.sessionId.slice(0, 8)}] explicit close refused: runtime/durable Mojo journal mismatch`);
|
|
3588
|
+
return {
|
|
3589
|
+
ok: false,
|
|
3590
|
+
error: 'mojo_close_reconciliation_required',
|
|
3591
|
+
retryable: true,
|
|
3592
|
+
...((storedClose?.taskId ?? runtimeClose.taskId ?? remoteId)
|
|
3593
|
+
? { taskId: (storedClose?.taskId ?? runtimeClose.taskId ?? remoteId) }
|
|
3594
|
+
: {}),
|
|
3595
|
+
};
|
|
3596
|
+
}
|
|
3597
|
+
}
|
|
3598
|
+
let durableClose = storedClose ?? runtimeClose;
|
|
3599
|
+
if (durableClose && !sessionStore.isValidMojoCloseJournal(durableClose)) {
|
|
3600
|
+
logger.error(`[${session.sessionId.slice(0, 8)}] explicit close refused: malformed Mojo close journal`);
|
|
3601
|
+
return {
|
|
3602
|
+
ok: false,
|
|
3603
|
+
error: 'mojo_close_reconciliation_required',
|
|
3604
|
+
retryable: true,
|
|
3605
|
+
...(remoteId ? { taskId: remoteId } : {}),
|
|
3606
|
+
};
|
|
3607
|
+
}
|
|
3608
|
+
// Already closed is normally an idempotent replay. A journal on a closed row
|
|
3609
|
+
// is NOT normal (the status transition clears it atomically), so do not flatten
|
|
3610
|
+
// that contradictory disk state into an ordinary success.
|
|
3611
|
+
if (stored && stored.status === 'closed') {
|
|
3612
|
+
if (durableClose) {
|
|
3613
|
+
return {
|
|
3614
|
+
ok: false,
|
|
3615
|
+
error: 'mojo_close_reconciliation_required',
|
|
3616
|
+
retryable: true,
|
|
3617
|
+
...((durableClose.taskId ?? remoteId)
|
|
3618
|
+
? { taskId: (durableClose.taskId ?? remoteId) }
|
|
3619
|
+
: {}),
|
|
3620
|
+
};
|
|
3621
|
+
}
|
|
3622
|
+
return parked ? { ok: true, residual: parked } : { ok: true };
|
|
3623
|
+
}
|
|
3624
|
+
if (durableClose?.phase === 'prepared') {
|
|
3625
|
+
// A prior daemon already received and durably recorded the irreversible
|
|
3626
|
+
// worker cancellation proof. Finish only the local status/lineage commit;
|
|
3627
|
+
// repeating the CLI cancel would turn an idempotent recovery into failure.
|
|
3628
|
+
// A parked remote lineage outranks the journal's LOCAL residual in the one
|
|
3629
|
+
// slot the response has (it names a live remote session); the local marker
|
|
3630
|
+
// still surfaces whenever nothing is parked.
|
|
3631
|
+
return {
|
|
3632
|
+
ok: true,
|
|
3633
|
+
...((durableClose.taskId ?? remoteId)
|
|
3634
|
+
? { taskId: (durableClose.taskId ?? remoteId) }
|
|
3635
|
+
: {}),
|
|
3636
|
+
// The journal's residual is a stored copy of the original proof; whether
|
|
3637
|
+
// it still describes reality is the ledger's call (a reboot between the
|
|
3638
|
+
// prepare and this replay may have discharged the handle).
|
|
3639
|
+
...(parked
|
|
3640
|
+
? { residual: parked }
|
|
3641
|
+
: residualField(liveLocalResidual(session.sessionId, durableClose.localResidual))),
|
|
3642
|
+
};
|
|
3643
|
+
}
|
|
3644
|
+
const matchingRuntimePreparedProof = durableClose?.phase === 'preparing'
|
|
3645
|
+
&& ds?.remoteCloseState?.phase === 'prepared'
|
|
3646
|
+
&& ds.remoteCloseState.requestId === durableClose.requestId;
|
|
3647
|
+
const matchingRuntimeAbortRestoreProof = durableClose?.phase === 'preparing'
|
|
3648
|
+
&& ds?.remoteCloseState?.phase === 'abort_restored'
|
|
3649
|
+
&& ds.remoteCloseState.requestId === durableClose.requestId;
|
|
3650
|
+
const hasLiveWorker = !!ds?.worker && !ds.worker.killed;
|
|
3651
|
+
if (ds && matchingRuntimePreparedProof && !hasLiveWorker) {
|
|
3652
|
+
// Workerless cancellation is irreversible too. If publishing `prepared`
|
|
3653
|
+
// failed after the CLI proved cancellation, retain the in-memory proof and
|
|
3654
|
+
// let a retry publish that SAME proof before committing the row. Never issue
|
|
3655
|
+
// a second cancel merely because the first durable write failed.
|
|
3656
|
+
try {
|
|
3657
|
+
const committed = sessionStore.markMojoClosePrepared(session.sessionId, ds.remoteCloseState.requestId, ds.remoteCloseState.taskId, ds.remoteCloseState.localResidual);
|
|
3658
|
+
session.mojoCloseJournal = committed.mojoCloseJournal;
|
|
3659
|
+
session.riffParentTaskId = committed.riffParentTaskId;
|
|
3660
|
+
return {
|
|
3661
|
+
ok: true,
|
|
3662
|
+
...(ds.remoteCloseState.taskId ? { taskId: ds.remoteCloseState.taskId } : {}),
|
|
3663
|
+
...(parked
|
|
3664
|
+
? { residual: parked }
|
|
3665
|
+
: residualField(liveLocalResidual(session.sessionId, ds.remoteCloseState.localResidual))),
|
|
3666
|
+
};
|
|
3667
|
+
}
|
|
3668
|
+
catch (err) {
|
|
3669
|
+
logger.error(`[${tag(ds)}] Workerless Mojo close proof could not be persisted for durable commit: `
|
|
3670
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
3671
|
+
return {
|
|
3672
|
+
ok: false,
|
|
3673
|
+
error: 'mojo_durable_close_failed',
|
|
3674
|
+
retryable: true,
|
|
3675
|
+
...(ds.remoteCloseState.taskId ? { taskId: ds.remoteCloseState.taskId } : {}),
|
|
3676
|
+
};
|
|
3677
|
+
}
|
|
3678
|
+
}
|
|
3679
|
+
if (ds && matchingRuntimeAbortRestoreProof && !hasLiveWorker) {
|
|
3680
|
+
// No worker admission changed on this path, but the durable intent must be
|
|
3681
|
+
// cleared before another cancel attempt is allowed. A failed clear keeps the
|
|
3682
|
+
// central admission fence aligned with disk.
|
|
3683
|
+
try {
|
|
3684
|
+
const restoredState = ds.remoteCloseState;
|
|
3685
|
+
const committed = sessionStore.finishMojoCloseAbort(session.sessionId, restoredState.requestId, {
|
|
3686
|
+
admissionRestored: true,
|
|
3687
|
+
...(restoredState.taskId ? { taskId: restoredState.taskId } : {}),
|
|
3688
|
+
});
|
|
3689
|
+
session.mojoCloseJournal = committed.mojoCloseJournal;
|
|
3690
|
+
session.riffParentTaskId = committed.riffParentTaskId;
|
|
3691
|
+
ds.remoteCloseState = undefined;
|
|
3692
|
+
durableClose = undefined;
|
|
3693
|
+
}
|
|
3694
|
+
catch (err) {
|
|
3695
|
+
logger.error(`[${tag(ds)}] Workerless Mojo abort proof could not clear the durable close journal: `
|
|
3696
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
3697
|
+
return {
|
|
3698
|
+
ok: false,
|
|
3699
|
+
error: 'mojo_durable_close_failed',
|
|
3700
|
+
retryable: true,
|
|
3701
|
+
...(ds.remoteCloseState.taskId ? { taskId: ds.remoteCloseState.taskId } : {}),
|
|
3702
|
+
};
|
|
3703
|
+
}
|
|
3704
|
+
}
|
|
3705
|
+
const livePreparedProof = matchingRuntimePreparedProof
|
|
3706
|
+
&& !!ds.worker
|
|
3707
|
+
&& !ds.worker.killed;
|
|
3708
|
+
const liveAbortRestoreProof = matchingRuntimeAbortRestoreProof
|
|
3709
|
+
&& !!ds.worker
|
|
3710
|
+
&& !ds.worker.killed;
|
|
3711
|
+
if (durableClose && !livePreparedProof && !liveAbortRestoreProof) {
|
|
3712
|
+
const journalLineage = durableClose.taskId ?? remoteId;
|
|
3713
|
+
// The RECONCILIATION EXIT this journal has always promised (types.ts:
|
|
3714
|
+
// "pending explicit reconciliation"). Before it existed, every journal
|
|
3715
|
+
// without a matching runtime proof was refused here forever: restore
|
|
3716
|
+
// downgraded crashed closes to `uncertain` and quarantined the row
|
|
3717
|
+
// unregistered, this branch then refused each /close as
|
|
3718
|
+
// reconciliation_required, and CLI-mismatch cleanup re-quarantined it —
|
|
3719
|
+
// a permanent brick whose remote session kept credentials, fixable only
|
|
3720
|
+
// by hand-editing JSON state (P1-1). Two journal shapes are reconcilable
|
|
3721
|
+
// by an EXPLICIT close:
|
|
3722
|
+
// * `uncertain` — no retry can classify the crashed cancel without a
|
|
3723
|
+
// remote status oracle, so the row DRAINS: it closes with the lineage
|
|
3724
|
+
// parked as an explicit residual (same isolation-over-deletion shape
|
|
3725
|
+
// as a quarantined lineage — cleanup is remote, the id is preserved,
|
|
3726
|
+
// and replays keep reporting it).
|
|
3727
|
+
// * `preparing` + recovery 'retryable' — the failed prepare durably
|
|
3728
|
+
// recorded that retrying the cancel is legitimate (P1-2). With a
|
|
3729
|
+
// registered owner the close falls through and re-runs the cancel
|
|
3730
|
+
// under a fresh requestId (beginMojoCloseJournal accepts the restart);
|
|
3731
|
+
// without one there is nothing to cancel through, so it drains like
|
|
3732
|
+
// `uncertain`.
|
|
3733
|
+
const retryableJournal = durableClose.phase === 'preparing'
|
|
3734
|
+
&& durableClose.recovery === 'retryable';
|
|
3735
|
+
// NEVER drain past a live worker (P0-new). Draining set the row closed and
|
|
3736
|
+
// deleted the registry entry while killWorker's remote guard (P0-2) refused
|
|
3737
|
+
// the request-less retirement — the still-running, credential-carrying mojo
|
|
3738
|
+
// worker became unreachable by every entry point. A live worker is also the
|
|
3739
|
+
// one owner that can produce FRESH evidence, so it falls through to
|
|
3740
|
+
// prepare/commit below instead: beginMojoCloseJournal accepts the takeover
|
|
3741
|
+
// of an uncertain/retryable journal under the new requestId, the worker
|
|
3742
|
+
// re-runs destroySession, and killWorker retires it legally with the commit
|
|
3743
|
+
// requestId. (An in-process runtime `uncertain` fence still refuses inside
|
|
3744
|
+
// prepareLiveRemoteWorkerClose — deliberately; refusal keeps the worker
|
|
3745
|
+
// reachable, which is exactly what the drain violated.)
|
|
3746
|
+
const drainable = !hasLiveWorker
|
|
3747
|
+
&& (durableClose.phase === 'uncertain' || (retryableJournal && !ds));
|
|
3748
|
+
if (drainable) {
|
|
3749
|
+
logger.warn(`[${session.sessionId.slice(0, 8)}] explicit close drains ${durableClose.phase} Mojo `
|
|
3750
|
+
+ `journal (${durableClose.requestId}): the row closes`
|
|
3751
|
+
+ (journalLineage
|
|
3752
|
+
? ` and lineage ${journalLineage} is parked as a residual for manual cleanup`
|
|
3753
|
+
: ' with no lineage to park'));
|
|
3754
|
+
if (!journalLineage)
|
|
3755
|
+
return parked ? { ok: true, residual: parked } : { ok: true };
|
|
3756
|
+
return {
|
|
3757
|
+
ok: true,
|
|
3758
|
+
residual: parked ?? { reason: 'mojo_lineage_quarantined', taskId: journalLineage },
|
|
3759
|
+
parkMojoLineage: journalLineage,
|
|
3760
|
+
};
|
|
3761
|
+
}
|
|
3762
|
+
const liveWorkerReconciles = hasLiveWorker
|
|
3763
|
+
&& (durableClose.phase === 'uncertain' || retryableJournal);
|
|
3764
|
+
if (!retryableJournal && !liveWorkerReconciles) {
|
|
3765
|
+
logger.warn(`[${session.sessionId.slice(0, 8)}] explicit close requires Mojo reconciliation: `
|
|
3766
|
+
+ `durable journal is ${durableClose.phase} (${durableClose.requestId})`);
|
|
3767
|
+
return {
|
|
3768
|
+
ok: false,
|
|
3769
|
+
error: 'mojo_close_reconciliation_required',
|
|
3770
|
+
retryable: true,
|
|
3771
|
+
...(journalLineage ? { taskId: journalLineage } : {}),
|
|
3772
|
+
};
|
|
3773
|
+
}
|
|
3774
|
+
// Fall through: a retryable journal with a registered owner re-runs the
|
|
3775
|
+
// cancel (live worker → prepare/commit; workerless → the cancel below),
|
|
3776
|
+
// and an uncertain journal with a LIVE worker goes to prepare/commit.
|
|
3777
|
+
}
|
|
3778
|
+
// A live worker is the only authority that can cover the pre-init window:
|
|
3779
|
+
// destroySession waits for system/init, then returns the exact lineage in its
|
|
3780
|
+
// close_result. This must run even when the durable row has no id yet.
|
|
3781
|
+
if (ds?.worker && !ds.worker.killed) {
|
|
3782
|
+
if (!stored || stored.status !== 'active') {
|
|
3783
|
+
return {
|
|
3784
|
+
ok: false,
|
|
3785
|
+
error: 'mojo_close_identity_missing',
|
|
3786
|
+
retryable: true,
|
|
3787
|
+
...(remoteId ? { taskId: remoteId } : {}),
|
|
3788
|
+
};
|
|
3789
|
+
}
|
|
3790
|
+
return prepareLiveRemoteWorkerClose(ds, 'mojo');
|
|
3791
|
+
}
|
|
3792
|
+
// Nothing ACTIVE to cancel: either a clean row, or a parked-only one.
|
|
3793
|
+
if (!remoteId)
|
|
3794
|
+
return parked ? { ok: true, residual: parked } : { ok: true };
|
|
3795
|
+
if (!ds) {
|
|
3796
|
+
// An open durable row carrying a lineage, with no owner in the active
|
|
3797
|
+
// registry. Everything the cancel needs (frozen identity, launch identity)
|
|
3798
|
+
// hangs off DaemonSession, so this cannot be cancelled from here — and
|
|
3799
|
+
// silently publishing it as closed would be the same lie as before, just
|
|
3800
|
+
// through a different door. Fail closed; a restore/adopt puts the owner back
|
|
3801
|
+
// and makes the retry work.
|
|
3802
|
+
logger.warn(`[${session.sessionId.slice(0, 8)}] explicit close refused: mojo lineage ${remoteId} `
|
|
3803
|
+
+ 'has no active owner to cancel through; row stays open');
|
|
3804
|
+
return { ok: false, error: 'mojo_close_identity_missing', retryable: true, taskId: remoteId };
|
|
3805
|
+
}
|
|
3806
|
+
const resolved = resolveMojoCancelLaunch(ds, remoteId, 'explicit close');
|
|
3807
|
+
if (!resolved.ok) {
|
|
3808
|
+
if (resolved.reason === 'bot_gone') {
|
|
3809
|
+
// NOT permanent: re-registering the bot restores the config this needs, so
|
|
3810
|
+
// this is retryable and must not close the row behind the operator's back.
|
|
3811
|
+
return { ok: false, error: 'mojo_config_missing', retryable: true, taskId: remoteId };
|
|
3812
|
+
}
|
|
3813
|
+
// Quarantined: cancelling could reach a different tenant, and no retry can
|
|
3814
|
+
// ever make an unverifiable control plane verifiable. Refusing forever would
|
|
3815
|
+
// strand the row, so the close proceeds — but as an EXPLICIT residual, never
|
|
3816
|
+
// as an ordinary success, and the lineage stays on the row for manual cleanup.
|
|
3817
|
+
//
|
|
3818
|
+
// The id is PARKED as part of the durable close (see parkMojoLineage), which is
|
|
3819
|
+
// what makes the decision replayable: a second close reads it back out of the
|
|
3820
|
+
// parked slot instead of tripping over an active lineage with no owner.
|
|
3821
|
+
return {
|
|
3822
|
+
ok: true,
|
|
3823
|
+
residual: parked ?? { reason: 'mojo_lineage_quarantined', taskId: remoteId },
|
|
3824
|
+
parkMojoLineage: remoteId,
|
|
2329
3825
|
};
|
|
2330
3826
|
}
|
|
2331
|
-
|
|
2332
|
-
return { ok: true };
|
|
2333
|
-
let riffConfig;
|
|
2334
|
-
const larkAppId = ds?.larkAppId ?? session.larkAppId;
|
|
3827
|
+
const requestId = randomUUID();
|
|
2335
3828
|
try {
|
|
2336
|
-
|
|
2337
|
-
|
|
3829
|
+
const committed = sessionStore.beginMojoCloseJournal(session.sessionId, requestId, remoteId);
|
|
3830
|
+
session.mojoCloseJournal = committed.mojoCloseJournal;
|
|
3831
|
+
session.riffParentTaskId = committed.riffParentTaskId;
|
|
2338
3832
|
}
|
|
2339
|
-
catch {
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
backendType,
|
|
2343
|
-
riffParentTaskId: taskId,
|
|
2344
|
-
riffConfig,
|
|
2345
|
-
});
|
|
2346
|
-
const closeLabel = ds ? tag(ds) : session.sessionId.slice(0, 8);
|
|
2347
|
-
if (!cleanup.ok) {
|
|
2348
|
-
logger.warn(`[${closeLabel}] explicit close refused: ${cleanup.kind}; `
|
|
2349
|
-
+ `Riff task ${taskId} remains active and retryable`);
|
|
3833
|
+
catch (err) {
|
|
3834
|
+
logger.error(`[${tag(ds)}] Workerless Mojo close journal could not be persisted before cancellation: `
|
|
3835
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
2350
3836
|
return {
|
|
2351
3837
|
ok: false,
|
|
2352
|
-
error:
|
|
3838
|
+
error: 'mojo_durable_close_failed',
|
|
2353
3839
|
retryable: true,
|
|
2354
|
-
taskId,
|
|
3840
|
+
taskId: remoteId,
|
|
2355
3841
|
};
|
|
2356
3842
|
}
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
3843
|
+
ds.remoteCloseState = { phase: 'preparing', requestId, taskId: remoteId };
|
|
3844
|
+
const outcome = await cancelMojoSessionById(resolved.launchCfg, remoteId).catch((err) => ({
|
|
3845
|
+
kind: 'failed',
|
|
3846
|
+
message: err instanceof Error ? err.message : String(err),
|
|
3847
|
+
retryable: true,
|
|
3848
|
+
}));
|
|
3849
|
+
// Structured, NOT a boolean: `if (!outcome)` on an object is always false and
|
|
3850
|
+
// would turn every failure into a silent success. `already_terminal` counts as
|
|
3851
|
+
// gone, but only when the classifier could PROVE it (see MojoCancelOutcome).
|
|
3852
|
+
if (!isMojoRemoteGone(outcome)) {
|
|
3853
|
+
try {
|
|
3854
|
+
const committed = sessionStore.finishMojoCloseAbort(session.sessionId, requestId, { admissionRestored: true, taskId: remoteId });
|
|
3855
|
+
session.mojoCloseJournal = committed.mojoCloseJournal;
|
|
3856
|
+
session.riffParentTaskId = committed.riffParentTaskId;
|
|
3857
|
+
ds.remoteCloseState = undefined;
|
|
3858
|
+
}
|
|
3859
|
+
catch (err) {
|
|
3860
|
+
// There was no live worker admission to restore, but accepting input while
|
|
3861
|
+
// disk still says `preparing` would recreate the same split-brain. Retain
|
|
3862
|
+
// an abort-restored runtime proof so a retry clears the journal first.
|
|
3863
|
+
ds.remoteCloseState = { phase: 'abort_restored', requestId, taskId: remoteId };
|
|
3864
|
+
logger.error(`[${tag(ds)}] Workerless Mojo close abort could not be committed durably: `
|
|
3865
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
3866
|
+
return {
|
|
3867
|
+
ok: false,
|
|
3868
|
+
error: 'mojo_durable_close_failed',
|
|
3869
|
+
retryable: true,
|
|
3870
|
+
taskId: remoteId,
|
|
3871
|
+
};
|
|
3872
|
+
}
|
|
3873
|
+
logger.warn(`[${tag(ds)}] explicit close refused: mojo session ${remoteId} could not be `
|
|
3874
|
+
+ 'cancelled; the row stays open and the lineage is retained so the close is retryable'
|
|
3875
|
+
+ (outcome.kind === 'failed' ? ` (${outcome.message})` : ''));
|
|
3876
|
+
return { ok: false, error: 'mojo_cancel_failed', retryable: true, taskId: remoteId };
|
|
3877
|
+
}
|
|
3878
|
+
// The workerless local-subtree proof travels ON the outcome (see
|
|
3879
|
+
// cancelMojoSessionById): a weak-only proof keeps its containment handle, and
|
|
3880
|
+
// this close must say so — durably, so retries and restarts republish it.
|
|
3881
|
+
const localResidual = outcome.kind !== 'failed' ? outcome.localResidual : undefined;
|
|
3882
|
+
try {
|
|
3883
|
+
const committed = sessionStore.markMojoClosePrepared(session.sessionId, requestId, remoteId, localResidual);
|
|
3884
|
+
session.mojoCloseJournal = committed.mojoCloseJournal;
|
|
3885
|
+
session.riffParentTaskId = committed.riffParentTaskId;
|
|
3886
|
+
}
|
|
3887
|
+
catch (err) {
|
|
3888
|
+
// Cancellation is proven and cannot be rolled back. Keep the runtime proof
|
|
3889
|
+
// prepared so the same daemon can publish it on retry; after a crash the
|
|
3890
|
+
// durable `preparing` journal remains fenced for explicit reconciliation.
|
|
3891
|
+
ds.remoteCloseState = {
|
|
3892
|
+
phase: 'prepared',
|
|
3893
|
+
requestId,
|
|
3894
|
+
taskId: remoteId,
|
|
3895
|
+
...(localResidual ? { localResidual } : {}),
|
|
3896
|
+
};
|
|
3897
|
+
logger.error(`[${tag(ds)}] Workerless Mojo close proof persistence failed after cancellation: `
|
|
3898
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
2367
3899
|
return {
|
|
2368
3900
|
ok: false,
|
|
2369
|
-
error: '
|
|
3901
|
+
error: 'mojo_durable_close_failed',
|
|
2370
3902
|
retryable: true,
|
|
2371
|
-
taskId:
|
|
3903
|
+
taskId: remoteId,
|
|
2372
3904
|
};
|
|
2373
3905
|
}
|
|
2374
|
-
|
|
2375
|
-
|
|
3906
|
+
ds.remoteCloseState = {
|
|
3907
|
+
phase: 'prepared',
|
|
3908
|
+
requestId,
|
|
3909
|
+
taskId: remoteId,
|
|
3910
|
+
...(localResidual ? { localResidual } : {}),
|
|
3911
|
+
};
|
|
3912
|
+
// Deliberately does NOT clear the runtime lineage here. sessionStore commonly
|
|
3913
|
+
// holds the very same Session object, so an early clear is visible to the durable
|
|
3914
|
+
// save — and if that save then fails, the rollback has already lost the id it
|
|
3915
|
+
// would restore while the on-disk row may still carry it. The lineage is cleared
|
|
3916
|
+
// atomically with the status change instead (clearRiffParentTaskId), and the
|
|
3917
|
+
// repeat-cancel is suppressed with an explicit flag rather than by mutating
|
|
3918
|
+
// shared state early.
|
|
3919
|
+
logger.info(`[${tag(ds)}] mojo session ${remoteId} cancelled for explicit close`);
|
|
3920
|
+
// Workerless twin of MojoBackend.destroySession()'s reap: the close verdict
|
|
3921
|
+
// above is already decided, so a reaping failure only leaks an idle daemon
|
|
3922
|
+
// (logged inside), never fails the close. Host-mode gating lives inside the
|
|
3923
|
+
// helper via the isolated dir's existence — a cloud session never created one.
|
|
3924
|
+
void cleanupMojoIsolatedWorkspace(session.sessionId).catch(() => undefined);
|
|
3925
|
+
// Cancelling the ACTIVE lineage says nothing about a previously parked one: a row
|
|
3926
|
+
// can carry both (restore parked the old id, the session then created a new one).
|
|
3927
|
+
// Reporting a plain success here would hide the parked remote session entirely.
|
|
3928
|
+
// With nothing parked, the LOCAL residual takes the slot for the same reason.
|
|
3929
|
+
return {
|
|
3930
|
+
ok: true,
|
|
3931
|
+
taskId: remoteId,
|
|
3932
|
+
...(parked
|
|
3933
|
+
? { residual: parked }
|
|
3934
|
+
: localResidual
|
|
3935
|
+
? { residual: { reason: localResidual } }
|
|
3936
|
+
: {}),
|
|
3937
|
+
};
|
|
3938
|
+
}
|
|
3939
|
+
/**
|
|
3940
|
+
* The residual a row carries because a lineage was PARKED as unverifiable.
|
|
3941
|
+
*
|
|
3942
|
+
* Read from the parked slot, never from the active one: restore-time quarantine
|
|
3943
|
+
* moves the id into `mojoQuarantinedLineage` and clears `riffParentTaskId`, so a
|
|
3944
|
+
* check against the active slot misses the production shape completely — the row
|
|
3945
|
+
* would close as an ordinary success while its remote session kept running.
|
|
3946
|
+
*
|
|
3947
|
+
* May hold more than one id (comma-joined) when two unverifiable lineages were
|
|
3948
|
+
* parked together; it is passed through verbatim so manual cleanup sees both.
|
|
3949
|
+
*/
|
|
3950
|
+
export function mojoCloseResidualForRow(session) {
|
|
3951
|
+
// A parked REMOTE lineage outranks a local residual: it names a live remote
|
|
3952
|
+
// session burning cloud time, the more urgent cleanup, and the response has a
|
|
3953
|
+
// single residual slot. The local residual surfaces whenever nothing is parked.
|
|
3954
|
+
const parked = session?.mojoQuarantinedLineage;
|
|
3955
|
+
if (parked)
|
|
3956
|
+
return { reason: 'mojo_lineage_quarantined', taskId: parked };
|
|
3957
|
+
if (session?.mojoLocalResidual)
|
|
3958
|
+
return liveLocalResidual(session.sessionId, session.mojoLocalResidual);
|
|
3959
|
+
return undefined;
|
|
3960
|
+
}
|
|
3961
|
+
/**
|
|
3962
|
+
* Surface a stored LOCAL residual only while its containment handle is still
|
|
3963
|
+
* outstanding.
|
|
3964
|
+
*
|
|
3965
|
+
* The containment-handle ledger is the SOURCE OF TRUTH for the device-isolation
|
|
3966
|
+
* blocker; `Session.mojoLocalResidual` (and a journal's `localResidual`) are
|
|
3967
|
+
* DERIVED display state with deliberately no clear path of their own: the two
|
|
3968
|
+
* paths that discharge a handle — boot reconciliation and operator revoke —
|
|
3969
|
+
* operate on the one global ledger, and having them also rewrite session rows
|
|
3970
|
+
* would tie a global cleanup to whichever bots' per-bot row files happen to be
|
|
3971
|
+
* loaded at the time. Deriving at read time keeps the two stores fork-free: the
|
|
3972
|
+
* handle is gone → the "credentialed process may remain on this host" claim is
|
|
3973
|
+
* no longer true, report nothing; the ledger cannot be read → we cannot rule the
|
|
3974
|
+
* claim out, keep reporting it (fail-closed, matching hasUnprovenContainment's
|
|
3975
|
+
* own contract of throwing rather than answering false).
|
|
3976
|
+
*/
|
|
3977
|
+
/** Spread helper: `{ residual }` when present, `{}` otherwise. */
|
|
3978
|
+
function residualField(residual) {
|
|
3979
|
+
return residual ? { residual } : {};
|
|
3980
|
+
}
|
|
3981
|
+
function liveLocalResidual(sessionId, residual) {
|
|
3982
|
+
if (!residual)
|
|
3983
|
+
return undefined;
|
|
3984
|
+
try {
|
|
3985
|
+
if (!hasUnprovenContainment(sessionId))
|
|
3986
|
+
return undefined;
|
|
3987
|
+
}
|
|
3988
|
+
catch {
|
|
3989
|
+
// Unreadable ledger: a handle may exist, so the residual stays reported.
|
|
3990
|
+
}
|
|
3991
|
+
return { reason: residual };
|
|
2376
3992
|
}
|
|
2377
3993
|
/**
|
|
2378
3994
|
* Reclaim a session's parked crash-diagnostic shell (`bmx-diag-<sid>`) and its
|
|
@@ -2477,8 +4093,14 @@ export function suspendWorker(ds, reason = 'suspended_idle') {
|
|
|
2477
4093
|
// probes. See sweepIdleWorkers + restoreActiveSessions.
|
|
2478
4094
|
ds.hasHistory = true;
|
|
2479
4095
|
ds.session.suspendedColdResume = true;
|
|
4096
|
+
// P1-13:suspend 会销毁 CLI 与其子进程,那一代 dev server 随之消失(端口号交还
|
|
4097
|
+
// 内核)。会话本身还活着,所以不会走 close 路径——预览必须在这里同步失效,否则
|
|
4098
|
+
// 「休眠中」的会话卡片仍然挂着一条指向空端口/别人进程的预览入口。
|
|
4099
|
+
const suspendedPreviewTarget = takeSessionPreviewTarget(ds.session);
|
|
2480
4100
|
sessionStore.updateSessionPid(ds.session.sessionId, null);
|
|
2481
4101
|
sessionStore.updateSession(ds.session);
|
|
4102
|
+
if (suspendedPreviewTarget !== undefined)
|
|
4103
|
+
publishSessionPreviewCleared(ds.session.sessionId);
|
|
2482
4104
|
if (!ds.exitEventEmitted) {
|
|
2483
4105
|
ds.exitEventEmitted = true;
|
|
2484
4106
|
dashboardEventBus.publish({
|
|
@@ -2774,19 +4396,64 @@ function teardownAuthoritativePersistentBackingBeforeCloseImpl(target, closeWins
|
|
|
2774
4396
|
export function teardownAuthoritativePersistentBackingBeforeClose(target) {
|
|
2775
4397
|
teardownAuthoritativePersistentBackingBeforeCloseImpl(target, false);
|
|
2776
4398
|
}
|
|
4399
|
+
/**
|
|
4400
|
+
* Close from a BACKGROUND path — one with no user surface to report to
|
|
4401
|
+
* (trigger-session cleanup, deferred-schedule settlement, sweeps).
|
|
4402
|
+
*
|
|
4403
|
+
* These callers cannot show a card or a toast, so the only honest thing they can
|
|
4404
|
+
* do with a residual or a refusal is make it OBSERVABLE: an uncancelled remote
|
|
4405
|
+
* session is still burning cloud time and holding an injected credential, and
|
|
4406
|
+
* silently discarding the result is how that became invisible everywhere else.
|
|
4407
|
+
*
|
|
4408
|
+
* Returns the untouched result so a caller that DOES care can still branch.
|
|
4409
|
+
*/
|
|
4410
|
+
export async function closeSessionForBackgroundCleanup(sessionId, context) {
|
|
4411
|
+
const result = await closeSession(sessionId);
|
|
4412
|
+
const tagId = sessionId.slice(0, 8);
|
|
4413
|
+
if (!result.ok) {
|
|
4414
|
+
logger.error(`[${tagId}] ${context}: close REFUSED (${result.error}); the row stays active `
|
|
4415
|
+
+ 'and its remote session may still be running'
|
|
4416
|
+
+ (result.taskId ? ` (remote ${result.taskId})` : ''));
|
|
4417
|
+
return result;
|
|
4418
|
+
}
|
|
4419
|
+
if (result.outcome === 'closed_with_residual') {
|
|
4420
|
+
// Kind-aware: a LOCAL-subtree residual has no taskId, so the old wording
|
|
4421
|
+
// logged "remote session undefined was NOT cancelled" and pointed cleanup at
|
|
4422
|
+
// a nonexistent remote session instead of the host process (round-11 P1-2).
|
|
4423
|
+
logger.warn(`[${tagId}] ${context}: closed locally — ${closeResidualClause(result.residual)} `
|
|
4424
|
+
+ `(${result.residual.reason})`);
|
|
4425
|
+
}
|
|
4426
|
+
return result;
|
|
4427
|
+
}
|
|
2777
4428
|
export async function closeSession(sessionId) {
|
|
2778
4429
|
const ds = findActiveBySessionId(sessionId);
|
|
2779
4430
|
const stored = sessionStore.getOwnedSession(sessionId);
|
|
4431
|
+
const closeFrozenBackendType = ds?.initConfig?.backendType
|
|
4432
|
+
?? ds?.session.backendType ?? stored?.backendType;
|
|
4433
|
+
const isOwnedClose = !ds?.initConfig?.adoptMode
|
|
4434
|
+
&& !ds?.adoptedFrom
|
|
4435
|
+
&& !stored?.adoptedFrom;
|
|
4436
|
+
const isOwnedRiffClose = isOwnedClose && closeFrozenBackendType === 'riff';
|
|
4437
|
+
const isOwnedMojoClose = isOwnedClose && closeFrozenBackendType === 'mojo';
|
|
4438
|
+
const closeJournal = ds?.session.mojoCloseJournal ?? stored?.mojoCloseJournal;
|
|
4439
|
+
if (closeJournal && !isOwnedMojoClose) {
|
|
4440
|
+
// Never let a Mojo cancellation journal silently disappear through another
|
|
4441
|
+
// backend's ordinary close path. It may be corrupt metadata, but only an
|
|
4442
|
+
// explicit reconciliation/repair may remove that fence.
|
|
4443
|
+
return {
|
|
4444
|
+
ok: false,
|
|
4445
|
+
alreadyClosed: false,
|
|
4446
|
+
error: 'mojo_close_reconciliation_required',
|
|
4447
|
+
retryable: true,
|
|
4448
|
+
...(closeJournal.taskId ? { taskId: closeJournal.taskId } : {}),
|
|
4449
|
+
};
|
|
4450
|
+
}
|
|
2780
4451
|
// Prove fail-closed ZMX teardown before any registry/store mutation. Repo
|
|
2781
4452
|
// replacement paths reuse the same helper before their own state transition.
|
|
2782
4453
|
const teardownTarget = ds ?? stored;
|
|
2783
4454
|
if (teardownTarget) {
|
|
2784
4455
|
teardownAuthoritativePersistentBackingBeforeCloseImpl(teardownTarget, true);
|
|
2785
4456
|
}
|
|
2786
|
-
const isOwnedRiffClose = !ds?.initConfig?.adoptMode
|
|
2787
|
-
&& !ds?.adoptedFrom
|
|
2788
|
-
&& !stored?.adoptedFrom
|
|
2789
|
-
&& (ds?.initConfig?.backendType ?? ds?.session.backendType ?? stored?.backendType) === 'riff';
|
|
2790
4457
|
let killedLive = false;
|
|
2791
4458
|
const hadLiveWorker = !!ds?.worker && !ds.worker.killed;
|
|
2792
4459
|
const closeWorkerGeneration = ds ? closeFenceGeneration(ds) : undefined;
|
|
@@ -2794,6 +4461,11 @@ export async function closeSession(sessionId) {
|
|
|
2794
4461
|
// sessionStore commonly holds the very same Session reference as `ds`.
|
|
2795
4462
|
const known = !!ds || !!stored;
|
|
2796
4463
|
const wasOpen = !!stored && stored.status !== 'closed';
|
|
4464
|
+
// P1-13:关闭必须显式广播 `preview: null`。sessionStore.closeSession 会把字段从磁盘
|
|
4465
|
+
// 上抹掉,但没有事件——浏览器侧只会收到 `session.exited`,会话卡片上的预览入口就
|
|
4466
|
+
// 那么留着,Dashboard 的 preview SSE/WS 也拿不到断流信号。
|
|
4467
|
+
const hadPreviewTarget = ds?.session.previewTarget !== undefined
|
|
4468
|
+
|| stored?.previewTarget !== undefined;
|
|
2797
4469
|
const storedHadDocCommentTargets = Object.keys(stored?.docCommentTargets ?? {}).length > 0;
|
|
2798
4470
|
const docReactionTargets = collectDocCommentReactionTargets(ds, stored);
|
|
2799
4471
|
if (ds) {
|
|
@@ -2801,17 +4473,27 @@ export async function closeSession(sessionId) {
|
|
|
2801
4473
|
// crash/limited turn may never have reached an idle edge).
|
|
2802
4474
|
recordUsageForDaemonSession(ds);
|
|
2803
4475
|
}
|
|
2804
|
-
// Riff
|
|
2805
|
-
// mutating routing, transient capabilities, or the durable
|
|
2806
|
-
//
|
|
4476
|
+
// Riff and mojo both own a remote, credential-bearing session. Prove
|
|
4477
|
+
// cancellation before mutating routing, transient capabilities, or the durable
|
|
4478
|
+
// row — publishing "closed" while the cancel is still in flight is how a failed
|
|
4479
|
+
// cancel became a silent leak. Other backends retain master's synchronous
|
|
4480
|
+
// state-transition path.
|
|
2807
4481
|
const prepared = isOwnedRiffClose
|
|
2808
4482
|
? await prepareRiffExplicitClose(ds, stored)
|
|
2809
|
-
:
|
|
4483
|
+
: isOwnedMojoClose
|
|
4484
|
+
? await prepareMojoExplicitClose(ds, stored)
|
|
4485
|
+
: { ok: true };
|
|
2810
4486
|
if (!prepared.ok) {
|
|
2811
4487
|
return { ...prepared, alreadyClosed: false };
|
|
2812
4488
|
}
|
|
2813
|
-
|
|
2814
|
-
|
|
4489
|
+
// Only when the cancel actually happened: a quarantined / bot-gone mojo session
|
|
4490
|
+
// keeps its lineage on the row for manual cleanup.
|
|
4491
|
+
const clearMojoLineage = isOwnedMojoClose && !!prepared.taskId;
|
|
4492
|
+
// NOTE: the residual is derived from the COMMITTED row at the end of this
|
|
4493
|
+
// function, not captured here — the store merges parked ids, so a value computed
|
|
4494
|
+
// before the transaction can disagree with what was actually persisted.
|
|
4495
|
+
const preparedRemoteRequestId = ds?.remoteCloseState?.phase === 'prepared'
|
|
4496
|
+
? ds.remoteCloseState.requestId
|
|
2815
4497
|
: undefined;
|
|
2816
4498
|
// 会话关闭即可回收其崩溃重启计数;否则每个曾崩溃过的 session 会在 daemon
|
|
2817
4499
|
// 生命周期内永久占位(restartCounts 此前无任何 delete)。
|
|
@@ -2831,16 +4513,42 @@ export async function closeSession(sessionId) {
|
|
|
2831
4513
|
if (!ds && stored && !isOwnedRiffClose)
|
|
2832
4514
|
destroyUnregisteredPersistentBacking(stored);
|
|
2833
4515
|
try {
|
|
4516
|
+
// Parking is handed to the store as part of ITS transaction rather than
|
|
4517
|
+
// pre-written here: the runtime object is not always the authoritative row,
|
|
4518
|
+
// and a pre-write survives a failed save (the store's rollback cannot restore
|
|
4519
|
+
// a field it never set). Same mistake as the early lineage clear fixed
|
|
4520
|
+
// earlier — one layer up.
|
|
2834
4521
|
sessionStore.closeSession(sessionId, {
|
|
2835
4522
|
cleanupBridgeMarkers: !hadLiveWorker,
|
|
2836
|
-
...(
|
|
4523
|
+
...(prepared.parkMojoLineage ? { parkMojoLineage: prepared.parkMojoLineage } : {}),
|
|
4524
|
+
// Park a LOCAL residual so an idempotent re-close still reports it — the
|
|
4525
|
+
// journal (its runtime home) is wiped by this same transaction.
|
|
4526
|
+
...(prepared.residual
|
|
4527
|
+
&& prepared.residual.reason !== 'mojo_lineage_quarantined'
|
|
4528
|
+
? { parkLocalResidual: prepared.residual.reason }
|
|
4529
|
+
: {}),
|
|
4530
|
+
...(isOwnedRiffClose || clearMojoLineage || prepared.parkMojoLineage
|
|
4531
|
+
? { clearRiffParentTaskId: true }
|
|
4532
|
+
: {}),
|
|
2837
4533
|
});
|
|
2838
4534
|
}
|
|
2839
4535
|
catch (err) {
|
|
4536
|
+
if (isOwnedMojoClose
|
|
4537
|
+
&& (preparedRemoteRequestId || stored?.mojoCloseJournal?.phase === 'prepared')) {
|
|
4538
|
+
logger.error(`[${sessionId.slice(0, 8)}] Durable session close failed after Mojo cancel; `
|
|
4539
|
+
+ `retaining prepared commit: ${err instanceof Error ? err.message : String(err)}`);
|
|
4540
|
+
return {
|
|
4541
|
+
ok: false,
|
|
4542
|
+
alreadyClosed: false,
|
|
4543
|
+
error: 'mojo_durable_close_failed',
|
|
4544
|
+
retryable: true,
|
|
4545
|
+
...(prepared.taskId ? { taskId: prepared.taskId } : {}),
|
|
4546
|
+
};
|
|
4547
|
+
}
|
|
2840
4548
|
if (!isOwnedRiffClose)
|
|
2841
4549
|
throw err;
|
|
2842
|
-
if (ds &&
|
|
2843
|
-
await
|
|
4550
|
+
if (ds && preparedRemoteRequestId) {
|
|
4551
|
+
await abortLiveRemoteWorkerClose(ds, preparedRemoteRequestId);
|
|
2844
4552
|
}
|
|
2845
4553
|
logger.error(`[${sessionId.slice(0, 8)}] Durable session close failed after Riff prepare; `
|
|
2846
4554
|
+ `worker admission restored: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -2856,11 +4564,32 @@ export async function closeSession(sessionId) {
|
|
|
2856
4564
|
if (ds) {
|
|
2857
4565
|
ds.session.status = 'closed';
|
|
2858
4566
|
ds.session.closedAt = after?.closedAt ?? ds.session.closedAt;
|
|
4567
|
+
// 活对象与 store 里的行未必是同一个引用;durable close 只清了后者。
|
|
4568
|
+
takeSessionPreviewTarget(ds.session);
|
|
4569
|
+
// Sync the parking back from the store's committed row. Only reached after a
|
|
4570
|
+
// SUCCESSFUL save, and skipped when the two are the same object anyway, so
|
|
4571
|
+
// the runtime view cannot end up carrying a park the disk does not have.
|
|
4572
|
+
if (after && after !== ds.session) {
|
|
4573
|
+
ds.session.mojoCloseJournal = after.mojoCloseJournal;
|
|
4574
|
+
if (clearMojoLineage || prepared.parkMojoLineage) {
|
|
4575
|
+
ds.session.riffParentTaskId = after.riffParentTaskId;
|
|
4576
|
+
}
|
|
4577
|
+
}
|
|
4578
|
+
if (prepared.parkMojoLineage && after && after !== ds.session) {
|
|
4579
|
+
ds.session.mojoQuarantinedLineage = after.mojoQuarantinedLineage;
|
|
4580
|
+
ds.session.mojoQuarantineNoticePending = after.mojoQuarantineNoticePending;
|
|
4581
|
+
}
|
|
2859
4582
|
}
|
|
4583
|
+
if (hadPreviewTarget)
|
|
4584
|
+
publishSessionPreviewCleared(sessionId);
|
|
2860
4585
|
}
|
|
2861
4586
|
if (ds) {
|
|
4587
|
+
disposeOrdinaryTurnRecovery(ds.session);
|
|
2862
4588
|
killWorker(ds, {
|
|
2863
|
-
...(
|
|
4589
|
+
...(preparedRemoteRequestId ? { remoteCloseCommitRequestId: preparedRemoteRequestId } : {}),
|
|
4590
|
+
// The prepare above already proved this cancel; the durable row cleared the
|
|
4591
|
+
// lineage in the same transaction. Suppress the best-effort repeat.
|
|
4592
|
+
...(clearMojoLineage ? { mojoCancelAlreadyProven: true } : {}),
|
|
2864
4593
|
});
|
|
2865
4594
|
// A transferred/restored exact object may remain under more than one alias.
|
|
2866
4595
|
// Remove only identity matches, never a same-key successor.
|
|
@@ -2950,7 +4679,26 @@ export async function closeSession(sessionId) {
|
|
|
2950
4679
|
}
|
|
2951
4680
|
// alreadyClosed = nothing happened on either path.
|
|
2952
4681
|
const alreadyClosed = !killedLive && !wasOpen;
|
|
2953
|
-
|
|
4682
|
+
// The local row IS closed, but a remote session was deliberately left running
|
|
4683
|
+
// (its control plane could not be verified, so cancelling it might have hit
|
|
4684
|
+
// another tenant). Callers must render this differently from an ordinary close.
|
|
4685
|
+
// Read the residual back off the authoritative row so it always matches disk
|
|
4686
|
+
// (including the merge when a second id was parked). `prepared.residual` is only
|
|
4687
|
+
// a fallback for the paths that never reached the store.
|
|
4688
|
+
const closeResidual = isOwnedMojoClose
|
|
4689
|
+
? (mojoCloseResidualForRow(sessionStore.getOwnedSession(sessionId) ?? stored ?? ds?.session)
|
|
4690
|
+
?? prepared.residual)
|
|
4691
|
+
: undefined;
|
|
4692
|
+
if (closeResidual) {
|
|
4693
|
+
return {
|
|
4694
|
+
ok: true,
|
|
4695
|
+
outcome: 'closed_with_residual',
|
|
4696
|
+
residual: closeResidual,
|
|
4697
|
+
alreadyClosed,
|
|
4698
|
+
known,
|
|
4699
|
+
};
|
|
4700
|
+
}
|
|
4701
|
+
return { ok: true, outcome: 'closed', alreadyClosed, known };
|
|
2954
4702
|
}
|
|
2955
4703
|
/**
|
|
2956
4704
|
* Close can arrive through daemon IPC before startup restore has registered the
|
|
@@ -3029,7 +4777,34 @@ async function closeUnregisteredCollisionLoser(map, loser) {
|
|
|
3029
4777
|
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
3030
4778
|
return { ok: false, error: 'durable_close_failed' };
|
|
3031
4779
|
}
|
|
3032
|
-
|
|
4780
|
+
// The registry entry is deleted right below, so the loser's worker must be
|
|
4781
|
+
// put on a guaranteed path to death here. killWorker refuses an unprepared
|
|
4782
|
+
// live remote worker (P0-2) — correct for lineage, but on this path the
|
|
4783
|
+
// refusal recreated the P0-new orphan: row closed, registry cleared,
|
|
4784
|
+
// credential-carrying worker alive and unreachable. A remote loser is
|
|
4785
|
+
// therefore retired process-only: SIGTERM now, SIGKILL by backstop. To be
|
|
4786
|
+
// precise about the claim (fourth-round review): this SIGNALS death and arms
|
|
4787
|
+
// the escalation — it does not synchronously prove the exit. That is
|
|
4788
|
+
// acceptable here because nothing downstream gates on this process being
|
|
4789
|
+
// gone (unlike the VC fence, which separately demands exit proof before
|
|
4790
|
+
// admitting replay); the registry delete below is safe either way.
|
|
4791
|
+
//
|
|
4792
|
+
// No remote cancel is issued: the lineage stays on the CLOSED row
|
|
4793
|
+
// (riffParentTaskId) as the manual-cleanup handle, mirroring the drain's
|
|
4794
|
+
// isolation-over-deletion shape — named in the log below so an operator can
|
|
4795
|
+
// actually find it. Local losers keep killWorker so their backing session
|
|
4796
|
+
// is destroyed.
|
|
4797
|
+
if (isRemoteBackendSession(loser)) {
|
|
4798
|
+
if (loser.session.riffParentTaskId) {
|
|
4799
|
+
logger.warn(`[${tag(loser)}] collision loser owns remote lineage `
|
|
4800
|
+
+ `${loser.session.riffParentTaskId}; it is NOT cancelled and stays on the closed `
|
|
4801
|
+
+ 'row for manual cleanup');
|
|
4802
|
+
}
|
|
4803
|
+
retireWorkerProcessOnly(loser, 'collision_loser');
|
|
4804
|
+
}
|
|
4805
|
+
else {
|
|
4806
|
+
killWorker(loser);
|
|
4807
|
+
}
|
|
3033
4808
|
for (const [registeredKey, candidate] of map) {
|
|
3034
4809
|
if (candidate === loser)
|
|
3035
4810
|
map.delete(registeredKey);
|
|
@@ -3274,6 +5049,10 @@ function failOrdinaryImDelivery(record, reason) {
|
|
|
3274
5049
|
logger.error(`[${tag(record.ds)}] Ordinary IM worker delivery failed `
|
|
3275
5050
|
+ `turn=${record.turnId.substring(0, 16)} generation=${record.workerGeneration} `
|
|
3276
5051
|
+ `attempts=${record.attempt} reason=${reason}`);
|
|
5052
|
+
if (record.turnId.startsWith('bmx-recovery-')) {
|
|
5053
|
+
requireOrdinaryTurnRecoveryAttention(record.ds.session, record.turnId, 'recovery_delivery_failed');
|
|
5054
|
+
return;
|
|
5055
|
+
}
|
|
3277
5056
|
if (record.ds.session.vcMeetingReceiver || isSilentScheduledTurn(record.ds, record.turnId))
|
|
3278
5057
|
return;
|
|
3279
5058
|
const loc = botLocale(getBot(record.ds.larkAppId).config);
|
|
@@ -3360,7 +5139,8 @@ function sendOrdinaryImDeliveryTracked(ds, message) {
|
|
|
3360
5139
|
return sendOrdinaryImDeliveryAttempt(record);
|
|
3361
5140
|
}
|
|
3362
5141
|
function shouldTrackOrdinaryImDelivery(ds, message) {
|
|
3363
|
-
return !!message.turnId
|
|
5142
|
+
return !!message.turnId
|
|
5143
|
+
&& (message.turnId.startsWith('om_') || message.turnId.startsWith('bmx-recovery-'))
|
|
3364
5144
|
&& message.dispatchAttempt === undefined
|
|
3365
5145
|
&& (message.type !== 'init' || (!!message.prompt && !message.adoptMode))
|
|
3366
5146
|
&& !ds.adoptedFrom
|
|
@@ -3943,6 +5723,7 @@ targetScope, opts) {
|
|
|
3943
5723
|
ds.streamCardId = undefined;
|
|
3944
5724
|
ds.streamCardNonce = undefined;
|
|
3945
5725
|
ds.currentImageKey = undefined;
|
|
5726
|
+
rehomeReplyTargetState(ds);
|
|
3946
5727
|
sessionStore.updateSession(ds.session);
|
|
3947
5728
|
const newAnchor = sessionAnchorId(ds);
|
|
3948
5729
|
if (activeSessionsRegistry) {
|
|
@@ -4011,13 +5792,13 @@ targetScope, opts) {
|
|
|
4011
5792
|
}
|
|
4012
5793
|
/** Backends whose conversation state is a local, copyable transcript file and
|
|
4013
5794
|
* whose CLI exposes a native "fork/branch this session" primitive that botmux
|
|
4014
|
-
* can drive at cold spawn (Claude family: `--fork-session`; Codex terminal:
|
|
5795
|
+
* can drive at cold spawn (Claude family / Grok: `--fork-session`; Codex terminal:
|
|
4015
5796
|
* `codex fork <id>`). App-server backends (codex-app, or a codex CLI running in
|
|
4016
5797
|
* Hybrid RPC mode) keep state in a live app-server process + SQLite and have no
|
|
4017
5798
|
* byte-level fork we can reproduce — they are refused. Riff / other pure-remote
|
|
4018
5799
|
* backends have no local rollout to fork either. */
|
|
4019
5800
|
const FORK_CAPABLE_CLI_IDS = new Set([
|
|
4020
|
-
'claude-code', 'seed', 'relay', 'codex',
|
|
5801
|
+
'claude-code', 'seed', 'relay', 'codex', 'grok',
|
|
4021
5802
|
]);
|
|
4022
5803
|
/** True when this session can be byte-level forked via a CLI-native primitive.
|
|
4023
5804
|
* Refuses codex-app outright, and refuses a plain `codex` session that is
|
|
@@ -4164,12 +5945,19 @@ export async function forkSession(sessionId, targetChatId, targetRootMessageId,
|
|
|
4164
5945
|
// (bwrap credential seal — bots.json deny / sibling appsecrets /
|
|
4165
5946
|
// master.key / network deny — silently dropped). This is a security
|
|
4166
5947
|
// escape, so copy the recorded decision and its path lists verbatim.
|
|
4167
|
-
// •
|
|
5948
|
+
// • reasoningEffort / cliPathOverride / wrapperCli / agentFrozen:
|
|
4168
5949
|
// without these the child's sessionAgentConfig() sees !agentFrozen and
|
|
4169
5950
|
// re-freezes from the CURRENT bot config, silently dropping any per-session
|
|
4170
|
-
//
|
|
4171
|
-
//
|
|
4172
|
-
//
|
|
5951
|
+
// /effort override the source carried (reasoningEffort has no botCfg
|
|
5952
|
+
// fallback at all → drops to undefined). Copying the frozen tuple keeps
|
|
5953
|
+
// the clone's launch identity == the source's. `model` is no longer a
|
|
5954
|
+
// FROZEN value — it resolves from the live bot config at every spawn — but
|
|
5955
|
+
// the record of what the source last launched with still has to travel:
|
|
5956
|
+
// when the source is pinned to a CLI the bot no longer runs, that record is
|
|
5957
|
+
// the only thing keeping either session off the CLI default (rule 3 in
|
|
5958
|
+
// resolveSessionLaunchModel). Copying it cannot resurrect the old frozen
|
|
5959
|
+
// bug, because the live bot model outranks it whenever the CLIs match.
|
|
5960
|
+
// An explicit per-trigger override rides on the runtime childDs below.
|
|
4173
5961
|
// readIsolation is intentionally NOT copied: it is not a persisted Session
|
|
4174
5962
|
// field (forkWorker derives it from botCfg at spawn), and the child runs the
|
|
4175
5963
|
// SAME bot, so it is preserved automatically. persistentBackendTarget is also
|
|
@@ -4180,8 +5968,8 @@ export async function forkSession(sessionId, targetChatId, targetRootMessageId,
|
|
|
4180
5968
|
childSession.sandboxHidePaths = ds.session.sandboxHidePaths;
|
|
4181
5969
|
childSession.sandboxReadonlyPaths = ds.session.sandboxReadonlyPaths;
|
|
4182
5970
|
childSession.sandboxNetwork = ds.session.sandboxNetwork;
|
|
4183
|
-
childSession.model = ds.session.model;
|
|
4184
5971
|
childSession.reasoningEffort = ds.session.reasoningEffort;
|
|
5972
|
+
childSession.model = ds.session.model;
|
|
4185
5973
|
childSession.cliRuntime = ds.session.cliRuntime
|
|
4186
5974
|
? { ...ds.session.cliRuntime, update: { ...ds.session.cliRuntime.update } }
|
|
4187
5975
|
: undefined;
|
|
@@ -4213,6 +6001,9 @@ export async function forkSession(sessionId, targetChatId, targetRootMessageId,
|
|
|
4213
6001
|
streamCardNonce: undefined,
|
|
4214
6002
|
displayMode: ds.displayMode ?? 'hidden',
|
|
4215
6003
|
suppressRecoveryCard: false,
|
|
6004
|
+
// Explicit per-trigger model override travels with the runtime session, so a
|
|
6005
|
+
// fork of a trigger session launches on the same model the caller asked for.
|
|
6006
|
+
...(ds.spawnModelOverride ? { spawnModelOverride: ds.spawnModelOverride } : {}),
|
|
4216
6007
|
};
|
|
4217
6008
|
if (activeSessionsRegistry) {
|
|
4218
6009
|
if (!(await setActiveSessionSafe(activeSessionsRegistry, targetKey, childDs))) {
|
|
@@ -4698,15 +6489,50 @@ function rollbackWorkerForkPreInit(ds, queuedSnapshot, acceptedDispatch) {
|
|
|
4698
6489
|
if (changed)
|
|
4699
6490
|
sessionStore.updateSession(ds.session);
|
|
4700
6491
|
}
|
|
6492
|
+
/** Finish ordinary-turn recovery bookkeeping only after a fresh Lark turn has
|
|
6493
|
+
* crossed its real admission boundary (durable activation tail or worker
|
|
6494
|
+
* input). The user turn is already accepted at this point, so persistence
|
|
6495
|
+
* failures here are logged without making the caller retry and risk a duplicate
|
|
6496
|
+
* execution. */
|
|
6497
|
+
function recordAdmittedOrdinaryUserTurn(ds, turnId, opts) {
|
|
6498
|
+
const priorRecoveryStatus = ds.session.ordinaryTurnRecovery?.status;
|
|
6499
|
+
let recoveryBookkeepingSucceeded = false;
|
|
6500
|
+
try {
|
|
6501
|
+
if (opts.beginRecovery) {
|
|
6502
|
+
const state = beginOrdinaryTurnRecovery(ds.session, turnId);
|
|
6503
|
+
recoveryBookkeepingSucceeded = state?.logicalTurnId === turnId
|
|
6504
|
+
&& state.currentTurnId === turnId;
|
|
6505
|
+
}
|
|
6506
|
+
else {
|
|
6507
|
+
const state = cancelOrdinaryRecoveryForUserInput(ds.session, turnId);
|
|
6508
|
+
recoveryBookkeepingSucceeded = state?.status === 'cancelled'
|
|
6509
|
+
&& state.cancelledByTurnId === turnId;
|
|
6510
|
+
}
|
|
6511
|
+
}
|
|
6512
|
+
catch (err) {
|
|
6513
|
+
logger.error(`[${tag(ds)}] Failed to persist ordinary-turn recovery bookkeeping `
|
|
6514
|
+
+ `after admitting ${turnId.substring(0, 16)}: `
|
|
6515
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
6516
|
+
}
|
|
6517
|
+
if (recoveryBookkeepingSucceeded
|
|
6518
|
+
&& (priorRecoveryStatus === 'exhausted' || priorRecoveryStatus === 'attention_required')
|
|
6519
|
+
&& ds.agentAttention) {
|
|
6520
|
+
ds.agentAttention = undefined;
|
|
6521
|
+
publishAttentionPatch(ds);
|
|
6522
|
+
}
|
|
6523
|
+
}
|
|
4701
6524
|
/** Send one normal (non-raw) worker turn while applying the per-bot Codex App
|
|
4702
6525
|
* clean-input gate at message acceptance time. This freezes the sidecar onto
|
|
4703
6526
|
* the IPC item, so later config flips do not mutate an already queued turn. */
|
|
4704
6527
|
export function sendWorkerInput(ds, payload, turnId, opts = {}) {
|
|
4705
|
-
const
|
|
4706
|
-
if (
|
|
4707
|
-
|
|
4708
|
-
|
|
4709
|
-
|
|
6528
|
+
const remoteRetirementPhase = remoteRetirementAdmissionPhase(ds);
|
|
6529
|
+
if (remoteRetirementPhase) {
|
|
6530
|
+
const remoteBackend = (ds.initConfig?.backendType ?? ds.session.backendType) === 'mojo'
|
|
6531
|
+
? 'Mojo'
|
|
6532
|
+
: 'Riff';
|
|
6533
|
+
logger.warn(`[${tag(ds)}] Rejected turn ${turnId ?? '?'} while ${remoteBackend} retirement fence is ${remoteRetirementPhase}`);
|
|
6534
|
+
void callbacks?.sessionReply(sessionAnchorId(ds), tr('worker.remote_close_in_progress', { backend: remoteBackend }, localeForBot(ds.larkAppId)), 'text', ds.larkAppId, turnId).catch(err => {
|
|
6535
|
+
logger.warn(`[${tag(ds)}] Failed to notify rejected remote close-race turn: `
|
|
4710
6536
|
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
4711
6537
|
});
|
|
4712
6538
|
return false;
|
|
@@ -4757,6 +6583,7 @@ export function sendWorkerInput(ds, payload, turnId, opts = {}) {
|
|
|
4757
6583
|
// tail's frozen payload so promote/repark/restore COPY it verbatim
|
|
4758
6584
|
// (admission computed once; never re-inferred downstream).
|
|
4759
6585
|
...(opts.codexAppSteerable ? { codexAppSteerable: true } : {}),
|
|
6586
|
+
...(opts.trustedCaller ? { trustedCaller: opts.trustedCaller } : {}),
|
|
4760
6587
|
},
|
|
4761
6588
|
turnId: queuedTurnId,
|
|
4762
6589
|
...(opts.dispatchAttempt !== undefined
|
|
@@ -4764,6 +6591,9 @@ export function sendWorkerInput(ds, payload, turnId, opts = {}) {
|
|
|
4764
6591
|
: {}),
|
|
4765
6592
|
});
|
|
4766
6593
|
logger.info(`[${tag(ds)}] Staged turn ${queuedTurnId} behind queued activation ACK`);
|
|
6594
|
+
if (turnId?.startsWith('om_')) {
|
|
6595
|
+
recordAdmittedOrdinaryUserTurn(ds, turnId, { beginRecovery: false });
|
|
6596
|
+
}
|
|
4767
6597
|
return true;
|
|
4768
6598
|
}
|
|
4769
6599
|
catch (err) {
|
|
@@ -4793,6 +6623,12 @@ export function sendWorkerInput(ds, payload, turnId, opts = {}) {
|
|
|
4793
6623
|
const message = {
|
|
4794
6624
|
type: 'message',
|
|
4795
6625
|
content: normalized.content,
|
|
6626
|
+
// A parked (crash-loop stopped) worker respawns the CLI from INSIDE the
|
|
6627
|
+
// worker when this message arrives, and that path has no restart IPC to
|
|
6628
|
+
// refresh its launch snapshot — so carry the current model on the message
|
|
6629
|
+
// itself. Only while parked: an ordinary turn changes no launch config, and
|
|
6630
|
+
// an unconditional field would be pure noise on every message.
|
|
6631
|
+
...(ds.crashDiagnosticParked ? { model: latestModelForRespawn(ds) } : {}),
|
|
4796
6632
|
...(codexAppInput ? { codexAppInput } : {}),
|
|
4797
6633
|
...(nativeSessionTitle ? { nativeSessionTitle } : {}),
|
|
4798
6634
|
...(nativeSessionTitlePrompt ? { nativeSessionTitlePrompt } : {}),
|
|
@@ -4802,10 +6638,15 @@ export function sendWorkerInput(ds, payload, turnId, opts = {}) {
|
|
|
4802
6638
|
...(codexAppDispatchId ? { codexAppDispatchId } : {}),
|
|
4803
6639
|
...(opts.codexAppSteerable ? { codexAppSteerable: true } : {}),
|
|
4804
6640
|
...(opts.atMostOnce ? { atMostOnce: true } : {}),
|
|
6641
|
+
...(opts.trustedCaller ? { trustedCaller: opts.trustedCaller } : {}),
|
|
4805
6642
|
...(vcMeetingImTurnOrigin
|
|
4806
6643
|
? { vcMeetingImTurnOrigin }
|
|
4807
6644
|
: {}),
|
|
6645
|
+
...(mojoLivePatchForSession(ds) ?? {}),
|
|
4808
6646
|
};
|
|
6647
|
+
// First moment with a reply context after a restore/resume-time quarantine.
|
|
6648
|
+
deliverPendingMojoQuarantineNotice(ds, turnId, opts.dispatchAttempt);
|
|
6649
|
+
deliverPendingMojoLegacyPinNotice(ds, turnId, opts.dispatchAttempt);
|
|
4809
6650
|
// #597: the accept-ledger entry is already appended above; if the IPC send
|
|
4810
6651
|
// fails (returns false or throws) we MUST roll it back or the dispatch ledger
|
|
4811
6652
|
// keeps a phantom accepted turn. master's ordinary-IM-delivery tracking runs
|
|
@@ -4826,8 +6667,199 @@ export function sendWorkerInput(ds, payload, turnId, opts = {}) {
|
|
|
4826
6667
|
rollbackAcceptedCodexAppDispatch(ds, codexAppDispatchId, effectiveTurnId, opts.dispatchAttempt);
|
|
4827
6668
|
return false;
|
|
4828
6669
|
}
|
|
6670
|
+
if (turnId?.startsWith('om_')) {
|
|
6671
|
+
recordAdmittedOrdinaryUserTurn(ds, turnId, { beginRecovery: true });
|
|
6672
|
+
}
|
|
4829
6673
|
return true;
|
|
4830
6674
|
}
|
|
6675
|
+
/**
|
|
6676
|
+
* Complete live-credential snapshot to ride along with a turn, so a live mojo
|
|
6677
|
+
* session picks up a rotated / cleared JWT without a refork.
|
|
6678
|
+
*
|
|
6679
|
+
* Sent on EVERY turn rather than only when it differs from the init snapshot. The
|
|
6680
|
+
* daemon cannot know which patches a live worker has already applied, so diffing
|
|
6681
|
+
* against init was wrong in both directions:
|
|
6682
|
+
* - a deleted `mojo.jwt` produced no patch, leaving the old token in place;
|
|
6683
|
+
* - `init A → patch B → config rolled back to A` compared A against A and sent
|
|
6684
|
+
* nothing, leaving the backend on B.
|
|
6685
|
+
* The backend de-duplicates against its own current value, which is the only
|
|
6686
|
+
* authoritative source.
|
|
6687
|
+
*
|
|
6688
|
+
* The control plane is NOT included — it stays frozen for the session's lifetime.
|
|
6689
|
+
*/
|
|
6690
|
+
export function mojoLivePatchForSession(ds) {
|
|
6691
|
+
const backendType = ds.initConfig?.backendType ?? ds.session.backendType;
|
|
6692
|
+
if (backendType !== 'mojo')
|
|
6693
|
+
return undefined;
|
|
6694
|
+
let botCfg;
|
|
6695
|
+
try {
|
|
6696
|
+
botCfg = getBot(ds.larkAppId).config;
|
|
6697
|
+
}
|
|
6698
|
+
catch {
|
|
6699
|
+
return undefined;
|
|
6700
|
+
}
|
|
6701
|
+
// Resolves jwtEnv daemon-side across the same layers buildEnv() uses, so the
|
|
6702
|
+
// worker receives a value and never an env map.
|
|
6703
|
+
return {
|
|
6704
|
+
mojoLivePatch: pickMojoLivePatch(botCfg.mojo, {
|
|
6705
|
+
genericEnv: botCfg.env ? sanitizePerBotEnv(botCfg.env) : undefined,
|
|
6706
|
+
}),
|
|
6707
|
+
};
|
|
6708
|
+
}
|
|
6709
|
+
/**
|
|
6710
|
+
* Is auxiliary UI an authorized output channel for this session/turn?
|
|
6711
|
+
*
|
|
6712
|
+
* The single source of truth for this policy: setupWorkerHandlers' own
|
|
6713
|
+
* `managedAuxUiSuppressed` now delegates here instead of keeping a parallel copy
|
|
6714
|
+
* that could drift. Deliberately conservative: a
|
|
6715
|
+
* dedicated VC receiver, a silent scheduled turn, or a session with no real Lark
|
|
6716
|
+
* transport must never receive a bypass message. Those cases still get the
|
|
6717
|
+
* persisted quarantine state for dashboard/audit — the notice is only about the
|
|
6718
|
+
* chat channel.
|
|
6719
|
+
*/
|
|
6720
|
+
/** Sessions with a quarantine notice send in flight, so it cannot double-post. */
|
|
6721
|
+
const mojoQuarantineNoticeInFlight = new Set();
|
|
6722
|
+
/**
|
|
6723
|
+
* THE auxiliary-UI suppression policy. One definition, two call sites.
|
|
6724
|
+
*
|
|
6725
|
+
* Previously setupWorkerHandlers held this as a closure and the quarantine notice
|
|
6726
|
+
* COPIED three of its four checks — dropping `ordinaryManagedSuppression`, so a
|
|
6727
|
+
* durable-suppressed turn still posted to Lark. Copying a policy is how the two
|
|
6728
|
+
* diverge; this is the shared implementation both now call.
|
|
6729
|
+
*
|
|
6730
|
+
* Returns true when auxiliary UI must be suppressed.
|
|
6731
|
+
*/
|
|
6732
|
+
export function auxUiSuppressedFor(ds, turnId, dispatchAttempt) {
|
|
6733
|
+
// No-transport session (apiOnly bot or HTTP virtual chat): there is no real
|
|
6734
|
+
// Feishu chat to render into.
|
|
6735
|
+
try {
|
|
6736
|
+
if (!larkTransportEnabled({
|
|
6737
|
+
chatId: ds.chatId,
|
|
6738
|
+
apiOnly: getBot(ds.larkAppId).config.apiOnly,
|
|
6739
|
+
}))
|
|
6740
|
+
return true;
|
|
6741
|
+
}
|
|
6742
|
+
catch {
|
|
6743
|
+
// Bot deregistered — fail closed.
|
|
6744
|
+
return true;
|
|
6745
|
+
}
|
|
6746
|
+
if (isSilentScheduledTurn(ds, turnId))
|
|
6747
|
+
return true;
|
|
6748
|
+
if (ds.session.vcMeetingReceiver)
|
|
6749
|
+
return true;
|
|
6750
|
+
// Durable ledger: an attempt at or below the armed watermark is a replay whose
|
|
6751
|
+
// output already happened (or was deliberately suppressed).
|
|
6752
|
+
const armedThrough = turnId ? ds.suppressedFinalOutputTurns?.get(turnId) : undefined;
|
|
6753
|
+
return dispatchAttempt !== undefined
|
|
6754
|
+
&& armedThrough !== undefined
|
|
6755
|
+
&& dispatchAttempt <= armedThrough;
|
|
6756
|
+
}
|
|
6757
|
+
/**
|
|
6758
|
+
* Deliver the one-time notice that a mojo session's earlier remote lineage was
|
|
6759
|
+
* parked, then clear the flag.
|
|
6760
|
+
*
|
|
6761
|
+
* Quarantining happens during restore/resume where there is no reply context, so
|
|
6762
|
+
* the notice is deferred to the first ACCEPTED turn — hot send and cold fork both
|
|
6763
|
+
* call this, since a cold-resumed session goes straight to forkWorker and would
|
|
6764
|
+
* otherwise never deliver it.
|
|
6765
|
+
*
|
|
6766
|
+
* The flag is marked delivered (`false`) only AFTER a successful send, so a
|
|
6767
|
+
* transient Lark failure retries on the next turn instead of losing the notice
|
|
6768
|
+
* permanently. For a suppressed channel (durable replay / silent schedule / VC
|
|
6769
|
+
* receiver / no transport) the flag STAYS PENDING: that turn merely had no
|
|
6770
|
+
* authorized output channel, so a later ordinary IM turn delivers it. The
|
|
6771
|
+
* quarantine state itself is persisted for dashboard/audit either way.
|
|
6772
|
+
*/
|
|
6773
|
+
function deliverPendingMojoQuarantineNotice(ds, turnId, dispatchAttempt) {
|
|
6774
|
+
if (ds.session.mojoQuarantineNoticePending !== true)
|
|
6775
|
+
return;
|
|
6776
|
+
// In-flight guard: two turns can be accepted before the first sessionReply
|
|
6777
|
+
// resolves, which would post the notice twice.
|
|
6778
|
+
if (mojoQuarantineNoticeInFlight.has(ds.session.sessionId))
|
|
6779
|
+
return;
|
|
6780
|
+
const lineage = ds.session.mojoQuarantinedLineage;
|
|
6781
|
+
if (!lineage) {
|
|
6782
|
+
ds.session.mojoQuarantineNoticePending = undefined;
|
|
6783
|
+
sessionStore.updateSession(ds.session);
|
|
6784
|
+
return;
|
|
6785
|
+
}
|
|
6786
|
+
let botCfg;
|
|
6787
|
+
try {
|
|
6788
|
+
botCfg = getBot(ds.larkAppId).config;
|
|
6789
|
+
}
|
|
6790
|
+
catch {
|
|
6791
|
+
return; // Bot deregistered — retry once it is back.
|
|
6792
|
+
}
|
|
6793
|
+
// `false` = DELIVERED, distinct from `undefined` = never queued. Clearing to
|
|
6794
|
+
// undefined made the restore migration read it as "old build, never notified"
|
|
6795
|
+
// and queue the notice again after every restart.
|
|
6796
|
+
const markDelivered = () => {
|
|
6797
|
+
ds.session.mojoQuarantineNoticePending = false;
|
|
6798
|
+
sessionStore.updateSession(ds.session);
|
|
6799
|
+
};
|
|
6800
|
+
if (auxUiSuppressedFor(ds, turnId, dispatchAttempt)) {
|
|
6801
|
+
// Stay PENDING: this turn simply had no authorized channel (durable replay,
|
|
6802
|
+
// silent schedule, VC receiver, no transport). A later ordinary IM turn
|
|
6803
|
+
// delivers it. The quarantine state itself is already persisted for
|
|
6804
|
+
// dashboard/audit either way.
|
|
6805
|
+
logger.info(`[${tag(ds)}] mojo quarantine notice deferred — no authorized output channel `
|
|
6806
|
+
+ 'for this turn; still pending');
|
|
6807
|
+
return;
|
|
6808
|
+
}
|
|
6809
|
+
mojoQuarantineNoticeInFlight.add(ds.session.sessionId);
|
|
6810
|
+
const message = tr('worker.mojo_lineage_quarantined', { lineage }, botLocale(botCfg));
|
|
6811
|
+
void callbacks?.sessionReply(sessionAnchorId(ds), message, 'text', ds.larkAppId,
|
|
6812
|
+
// The turn that actually carried this delivery, not a stale reply target.
|
|
6813
|
+
turnId, ds.session.vcMeetingReceiver ? { sourceSessionId: ds.session.sessionId } : undefined).then(markDelivered).catch((err) => {
|
|
6814
|
+
// Flag intentionally left set: the next turn retries.
|
|
6815
|
+
logger.warn(`[${tag(ds)}] failed to deliver mojo quarantine notice (will retry): ${err}`);
|
|
6816
|
+
}).finally(() => {
|
|
6817
|
+
mojoQuarantineNoticeInFlight.delete(ds.session.sessionId);
|
|
6818
|
+
});
|
|
6819
|
+
}
|
|
6820
|
+
const mojoLegacyPinNoticeInFlight = new Set();
|
|
6821
|
+
/** In-chat twin of the sessionMojoConfig legacy pin's daemon-log warn: without
|
|
6822
|
+
* it the user keeps retrying a session whose tools and replies silently fail
|
|
6823
|
+
* (observed live). Same pending/delivered protocol and suppression rules as
|
|
6824
|
+
* deliverPendingMojoQuarantineNotice above. */
|
|
6825
|
+
function deliverPendingMojoLegacyPinNotice(ds, turnId, dispatchAttempt) {
|
|
6826
|
+
if (ds.session.mojoLegacyPinNoticePending !== true)
|
|
6827
|
+
return;
|
|
6828
|
+
if (mojoLegacyPinNoticeInFlight.has(ds.session.sessionId))
|
|
6829
|
+
return;
|
|
6830
|
+
let botCfg;
|
|
6831
|
+
try {
|
|
6832
|
+
botCfg = getBot(ds.larkAppId).config;
|
|
6833
|
+
}
|
|
6834
|
+
catch {
|
|
6835
|
+
return; // Bot deregistered — retry once it is back.
|
|
6836
|
+
}
|
|
6837
|
+
if (auxUiSuppressedFor(ds, turnId, dispatchAttempt)) {
|
|
6838
|
+
logger.info(`[${tag(ds)}] mojo legacy-pin notice deferred — no authorized output channel `
|
|
6839
|
+
+ 'for this turn; still pending');
|
|
6840
|
+
return;
|
|
6841
|
+
}
|
|
6842
|
+
mojoLegacyPinNoticeInFlight.add(ds.session.sessionId);
|
|
6843
|
+
const message = tr('worker.mojo_legacy_pinned', {}, botLocale(botCfg));
|
|
6844
|
+
void callbacks?.sessionReply(sessionAnchorId(ds), message, 'text', ds.larkAppId, turnId, ds.session.vcMeetingReceiver ? { sourceSessionId: ds.session.sessionId } : undefined).then(() => {
|
|
6845
|
+
ds.session.mojoLegacyPinNoticePending = false;
|
|
6846
|
+
// Best-effort, same as the queueing side (review N2): the notice IS
|
|
6847
|
+
// delivered at this point — a failed persist only risks one duplicate
|
|
6848
|
+
// send after a restart, and must not surface as a delivery failure.
|
|
6849
|
+
try {
|
|
6850
|
+
sessionStore.updateSession(ds.session);
|
|
6851
|
+
}
|
|
6852
|
+
catch (err) {
|
|
6853
|
+
logger.warn(`[${tag(ds)}] mojo legacy-pin notice delivered but the delivered marker could not be `
|
|
6854
|
+
+ `persisted (may re-send once after a restart): ${err instanceof Error ? err.message : String(err)}`);
|
|
6855
|
+
}
|
|
6856
|
+
}).catch((err) => {
|
|
6857
|
+
// Flag intentionally left set: the next turn retries.
|
|
6858
|
+
logger.warn(`[${tag(ds)}] failed to deliver mojo legacy-pin notice (will retry): ${err}`);
|
|
6859
|
+
}).finally(() => {
|
|
6860
|
+
mojoLegacyPinNoticeInFlight.delete(ds.session.sessionId);
|
|
6861
|
+
});
|
|
6862
|
+
}
|
|
4831
6863
|
/** Promote the oldest durable activation successor into a fresh tokened
|
|
4832
6864
|
* journal and (for Codex App) its single accepted-ledger owner in one store
|
|
4833
6865
|
* update, then hand it to the current worker. The tail is never shifted merely
|
|
@@ -4871,6 +6903,7 @@ export function promoteQueuedActivationTail(ds, opts = {}) {
|
|
|
4871
6903
|
// queuedActivationInput so the ensuing fork/accept-ledger COPY (below) sees
|
|
4872
6904
|
// it. Only `=== true`; a missing/false tail head stays forced-serial.
|
|
4873
6905
|
...(head.cliInput.codexAppSteerable === true ? { codexAppSteerable: true } : {}),
|
|
6906
|
+
...(head.cliInput.trustedCaller ? { trustedCaller: head.cliInput.trustedCaller } : {}),
|
|
4874
6907
|
};
|
|
4875
6908
|
const vcMeetingImTurnOrigin = resolveVcMeetingImTurnOrigin(ds.session, head.turnId);
|
|
4876
6909
|
ds.session.queued = false;
|
|
@@ -4922,6 +6955,8 @@ export function promoteQueuedActivationTail(ds, opts = {}) {
|
|
|
4922
6955
|
ds.worker.send({
|
|
4923
6956
|
type: 'message',
|
|
4924
6957
|
content: exactInput.content,
|
|
6958
|
+
// Same parked-worker launch-snapshot refresh as the ordinary send.
|
|
6959
|
+
...(ds.crashDiagnosticParked ? { model: latestModelForRespawn(ds) } : {}),
|
|
4925
6960
|
...(codexAppInput ? { codexAppInput } : {}),
|
|
4926
6961
|
turnId: head.turnId,
|
|
4927
6962
|
...(head.dispatchAttempt !== undefined
|
|
@@ -4933,9 +6968,13 @@ export function promoteQueuedActivationTail(ds, opts = {}) {
|
|
|
4933
6968
|
// ledger head steerable=true but the worker reservation false, so a
|
|
4934
6969
|
// legitimate superseded settlement would be wrongly rejected.
|
|
4935
6970
|
...(exactInput.codexAppSteerable === true ? { codexAppSteerable: true } : {}),
|
|
6971
|
+
...(exactInput.trustedCaller ? { trustedCaller: exactInput.trustedCaller } : {}),
|
|
4936
6972
|
queuedActivationToken: token,
|
|
4937
6973
|
...(vcMeetingImTurnOrigin ? { vcMeetingImTurnOrigin } : {}),
|
|
4938
6974
|
});
|
|
6975
|
+
if (head.turnId.startsWith('om_')) {
|
|
6976
|
+
recordAdmittedOrdinaryUserTurn(ds, head.turnId, { beginRecovery: true });
|
|
6977
|
+
}
|
|
4939
6978
|
}
|
|
4940
6979
|
catch (err) {
|
|
4941
6980
|
// Durable ownership already moved to the journal (and Codex ledger). Never
|
|
@@ -4984,6 +7023,7 @@ export function admitQueuedActivationTail(ds, entry, reservation = reserveQueued
|
|
|
4984
7023
|
// (only `=== true`, never truthy). Dropping it here silently un-authorized
|
|
4985
7024
|
// every queued/opening turn — the strip point that defeated the R4 fix.
|
|
4986
7025
|
...(entry.cliInput.codexAppSteerable === true ? { codexAppSteerable: true } : {}),
|
|
7026
|
+
...(entry.cliInput.trustedCaller ? { trustedCaller: entry.cliInput.trustedCaller } : {}),
|
|
4987
7027
|
},
|
|
4988
7028
|
};
|
|
4989
7029
|
const priorTail = ds.session.queuedActivationTail;
|
|
@@ -5080,6 +7120,35 @@ export function resolveQuarantinedForkPlan(ds, promptInput, resumeOrTurnId) {
|
|
|
5080
7120
|
*/
|
|
5081
7121
|
export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
5082
7122
|
const gatedPrompt = typeof promptInput === 'string' ? { content: promptInput } : promptInput;
|
|
7123
|
+
const remoteRetirementPhase = remoteRetirementAdmissionPhase(ds);
|
|
7124
|
+
if (remoteRetirementPhase) {
|
|
7125
|
+
// A close/shutdown coordinator owns this exact remote generation. Never
|
|
7126
|
+
// spawn a replacement behind its back: a fork can materialize the opening
|
|
7127
|
+
// prompt before sendWorkerInput gets a chance to reject it. Route non-empty
|
|
7128
|
+
// input through the ordinary fence solely for the user-visible warning.
|
|
7129
|
+
if (gatedPrompt.content !== '') {
|
|
7130
|
+
const gatedTurnId = typeof resumeOrTurnId === 'string'
|
|
7131
|
+
? resumeOrTurnId
|
|
7132
|
+
: typeof resumeOrTurnId === 'object' && resumeOrTurnId !== null
|
|
7133
|
+
? resumeOrTurnId.turnId
|
|
7134
|
+
: undefined;
|
|
7135
|
+
const gatedDispatchAttempt = typeof resumeOrTurnId === 'object' && resumeOrTurnId !== null
|
|
7136
|
+
? resumeOrTurnId.dispatchAttempt
|
|
7137
|
+
: undefined;
|
|
7138
|
+
sendWorkerInput(ds, promptInput, gatedTurnId, {
|
|
7139
|
+
...(gatedDispatchAttempt !== undefined
|
|
7140
|
+
? { dispatchAttempt: gatedDispatchAttempt }
|
|
7141
|
+
: {}),
|
|
7142
|
+
...(gatedPrompt.codexAppSteerable === true
|
|
7143
|
+
? { codexAppSteerable: true }
|
|
7144
|
+
: {}),
|
|
7145
|
+
});
|
|
7146
|
+
}
|
|
7147
|
+
logger.warn(`[${tag(ds)}] Refused worker fork while remote retirement fence is ${remoteRetirementPhase}`);
|
|
7148
|
+
// The request was handled by the retirement fence; false is reserved for
|
|
7149
|
+
// tail-only quarantine promotion failures by this function's contract.
|
|
7150
|
+
return true;
|
|
7151
|
+
}
|
|
5083
7152
|
const transferGate = transferInputGates.get(ds);
|
|
5084
7153
|
if (transferGate && !transferReplacementForkBypass.has(ds)) {
|
|
5085
7154
|
if (gatedPrompt.content !== '') {
|
|
@@ -5094,6 +7163,10 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5094
7163
|
const gatedAtMostOnce = typeof resumeOrTurnId === 'object' && resumeOrTurnId !== null
|
|
5095
7164
|
? resumeOrTurnId.atMostOnce
|
|
5096
7165
|
: undefined;
|
|
7166
|
+
const gatedTrustedCaller = gatedPrompt.trustedCaller
|
|
7167
|
+
?? (typeof resumeOrTurnId === 'object' && resumeOrTurnId !== null
|
|
7168
|
+
? resumeOrTurnId.trustedCaller
|
|
7169
|
+
: undefined);
|
|
5097
7170
|
sendWorkerInput(ds, promptInput, gatedTurnId, {
|
|
5098
7171
|
...(gatedDispatchAttempt !== undefined
|
|
5099
7172
|
? { dispatchAttempt: gatedDispatchAttempt }
|
|
@@ -5109,6 +7182,7 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5109
7182
|
// terminalized the turn re-runs it. forkWorker's own init path sets this
|
|
5110
7183
|
// from resumeOrTurnId.atMostOnce; the reroute must forward it identically.
|
|
5111
7184
|
...(gatedAtMostOnce ? { atMostOnce: true } : {}),
|
|
7185
|
+
...(gatedTrustedCaller ? { trustedCaller: gatedTrustedCaller } : {}),
|
|
5112
7186
|
});
|
|
5113
7187
|
}
|
|
5114
7188
|
else {
|
|
@@ -5180,6 +7254,10 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5180
7254
|
else {
|
|
5181
7255
|
resume = resumeOrTurnId;
|
|
5182
7256
|
}
|
|
7257
|
+
const initTrustedCaller = promptPayload.trustedCaller
|
|
7258
|
+
?? (typeof resumeOrTurnId === 'object' && resumeOrTurnId !== null
|
|
7259
|
+
? resumeOrTurnId.trustedCaller
|
|
7260
|
+
: undefined);
|
|
5183
7261
|
if (ds.session.queuedActivationPending && ds.session.queuedActivationResume !== undefined) {
|
|
5184
7262
|
resume = ds.session.queuedActivationResume;
|
|
5185
7263
|
}
|
|
@@ -5207,6 +7285,7 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5207
7285
|
// R5-B1-2: preserve the frozen steer authorization on the double-fork
|
|
5208
7286
|
// staged tail entry (only `=== true`).
|
|
5209
7287
|
...(initCodexAppSteerable ? { codexAppSteerable: true } : {}),
|
|
7288
|
+
...(initTrustedCaller ? { trustedCaller: initTrustedCaller } : {}),
|
|
5210
7289
|
},
|
|
5211
7290
|
turnId,
|
|
5212
7291
|
...(initDispatchAttempt !== undefined
|
|
@@ -5225,6 +7304,7 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5225
7304
|
// opts too. The gate branch above already admits with the flag; this
|
|
5226
7305
|
// sibling branch would otherwise silently downgrade true → false.
|
|
5227
7306
|
...(initCodexAppSteerable ? { codexAppSteerable: true } : {}),
|
|
7307
|
+
...(initTrustedCaller ? { trustedCaller: initTrustedCaller } : {}),
|
|
5228
7308
|
});
|
|
5229
7309
|
logger[routed ? 'info' : 'warn'](`[${tag(ds)}] ${routed ? 'Routed' : 'Failed to route'} double-fork prompt through existing durable owner`);
|
|
5230
7310
|
return true;
|
|
@@ -5603,15 +7683,15 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5603
7683
|
else {
|
|
5604
7684
|
reparkQueuedActivationFollowUpTail(ds, 'worker error during activation follow-up handoff');
|
|
5605
7685
|
}
|
|
5606
|
-
const retainExactRetirementGeneration = ds.
|
|
5607
|
-
|| ds.
|
|
7686
|
+
const retainExactRetirementGeneration = ds.remoteShutdownState !== undefined
|
|
7687
|
+
|| ds.remoteCloseState !== undefined;
|
|
5608
7688
|
if (!retainExactRetirementGeneration) {
|
|
5609
7689
|
ds.worker = null;
|
|
5610
7690
|
ds.workerPort = null;
|
|
5611
7691
|
ds.workerToken = null;
|
|
5612
7692
|
ds.workerViewToken = null;
|
|
5613
7693
|
ds.managedTurnOrigin = undefined;
|
|
5614
|
-
ds.
|
|
7694
|
+
ds.remoteCloseState = undefined;
|
|
5615
7695
|
}
|
|
5616
7696
|
// The retirement coordinator owns a prepared generation. Keep the exact
|
|
5617
7697
|
// ChildProcess pointer until exit so it can decide whether abort/commit
|
|
@@ -5701,16 +7781,23 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5701
7781
|
// Per-bot local read isolation (enforced worker-side; the worker gates it).
|
|
5702
7782
|
// Sibling data needs no app-id enumeration: per-bot dirs are denied wholesale
|
|
5703
7783
|
// and per-bot session files by filename pattern (see buildV2DenyPaths).
|
|
5704
|
-
//
|
|
5705
|
-
//
|
|
5706
|
-
//
|
|
5707
|
-
//
|
|
5708
|
-
//
|
|
5709
|
-
//
|
|
5710
|
-
//
|
|
5711
|
-
//
|
|
5712
|
-
|
|
5713
|
-
|
|
7784
|
+
// Opt-in only, driven purely by explicit per-bot `readIsolation`. A
|
|
7785
|
+
// no-transport session (apiOnly bot OR HTTP virtual chat) is NO LONGER
|
|
7786
|
+
// force-isolated: disk read scope now follows the owner's own sandbox config,
|
|
7787
|
+
// symmetric with a normal chat session (unset/false → not isolated). Accepted
|
|
7788
|
+
// trade-off: a no-transport session with no sandbox config can read the full
|
|
7789
|
+
// bots.json / sibling BOT_HOME on disk; protecting sibling creds from lateral
|
|
7790
|
+
// read on a multi-bot host now depends on the owner explicitly enabling
|
|
7791
|
+
// sandbox/readIsolation, not on this force. Two adjacent boundaries are
|
|
7792
|
+
// unchanged and independent: (1) this bot's own transport secret is still
|
|
7793
|
+
// withheld from the CLI env (gated on larkTransportEnabled below), so a
|
|
7794
|
+
// no-transport session cannot drive Botmux's own send path even though it can
|
|
7795
|
+
// read the file; (2) mandatory device-credential isolation (worker.ts) still
|
|
7796
|
+
// masks the device authority dir / enrolled creds on enrolled hosts. Full-file
|
|
7797
|
+
// sandbox stays independently driven worker-side by sandboxRequested
|
|
7798
|
+
// (cfg.sandbox || cfg.readIsolation || BOTMUX_SANDBOX=1); session.sandbox is
|
|
7799
|
+
// frozen from botCfg.sandbox at create time, so "follow local sandbox" holds.
|
|
7800
|
+
readIsolation: botCfg.readIsolation === true,
|
|
5714
7801
|
readDenyExtraPaths: botCfg.readDenyExtraPaths ?? [],
|
|
5715
7802
|
// Identifies THIS daemon lifetime. Stamped onto isolated panes so the worker
|
|
5716
7803
|
// can tell a suspend→resume reattach (same boot id, still isolated) from a
|
|
@@ -5726,7 +7813,18 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5726
7813
|
// Shared Herdr is not derivable from sessionId: preserve the exact host +
|
|
5727
7814
|
// managed-agent affinity across daemon/worker replacement.
|
|
5728
7815
|
persistentBackendTarget: ds.session.persistentBackendTarget,
|
|
5729
|
-
backendConfig
|
|
7816
|
+
// One backendConfig channel shared by the remote backends; the selector
|
|
7817
|
+
// picks the shape matching backendType. A mojo bot with no `mojo` block is
|
|
7818
|
+
// valid (all fields optional), so an undefined here is not an error.
|
|
7819
|
+
//
|
|
7820
|
+
// mojo goes through sessionMojoConfig so the control-plane identity frozen at
|
|
7821
|
+
// creation survives a cold resume: without it, editing the bot between two
|
|
7822
|
+
// messages would silently move a live session from cloud to host execution,
|
|
7823
|
+
// or resume it against a different tenant/workspace than the one holding its
|
|
7824
|
+
// remote session.
|
|
7825
|
+
backendConfig: botCfg.backendType === 'mojo' || agentCfg.cliId === 'mojo'
|
|
7826
|
+
? sessionMojoConfig(ds, botCfg, { freeze: true }).config
|
|
7827
|
+
: botCfg.riff,
|
|
5730
7828
|
riffParentTaskId: ds.session.riffParentTaskId,
|
|
5731
7829
|
riffRepoDirs: ds.session.riffRepoDirs,
|
|
5732
7830
|
deferredScheduleRun: ds.session.deferredScheduleRun,
|
|
@@ -5789,6 +7887,7 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5789
7887
|
// worker tags the keyed init prompt no-replay (codex #776 round-7 #1).
|
|
5790
7888
|
...(initAtMostOnce ? { atMostOnce: true } : {}),
|
|
5791
7889
|
vcMeetingImTurnOrigin: initVcMeetingImTurnOrigin,
|
|
7890
|
+
...(initTrustedCaller ? { trustedCaller: initTrustedCaller } : {}),
|
|
5792
7891
|
pluginBindings: botCfg.plugins,
|
|
5793
7892
|
skillPolicy: botCfg.skills,
|
|
5794
7893
|
...(runtimeIdentity.status === 'known'
|
|
@@ -5808,6 +7907,14 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5808
7907
|
}
|
|
5809
7908
|
}
|
|
5810
7909
|
ds.initConfig = initMsg;
|
|
7910
|
+
// New generation ledger. NOT a plain reset: the previous generation's keys are
|
|
7911
|
+
// parked until its worker is observed to exit, because the double-fork guard
|
|
7912
|
+
// kills asynchronously and spawns the replacement synchronously.
|
|
7913
|
+
startNewGenerationEnvLedger(ds, initMsg.env);
|
|
7914
|
+
// Cold-resume path: a workerless session goes straight here without passing
|
|
7915
|
+
// through sendWorkerInput, so without this the "notice on the next message"
|
|
7916
|
+
// promise was never kept for exactly the sessions most likely to need it.
|
|
7917
|
+
deliverPendingMojoQuarantineNotice(ds, initAttributionTurnId, initDispatchAttempt);
|
|
5811
7918
|
// Stamp cliId on the persisted session so the dashboard can show a CLI badge
|
|
5812
7919
|
// even after the session is closed. Do this before installing worker handlers:
|
|
5813
7920
|
// a fast worker can emit `ready` immediately after init, and card rendering
|
|
@@ -5836,6 +7943,9 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5836
7943
|
else {
|
|
5837
7944
|
worker.send(initMsg);
|
|
5838
7945
|
}
|
|
7946
|
+
if (prompt.length > 0 && initAttributionTurnId?.startsWith('om_')) {
|
|
7947
|
+
recordAdmittedOrdinaryUserTurn(ds, initAttributionTurnId, { beginRecovery: true });
|
|
7948
|
+
}
|
|
5839
7949
|
ds.spawnedAt = Date.now();
|
|
5840
7950
|
// master: per-runtime-key CLI version (the init send already happened above via
|
|
5841
7951
|
// the tracked-delivery if/else — do NOT re-send initMsg here).
|
|
@@ -5873,6 +7983,10 @@ export function forkWorker(ds, promptInput, resumeOrTurnId = false) {
|
|
|
5873
7983
|
throw err;
|
|
5874
7984
|
}
|
|
5875
7985
|
ds.initConfig = initMsg;
|
|
7986
|
+
// New generation ledger. NOT a plain reset: the previous generation's keys are
|
|
7987
|
+
// parked until its worker is observed to exit, because the double-fork guard
|
|
7988
|
+
// kills asynchronously and spawns the replacement synchronously.
|
|
7989
|
+
startNewGenerationEnvLedger(ds, initMsg.env);
|
|
5876
7990
|
try {
|
|
5877
7991
|
sessionStore.updateSessionPid(ds.session.sessionId, worker.pid ?? null);
|
|
5878
7992
|
}
|
|
@@ -6044,7 +8158,7 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6044
8158
|
&& ds.larkAppId === handlerLarkAppId
|
|
6045
8159
|
&& sessionAnchorId(ds) === handlerAnchor
|
|
6046
8160
|
&& (!activeSessionsRegistry
|
|
6047
|
-
|| activeSessionsRegistry.get(
|
|
8161
|
+
|| activeSessionsRegistry.get(activeSessionKey(ds)) === ds);
|
|
6048
8162
|
const ownsLifecycleMutation = () => ownsWorkerSession() && !isSessionTransferring(ds);
|
|
6049
8163
|
// A new worker generation is starting. As a backstop, invalidate any
|
|
6050
8164
|
// stuck-warning card posted by the previous generation — explicit kill/suspend/
|
|
@@ -6077,22 +8191,13 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6077
8191
|
};
|
|
6078
8192
|
/** Auxiliary worker UI is never an authorized output channel for a dedicated
|
|
6079
8193
|
* VC receiver. Dashboard/audit state is still updated before these guards. */
|
|
6080
|
-
|
|
6081
|
-
|
|
6082
|
-
|
|
6083
|
-
|
|
6084
|
-
|
|
6085
|
-
|
|
6086
|
-
|
|
6087
|
-
// still updated before these guards, so the terminal view is unaffected.
|
|
6088
|
-
if (!larkTransportEnabled({ chatId: ds.chatId, apiOnly: getBot(ds.larkAppId).config.apiOnly }))
|
|
6089
|
-
return true;
|
|
6090
|
-
if (isSilentScheduledTurn(ds, turnId))
|
|
6091
|
-
return true;
|
|
6092
|
-
if (ds.session.vcMeetingReceiver)
|
|
6093
|
-
return true;
|
|
6094
|
-
return ordinaryManagedSuppression(turnId, dispatchAttempt);
|
|
6095
|
-
};
|
|
8194
|
+
// Delegates to the shared policy (auxUiSuppressedFor) so this and the mojo
|
|
8195
|
+
// quarantine notice cannot drift apart. Suppressing here is the authoritative
|
|
8196
|
+
// fix, NOT a fake "success" message id from sessionReply (a synthetic id would
|
|
8197
|
+
// get stored as streamCardId and later scheduleCardPatch → updateMessage would
|
|
8198
|
+
// still dial Feishu). Dashboard/web-terminal state is updated before these
|
|
8199
|
+
// guards, so the terminal view is unaffected.
|
|
8200
|
+
const managedAuxUiSuppressed = (turnId, dispatchAttempt) => auxUiSuppressedFor(ds, turnId, dispatchAttempt);
|
|
6096
8201
|
/** final_output is the sole exception: listener_thread and exact IM replies
|
|
6097
8202
|
* may proceed into the durable action ledger; silent/stale attempts do not. */
|
|
6098
8203
|
const managedFinalOutputSuppressed = (turnId, dispatchAttempt) => {
|
|
@@ -6120,6 +8225,7 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6120
8225
|
const bot = getBot(ds.larkAppId);
|
|
6121
8226
|
const botCfg = bot.config;
|
|
6122
8227
|
const loc = botLocale(botCfg);
|
|
8228
|
+
ensureOrdinaryTurnRecoveryAttached(ds, botCfg);
|
|
6123
8229
|
const notifyStartupFailure = async (reason, turnId, dispatchAttempt) => {
|
|
6124
8230
|
if (startupState.failureNotified)
|
|
6125
8231
|
return;
|
|
@@ -6303,6 +8409,23 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6303
8409
|
}
|
|
6304
8410
|
startupState.ready = true;
|
|
6305
8411
|
ds.workerReady = true;
|
|
8412
|
+
// Restore the daemon-owned display mode BEFORE any card handling: a
|
|
8413
|
+
// fresh worker always boots with displayMode='hidden', and every early
|
|
8414
|
+
// break below (managed / replyAlreadySent / streamingCardDisabled /
|
|
8415
|
+
// CARD_POSTING_SENTINEL) used to skip the re-sync entirely. The
|
|
8416
|
+
// sentinel break is not even rare: a headless backend (mojo) reaches
|
|
8417
|
+
// prompt-ready synchronously on spawn, so screen_update reliably wins
|
|
8418
|
+
// the card-POST race and `ready` breaks at the sentinel — the worker
|
|
8419
|
+
// then never starts its screenshot loop while the daemon keeps
|
|
8420
|
+
// rendering "waiting for first screenshot" forever. Sending before the
|
|
8421
|
+
// card exists is safe: at worst the first screenshot_uploaded is
|
|
8422
|
+
// dropped by the streamCardPending gate and the next 10s cycle
|
|
8423
|
+
// delivers a fresh frame.
|
|
8424
|
+
syncWorkerDisplayMode(ds);
|
|
8425
|
+
// A live CLI is up again: the parked-diagnostic recovery is over, so the
|
|
8426
|
+
// per-message launch-model refresh (see sendWorkerInput) stops here and
|
|
8427
|
+
// ordinary restart/refork paths take over again.
|
|
8428
|
+
ds.crashDiagnosticParked = undefined;
|
|
6306
8429
|
// Treat `ready` as a full state boundary for the usage refresh: clear
|
|
6307
8430
|
// any timer inherited from a PRIOR worker generation up front, so the
|
|
6308
8431
|
// managed/replyAlreadySent/disabled/recovery/sentinel early-breaks below
|
|
@@ -6386,7 +8509,6 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6386
8509
|
// (if any) is left untouched. The next real user turn clears this flag
|
|
6387
8510
|
// (rememberLastCliInput) and the normal card flow resumes.
|
|
6388
8511
|
if (ds.suppressRecoveryCard) {
|
|
6389
|
-
syncWorkerDisplayMode(ds);
|
|
6390
8512
|
logger.info(`[${t}] Restored session — suppressing recovery streaming card (silent restart)`);
|
|
6391
8513
|
break;
|
|
6392
8514
|
}
|
|
@@ -6427,8 +8549,6 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6427
8549
|
scheduleCodexServiceTierPatch(ds);
|
|
6428
8550
|
}
|
|
6429
8551
|
persistStreamCardState(ds);
|
|
6430
|
-
// Re-sync worker's display mode (it starts fresh in 'hidden')
|
|
6431
|
-
syncWorkerDisplayMode(ds);
|
|
6432
8552
|
// The restored card is now the active one — withdraw any cards
|
|
6433
8553
|
// frozen before the daemon went down so they don't pile up in the
|
|
6434
8554
|
// thread on each restart.
|
|
@@ -6460,6 +8580,7 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6460
8580
|
if (ds.streamCardId === CARD_POSTING_SENTINEL)
|
|
6461
8581
|
break;
|
|
6462
8582
|
const postingGeneration = ds.streamCardTurnGeneration ?? 0;
|
|
8583
|
+
const cardReplyTarget = captureStreamingCardReplyTarget(ds, msg.turnId);
|
|
6463
8584
|
ds.streamCardId = CARD_POSTING_SENTINEL;
|
|
6464
8585
|
try {
|
|
6465
8586
|
ds.streamCardNonce = randomBytes(4).toString('hex');
|
|
@@ -6474,12 +8595,13 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6474
8595
|
// before the first screen_update lands. Fresh worker spawn
|
|
6475
8596
|
// has lastScreenContent undefined → '' (unchanged).
|
|
6476
8597
|
ds.lastScreenContent ?? '', initStatus, effectiveCliId, ds.displayMode ?? 'hidden', ds.streamCardNonce, ds.currentImageKey, isAdopt, showTakeover, loc, initStatus === 'limited' ? ds.usageLimit : undefined, writableTerminalLinkFor(ds), isLocalCliOpenReady(ds, { cliId: effectiveCliId }), getDaemonStreamingCardUsageSnapshot(ds, effectiveCliId), sessionRuntimeDisplayName(ds, botCfg), codexServiceTierBadge(effectiveCliId, ds.codexServiceTier));
|
|
6477
|
-
const postedCardId = await scopedReply(streamCardJson, 'interactive',
|
|
8598
|
+
const postedCardId = await scopedReply(streamCardJson, 'interactive', cardReplyTarget.turnId);
|
|
6478
8599
|
if (!ownsLifecycleMutation()) {
|
|
6479
8600
|
void deleteMessage(ds.larkAppId, postedCardId).catch(() => { });
|
|
6480
8601
|
break;
|
|
6481
8602
|
}
|
|
6482
8603
|
ds.streamCardId = postedCardId;
|
|
8604
|
+
ds.streamCardReplyTargetKey = cardReplyTarget.replyTargetKey;
|
|
6483
8605
|
// This card IS the current turn's live card — clear the new-turn flag
|
|
6484
8606
|
// so subsequent screen_updates PATCH it (starting → working) instead of
|
|
6485
8607
|
// POSTing a second card. Without this, a re-fork that happens while
|
|
@@ -6495,8 +8617,6 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6495
8617
|
}
|
|
6496
8618
|
ds.parkedStreamCardNonce = undefined;
|
|
6497
8619
|
persistStreamCardState(ds);
|
|
6498
|
-
// Re-sync worker's display mode (it starts fresh in 'hidden')
|
|
6499
|
-
syncWorkerDisplayMode(ds);
|
|
6500
8620
|
// New card is live — recall any cards frozen by previous turns.
|
|
6501
8621
|
// Done after `streamCardId` is committed so we never delete the old
|
|
6502
8622
|
// card without a successor visible to the user.
|
|
@@ -6591,6 +8711,10 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6591
8711
|
sendWorkerSessionInput(ds, {
|
|
6592
8712
|
type: 'raw_input',
|
|
6593
8713
|
content: rawInput,
|
|
8714
|
+
// Passthrough commands (/compact, /model, ...) become REAL mojo turns,
|
|
8715
|
+
// so they must carry the same credential snapshot a normal message
|
|
8716
|
+
// does — otherwise a cleared or rotated JWT simply did not apply here.
|
|
8717
|
+
...(mojoLivePatchForSession(ds) ?? {}),
|
|
6594
8718
|
...((ds.session.queuedActivationTurnId ?? rawTurnId)
|
|
6595
8719
|
? { turnId: ds.session.queuedActivationTurnId ?? rawTurnId }
|
|
6596
8720
|
: {}),
|
|
@@ -6728,6 +8852,7 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6728
8852
|
updateUsageLimitState(ds, msg.usageLimit);
|
|
6729
8853
|
ds.lastScreenContent = msg.content;
|
|
6730
8854
|
ds.lastScreenStatus = (msg.usageLimit ?? ds.usageLimit) ? 'limited' : msg.status;
|
|
8855
|
+
bumpStreamCardStatusRevision(ds);
|
|
6731
8856
|
// A suspend that arrived mid-turn parked itself here. Defer until this
|
|
6732
8857
|
// screen_update has finished using process state — suspendWorker nulls
|
|
6733
8858
|
// `worker` + `lastScreenStatus`, which everything below still reads
|
|
@@ -6843,13 +8968,15 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6843
8968
|
// not POSTed as duplicate cards.
|
|
6844
8969
|
ds.streamCardPending = false;
|
|
6845
8970
|
ds.streamCardId = CARD_POSTING_SENTINEL;
|
|
6846
|
-
|
|
8971
|
+
const cardReplyTarget = captureStreamingCardReplyTarget(ds, msg.turnId);
|
|
8972
|
+
scopedReply(cardJson, 'interactive', cardReplyTarget.turnId)
|
|
6847
8973
|
.then(msgId => {
|
|
6848
8974
|
if (!ownsLifecycleMutation()) {
|
|
6849
8975
|
void deleteMessage(ds.larkAppId, msgId).catch(() => { });
|
|
6850
8976
|
return;
|
|
6851
8977
|
}
|
|
6852
8978
|
ds.streamCardId = msgId;
|
|
8979
|
+
ds.streamCardReplyTargetKey = cardReplyTarget.replyTargetKey;
|
|
6853
8980
|
const superseded = (ds.streamCardTurnGeneration ?? 0) !== postingGeneration;
|
|
6854
8981
|
if (!superseded)
|
|
6855
8982
|
ds.streamCardPendingTurnId = undefined;
|
|
@@ -6917,10 +9044,10 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6917
9044
|
// reflect previous turn's content. Next 10s cycle picks up fresh content.
|
|
6918
9045
|
if (ds.streamCardPending)
|
|
6919
9046
|
break;
|
|
6920
|
-
ds.currentImageKey = msg.imageKey;
|
|
6921
9047
|
const prevStatus = ds.lastScreenStatus;
|
|
6922
9048
|
updateUsageLimitState(ds, msg.usageLimit);
|
|
6923
9049
|
ds.lastScreenStatus = (msg.usageLimit ?? ds.usageLimit) ? 'limited' : msg.status;
|
|
9050
|
+
bumpStreamCardStatusRevision(ds);
|
|
6924
9051
|
// Same deferred-suspend checkpoint as the screen_update branch, and
|
|
6925
9052
|
// deferred for the same reason (see runPendingSuspendIfSettled).
|
|
6926
9053
|
// The predicate is defense-in-depth here: the handler's fence already
|
|
@@ -6934,7 +9061,6 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6934
9061
|
imageKey: msg.imageKey,
|
|
6935
9062
|
content: ds.lastScreenContent ?? '',
|
|
6936
9063
|
});
|
|
6937
|
-
persistStreamCardState(ds);
|
|
6938
9064
|
// screenshot_uploaded never ARMS the usage refresh — screen_update owns
|
|
6939
9065
|
// the authorized arm. Here we only tear it down: unconditionally on
|
|
6940
9066
|
// leaving `working`, and on a managed/silent turn (below) that must not
|
|
@@ -6945,8 +9071,16 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
6945
9071
|
clearUsageRefreshTimer(ds);
|
|
6946
9072
|
if (managedAuxUiSuppressed(msg.turnId, msg.dispatchAttempt)) {
|
|
6947
9073
|
clearUsageRefreshTimer(ds);
|
|
9074
|
+
persistStreamCardState(ds);
|
|
6948
9075
|
break;
|
|
6949
9076
|
}
|
|
9077
|
+
// Store the image key only AFTER the managed gate: now that the ready
|
|
9078
|
+
// handler re-syncs display mode on every path, a worker can be
|
|
9079
|
+
// uploading during a suppressed managed/silent turn — letting that
|
|
9080
|
+
// frame into ds.currentImageKey would paste it onto the next visible
|
|
9081
|
+
// card render (the same leak class the comment above names).
|
|
9082
|
+
ds.currentImageKey = msg.imageKey;
|
|
9083
|
+
persistStreamCardState(ds);
|
|
6950
9084
|
if ((ds.displayMode ?? 'hidden') !== 'screenshot')
|
|
6951
9085
|
break;
|
|
6952
9086
|
if (!ds.streamCardId || ds.streamCardId === CARD_POSTING_SENTINEL || !workerHasInitialized(ds))
|
|
@@ -7296,20 +9430,21 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
7296
9430
|
}
|
|
7297
9431
|
break;
|
|
7298
9432
|
}
|
|
7299
|
-
//
|
|
7300
|
-
//
|
|
7301
|
-
//
|
|
7302
|
-
//
|
|
7303
|
-
//
|
|
7304
|
-
|
|
7305
|
-
|
|
7306
|
-
|
|
9433
|
+
// EVERY remote backend is a lineage-owning backend, not a local CLI
|
|
9434
|
+
// process. Stop before crash-loop accounting: for riff the restart IPC
|
|
9435
|
+
// is a guaranteed no-op, and for mojo it is WORSE than a no-op — the
|
|
9436
|
+
// worker executes it, cancelling the remote session and cold-booting a
|
|
9437
|
+
// context-less replacement, so treating a mojo backend exit as a local
|
|
9438
|
+
// CLI crash silently destroyed remote context (fourth-round review).
|
|
9439
|
+
if (isRemoteBackendSession(ds)) {
|
|
9440
|
+
const retirementPhase = remoteRetirementAdmissionPhase(ds);
|
|
9441
|
+
logger.warn(`[${t}] Remote backend exited; automatic restart is unsupported`
|
|
7307
9442
|
+ (retirementPhase ? ` (${retirementPhase})` : ''));
|
|
7308
9443
|
// Explicit /close and daemon shutdown already own the user-visible
|
|
7309
9444
|
// lifecycle. Only an unexpected backend exit needs recovery guidance.
|
|
7310
9445
|
if (!retirementPhase && !suppressExitUi) {
|
|
7311
9446
|
try {
|
|
7312
|
-
await scopedReply(tr('cmd.restart.
|
|
9447
|
+
await scopedReply(tr('cmd.restart.remote_unsupported', undefined, loc), 'text', undefined);
|
|
7313
9448
|
}
|
|
7314
9449
|
catch (replyErr) {
|
|
7315
9450
|
if (replyErr instanceof MessageWithdrawnError) {
|
|
@@ -7348,6 +9483,11 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
7348
9483
|
// terminal URL can show the startup failure; the next user message
|
|
7349
9484
|
// tells that same worker to destroy the diagnostic shell and retry.
|
|
7350
9485
|
ds.workerReady = false;
|
|
9486
|
+
// The worker now holds a parked diagnostic shell and will respawn the
|
|
9487
|
+
// CLI itself on the next message (no restart IPC in that path), so the
|
|
9488
|
+
// next message must carry the current launch model. Cleared once a
|
|
9489
|
+
// worker generation reports ready again.
|
|
9490
|
+
ds.crashDiagnosticParked = true;
|
|
7351
9491
|
ds.worker.send({ type: 'park_diagnostic' });
|
|
7352
9492
|
restartCounts.delete(key);
|
|
7353
9493
|
ds.lastScreenStatus = 'idle';
|
|
@@ -7396,7 +9536,7 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
7396
9536
|
if (ds.worker && !ds.worker.killed) {
|
|
7397
9537
|
logger.info(`[${t}] Auto-restarting ${sessionCliDisplayName(ds, botCfg)}...`);
|
|
7398
9538
|
ds.workerReady = false;
|
|
7399
|
-
ds.worker.send({ type: 'restart', reason: 'cli_crash', env: latestPerBotEnvForRestart(ds) });
|
|
9539
|
+
ds.worker.send({ type: 'restart', reason: 'cli_crash', env: latestPerBotEnvForRestart(ds), model: latestModelForRespawn(ds) });
|
|
7400
9540
|
}
|
|
7401
9541
|
break;
|
|
7402
9542
|
}
|
|
@@ -7465,19 +9605,28 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
7465
9605
|
break;
|
|
7466
9606
|
}
|
|
7467
9607
|
case 'close_result': {
|
|
7468
|
-
const pending =
|
|
9608
|
+
const pending = pendingRemoteWorkerCloses.get(msg.requestId);
|
|
7469
9609
|
if (!pending
|
|
7470
9610
|
|| pending.worker !== worker
|
|
7471
9611
|
|| pending.sessionId !== ds.session.sessionId
|
|
7472
9612
|
|| ds.worker !== worker) {
|
|
7473
|
-
logger.warn(`[${t}] Ignored stale/unmatched
|
|
9613
|
+
logger.warn(`[${t}] Ignored stale/unmatched remote close result ${msg.requestId}`);
|
|
7474
9614
|
break;
|
|
7475
9615
|
}
|
|
7476
|
-
|
|
9616
|
+
pendingRemoteWorkerCloses.delete(msg.requestId);
|
|
7477
9617
|
pending.resolve({
|
|
7478
9618
|
ok: msg.ok,
|
|
7479
9619
|
...(msg.taskId ? { taskId: msg.taskId } : {}),
|
|
7480
9620
|
...(msg.error ? { error: msg.error } : {}),
|
|
9621
|
+
...(msg.recovery ? { recovery: msg.recovery } : {}),
|
|
9622
|
+
// Forwarded, never re-derived: the backend may keep writes fenced on a
|
|
9623
|
+
// close that is otherwise retryable, and only it knows that.
|
|
9624
|
+
...(msg.admission ? { admission: msg.admission } : {}),
|
|
9625
|
+
// Same reason, and this one was the leak: an ok:true close carrying a
|
|
9626
|
+
// local residual arrived here, lost the field, and was published as an
|
|
9627
|
+
// ordinary closed row while the backend still held the containment
|
|
9628
|
+
// handle for a subtree it could not prove gone.
|
|
9629
|
+
...(msg.residual ? { residual: msg.residual } : {}),
|
|
7481
9630
|
});
|
|
7482
9631
|
break;
|
|
7483
9632
|
}
|
|
@@ -7599,6 +9748,11 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
7599
9748
|
&& ds.managedTurnOrigin.dispatchAttempt === msg.dispatchAttempt) {
|
|
7600
9749
|
ds.managedTurnOrigin = undefined;
|
|
7601
9750
|
}
|
|
9751
|
+
const isClaudeProviderFailure = msg.status !== 'completed'
|
|
9752
|
+
&& sessionCliId(ds, botCfg) === 'claude-code'
|
|
9753
|
+
&& (msg.errorCode?.startsWith('provider_') ?? false);
|
|
9754
|
+
const recoveryOwnsTerminal = ordinaryTurnRecoveryHandlesTerminal(ds.session, msg);
|
|
9755
|
+
let recoveryHandled = false;
|
|
7602
9756
|
try {
|
|
7603
9757
|
await cb.onTurnTerminal?.(ds, msg, { workerGeneration });
|
|
7604
9758
|
}
|
|
@@ -7607,6 +9761,98 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
7607
9761
|
// never let a projection/store failure crash the worker IPC loop.
|
|
7608
9762
|
logger.error(`[${t}] Failed to persist turn_terminal for ${msg.turnId.substring(0, 8)}: ${err.message}`);
|
|
7609
9763
|
}
|
|
9764
|
+
try {
|
|
9765
|
+
handleOrdinaryTurnRecoveryTerminal(ds.session, msg);
|
|
9766
|
+
recoveryHandled = recoveryOwnsTerminal;
|
|
9767
|
+
}
|
|
9768
|
+
catch (err) {
|
|
9769
|
+
// Recovery projection is durable fail-closed state, but a temporary
|
|
9770
|
+
// session-store failure must not escape the worker IPC handler. The
|
|
9771
|
+
// prior in-memory/persisted state was restored by the coordinator and
|
|
9772
|
+
// can be reconciled on the next terminal or daemon restart.
|
|
9773
|
+
logger.error(`[${t}] Failed to persist ordinary-turn recovery terminal for `
|
|
9774
|
+
+ `${msg.turnId.substring(0, 8)}: `
|
|
9775
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
9776
|
+
}
|
|
9777
|
+
let nonLarkFailureHandled = false;
|
|
9778
|
+
if (isClaudeProviderFailure && !ds.session.vcMeetingReceiver) {
|
|
9779
|
+
const failureCode = msg.errorCode ?? msg.status;
|
|
9780
|
+
const waitPromise = ds.pendingWaitPromises?.get(msg.turnId);
|
|
9781
|
+
if (waitPromise) {
|
|
9782
|
+
nonLarkFailureHandled = true;
|
|
9783
|
+
ds.pendingWaitPromises?.delete(msg.turnId);
|
|
9784
|
+
const failure = new Error(`Claude turn failed: ${failureCode}`);
|
|
9785
|
+
if (waitPromise.reject)
|
|
9786
|
+
waitPromise.reject(failure);
|
|
9787
|
+
else
|
|
9788
|
+
waitPromise.resolve(`ERROR: ${failure.message}`);
|
|
9789
|
+
logger.info(`[${t}] Settled Wait Mode HTTP turn ${msg.turnId.substring(0, 8)} `
|
|
9790
|
+
+ `failed (${failureCode})`);
|
|
9791
|
+
}
|
|
9792
|
+
const asyncResult = ds.asyncTriggerResults?.get(msg.turnId);
|
|
9793
|
+
const asyncSink = !!asyncResult || ds.chatId.startsWith('http_async_');
|
|
9794
|
+
if (asyncSink) {
|
|
9795
|
+
nonLarkFailureHandled = true;
|
|
9796
|
+
const failedAt = Date.now();
|
|
9797
|
+
if (asyncResult?.status === 'completed') {
|
|
9798
|
+
// final_output is stronger and may have settled immediately before
|
|
9799
|
+
// the ordered terminal IPC. Never replace known output with a
|
|
9800
|
+
// later failure boundary, even if its best-effort durable write
|
|
9801
|
+
// has not landed yet.
|
|
9802
|
+
ds.idempotentAsyncTurns?.delete(msg.turnId);
|
|
9803
|
+
logger.info(`[${t}] Ignored failed terminal for completed async HTTP turn `
|
|
9804
|
+
+ msg.turnId.substring(0, 8));
|
|
9805
|
+
}
|
|
9806
|
+
else {
|
|
9807
|
+
if (asyncResult?.status === 'pending') {
|
|
9808
|
+
asyncResult.status = 'failed';
|
|
9809
|
+
asyncResult.failedAt = failedAt;
|
|
9810
|
+
asyncResult.errorCode = 'trigger_failed';
|
|
9811
|
+
asyncResult.terminalErrorCode = failureCode;
|
|
9812
|
+
}
|
|
9813
|
+
try {
|
|
9814
|
+
const outcome = asyncTriggerStore.recordTerminalFailureStrict(ds.session.sessionId, msg.turnId, failedAt, ds.larkAppId, failureCode);
|
|
9815
|
+
if (outcome === 'already_completed') {
|
|
9816
|
+
// Durable completed proof is stronger. Drop a stale in-memory
|
|
9817
|
+
// pending/failed projection so trigger-result reads that proof.
|
|
9818
|
+
ds.asyncTriggerResults?.delete(msg.turnId);
|
|
9819
|
+
}
|
|
9820
|
+
}
|
|
9821
|
+
catch (err) {
|
|
9822
|
+
// The live failed projection still terminates current-daemon
|
|
9823
|
+
// polling. Keep it when durable persistence is unavailable and
|
|
9824
|
+
// report the lost restart guarantee explicitly.
|
|
9825
|
+
logger.error(`[${t}] Failed to persist async worker terminal for `
|
|
9826
|
+
+ `${msg.turnId.substring(0, 8)}: `
|
|
9827
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
9828
|
+
}
|
|
9829
|
+
ds.idempotentAsyncTurns?.delete(msg.turnId);
|
|
9830
|
+
logger.info(`[${t}] Settled async HTTP turn ${msg.turnId.substring(0, 8)} `
|
|
9831
|
+
+ `failed (${failureCode})`);
|
|
9832
|
+
}
|
|
9833
|
+
}
|
|
9834
|
+
}
|
|
9835
|
+
if (isClaudeProviderFailure
|
|
9836
|
+
&& !nonLarkFailureHandled
|
|
9837
|
+
&& !recoveryHandled
|
|
9838
|
+
&& !managedAuxUiSuppressed(msg.turnId, msg.dispatchAttempt)) {
|
|
9839
|
+
const failureCode = msg.errorCode ?? msg.status;
|
|
9840
|
+
const warning = tr('worker.claude_terminal_failure_unrecovered', { errorCode: failureCode }, loc);
|
|
9841
|
+
ds.agentAttention = { kind: 'blocked', reason: warning, at: Date.now() };
|
|
9842
|
+
publishAttentionPatch(ds);
|
|
9843
|
+
emitSessionLifecycleHook(ds, 'session.requires_attention', {
|
|
9844
|
+
reason: 'claude_terminal_failure_unrecovered',
|
|
9845
|
+
errorCode: failureCode,
|
|
9846
|
+
turnId: msg.turnId,
|
|
9847
|
+
});
|
|
9848
|
+
try {
|
|
9849
|
+
await scopedReply(warning, 'text', msg.turnId);
|
|
9850
|
+
}
|
|
9851
|
+
catch (err) {
|
|
9852
|
+
logger.error(`[${t}] Failed to deliver Claude terminal failure warning: `
|
|
9853
|
+
+ `${err instanceof Error ? err.message : String(err)}`);
|
|
9854
|
+
}
|
|
9855
|
+
}
|
|
7610
9856
|
// Async-HTTP settle-on-terminal (core-only completion bug #70): a turn
|
|
7611
9857
|
// the worker's bridge gate suppressed as GENUINE SILENCE (the model
|
|
7612
9858
|
// terminated with a bare nothing-to-send sentinel, no `botmux send`)
|
|
@@ -8141,10 +10387,10 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
8141
10387
|
ds.workerToken = null;
|
|
8142
10388
|
ds.workerViewToken = null;
|
|
8143
10389
|
ds.managedTurnOrigin = undefined;
|
|
8144
|
-
if (ds.
|
|
8145
|
-
ds.
|
|
10390
|
+
if (ds.remoteCloseState) {
|
|
10391
|
+
ds.remoteCloseState = { ...ds.remoteCloseState, phase: 'uncertain' };
|
|
8146
10392
|
}
|
|
8147
|
-
// Do not clear
|
|
10393
|
+
// Do not clear remoteShutdownState here. Only the shutdown coordinator can
|
|
8148
10394
|
// release a generation after lineage persistence or admission restore.
|
|
8149
10395
|
// This worker generation is gone. Invalidate any stuck-warning card it
|
|
8150
10396
|
// posted so a late click cannot inject keys into a replacement worker.
|
|
@@ -8159,7 +10405,12 @@ function setupWorkerHandlers(ds, worker, startupState = { ready: false, failureN
|
|
|
8159
10405
|
ds.workerGeneration = fencedGeneration;
|
|
8160
10406
|
ds.session.workerGeneration = fencedGeneration;
|
|
8161
10407
|
ds.session.pid = undefined;
|
|
10408
|
+
// P1-13:worker 退出(崩溃、被杀、正常结束)同样是权威换代——它带走整棵 CLI
|
|
10409
|
+
// 进程树,包括那一代注册的 dev server。与代次围栏并进同一次落盘。
|
|
10410
|
+
const exitedPreviewTarget = takeSessionPreviewTarget(ds.session);
|
|
8162
10411
|
sessionStore.updateSession(ds.session);
|
|
10412
|
+
if (exitedPreviewTarget !== undefined)
|
|
10413
|
+
publishSessionPreviewCleared(ds.session.sessionId);
|
|
8163
10414
|
}
|
|
8164
10415
|
if (!transferRetirement) {
|
|
8165
10416
|
try {
|
|
@@ -8224,6 +10475,26 @@ export function codexAppFinalSettlementCount() {
|
|
|
8224
10475
|
function finalOutputDedupeKey(ds, msg) {
|
|
8225
10476
|
return `${msg.sessionId ?? ds.session.sessionId}:${msg.lastUuid || msg.turnId}`;
|
|
8226
10477
|
}
|
|
10478
|
+
/**
|
|
10479
|
+
* Stable Lark `uuid` idempotency token for an ordinary bridge final_output
|
|
10480
|
+
* delivery. Derived from the same identity as the daemon-side dedupe key, so
|
|
10481
|
+
* every retry of the same answer — and any duplicate IPC/re-emit of the same
|
|
10482
|
+
* turn — collapses into ONE Lark message (the Feishu `uuid` field is
|
|
10483
|
+
* idempotent for 1h and returns the original message_id on repeat).
|
|
10484
|
+
*
|
|
10485
|
+
* Without it, the daemon's own transient-failure retry
|
|
10486
|
+
* (`FINAL_OUTPUT_RETRY_BACKOFF_MS`, up to 3 attempts) is at-least-once: an
|
|
10487
|
+
* ambiguous first attempt whose reply the server accepted but whose response
|
|
10488
|
+
* the client never saw (network error/timeout) creates a brand-new copy on
|
|
10489
|
+
* each retry — the user-visible "the answer repeated 3 times" incident. The
|
|
10490
|
+
* VC-managed (`vcp_*`) and Codex App settlement (`ca_*`) paths already carry
|
|
10491
|
+
* their own stable provider keys; this closes the same gap for the ordinary
|
|
10492
|
+
* thread/chat fallback path.
|
|
10493
|
+
*/
|
|
10494
|
+
function bridgeFinalOutputUuid(ds, msg) {
|
|
10495
|
+
const seed = finalOutputDedupeKey(ds, msg);
|
|
10496
|
+
return `bf_${createHash('sha256').update(seed, 'utf8').digest('hex').slice(0, 46)}`;
|
|
10497
|
+
}
|
|
8227
10498
|
function shouldDropMismatchedFinalOutput(ds, msg, t) {
|
|
8228
10499
|
if (!msg.sessionId || msg.sessionId === ds.session.sessionId)
|
|
8229
10500
|
return false;
|
|
@@ -8688,7 +10959,7 @@ function deliverFinalOutput(ds, msg, t, attempt, onComplete, isStillOwned = () =
|
|
|
8688
10959
|
}
|
|
8689
10960
|
: {}),
|
|
8690
10961
|
}
|
|
8691
|
-
: codexAppSettlementReply;
|
|
10962
|
+
: codexAppSettlementReply ?? { uuid: bridgeFinalOutputUuid(ds, msg) };
|
|
8692
10963
|
const messageId = await scopedReply(canonicalOutput.content, canonicalOutput.msgType, msg.replyTurnId ?? msg.turnId, frozenReplyTarget && !managedReceiver
|
|
8693
10964
|
? { ...deliveryReplyOptions, replyTarget: frozenReplyTarget }
|
|
8694
10965
|
: deliveryReplyOptions);
|
|
@@ -8765,6 +11036,11 @@ function reserveWorkerGeneration(ds) {
|
|
|
8765
11036
|
const workerGeneration = Math.max(previousDaemonGeneration ?? 0, previousSessionGeneration ?? 0) + 1;
|
|
8766
11037
|
ds.workerGeneration = workerGeneration;
|
|
8767
11038
|
ds.session.workerGeneration = workerGeneration;
|
|
11039
|
+
// P1-13:换代边界统一清 previewTarget。fork / refork / 切 CLI / adopt 全部经由这里
|
|
11040
|
+
// 预留代次,所以这一处就是「上一代 CLI 起的 dev server 已经失去权威」的唯一判据。
|
|
11041
|
+
// 并进同一次 updateSession,代次推进与预览清空要么一起落盘、要么一起回滚——否则
|
|
11042
|
+
// 会出现「代次已经是新的、磁盘上还留着上一代端口」的中间态,重启后被当成有效路由。
|
|
11043
|
+
const previousPreviewTarget = takeSessionPreviewTarget(ds.session);
|
|
8768
11044
|
try {
|
|
8769
11045
|
sessionStore.updateSession(ds.session);
|
|
8770
11046
|
}
|
|
@@ -8777,8 +11053,13 @@ function reserveWorkerGeneration(ds) {
|
|
|
8777
11053
|
delete ds.session.workerGeneration;
|
|
8778
11054
|
else
|
|
8779
11055
|
ds.session.workerGeneration = previousSessionGeneration;
|
|
11056
|
+
// 只在原本有目标时写回:预留失败不该给一个从未注册过预览的会话凭空加上这个键。
|
|
11057
|
+
if (previousPreviewTarget !== undefined)
|
|
11058
|
+
ds.session.previewTarget = previousPreviewTarget;
|
|
8780
11059
|
throw error;
|
|
8781
11060
|
}
|
|
11061
|
+
if (previousPreviewTarget !== undefined)
|
|
11062
|
+
publishSessionPreviewCleared(ds.session.sessionId);
|
|
8782
11063
|
// Reservation is the first durable proof that the previous generation has
|
|
8783
11064
|
// lost authority. Clear its TUI slot before any environment check, adapter
|
|
8784
11065
|
// creation, or fork can throw and strand a clicked card in "processing".
|
|
@@ -9070,6 +11351,10 @@ export function forkAdoptWorker(ds, opts) {
|
|
|
9070
11351
|
};
|
|
9071
11352
|
worker.send(initMsg);
|
|
9072
11353
|
ds.initConfig = initMsg;
|
|
11354
|
+
// New generation ledger. NOT a plain reset: the previous generation's keys are
|
|
11355
|
+
// parked until its worker is observed to exit, because the double-fork guard
|
|
11356
|
+
// kills asynchronously and spawns the replacement synchronously.
|
|
11357
|
+
startNewGenerationEnvLedger(ds, initMsg.env);
|
|
9073
11358
|
// Stamp cliId on the persisted session so the dashboard can show a CLI badge
|
|
9074
11359
|
// even after the session is closed. Adopt sessions inherit the adopted CLI's id.
|
|
9075
11360
|
// Do this before installing worker handlers: a fast worker can emit `ready`
|