@adhdev/daemon-core 0.9.82-rc.45 → 0.9.82-rc.450
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/daemon-lifecycle.d.ts +9 -0
- package/dist/boot/process-hardening.d.ts +50 -0
- package/dist/build-info.d.ts +37 -0
- package/dist/chat/source-machine.d.ts +166 -0
- package/dist/chat/source-resolver.d.ts +104 -0
- package/dist/chat/subscription-updates.d.ts +1 -0
- package/dist/cli-adapter-types.d.ts +68 -2
- package/dist/cli-adapters/cli-script-runner.d.ts +117 -0
- package/dist/cli-adapters/cli-state-engine.d.ts +200 -0
- package/dist/cli-adapters/provider-cli-adapter.d.ts +126 -64
- package/dist/cli-adapters/provider-cli-parse.d.ts +5 -0
- package/dist/cli-adapters/provider-cli-runtime.d.ts +34 -0
- package/dist/cli-adapters/provider-cli-shared.d.ts +74 -0
- package/dist/cli-adapters/pty-write-chunking.d.ts +34 -0
- package/dist/cli-adapters/raw-terminal-io.d.ts +37 -0
- package/dist/cli-adapters/resolve-executable.d.ts +69 -0
- package/dist/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +6 -3
- package/dist/cli-adapters/terminal-backends/types.d.ts +10 -2
- package/dist/cli-adapters/terminal-screen.d.ts +4 -11
- package/dist/commands/chat-commands-debug-bundle.d.ts +14 -0
- package/dist/commands/chat-commands-read.d.ts +7 -0
- package/dist/commands/chat-commands-scope.d.ts +39 -0
- package/dist/commands/chat-commands-shared.d.ts +33 -0
- package/dist/commands/chat-commands-write.d.ts +14 -0
- package/dist/commands/chat-commands.d.ts +10 -23
- package/dist/commands/cli-manager.d.ts +25 -1
- package/dist/commands/handler.d.ts +125 -0
- package/dist/commands/high-family/index.d.ts +3 -0
- package/dist/commands/high-family/mesh-coordinator-launch.d.ts +2 -0
- package/dist/commands/high-family/mesh-events.d.ts +2 -0
- package/dist/commands/high-family/mesh-status.d.ts +2 -0
- package/dist/commands/high-family/types.d.ts +64 -0
- package/dist/commands/low-family/coordinator-prompt.d.ts +9 -0
- package/dist/commands/low-family/daemon-lifecycle.d.ts +2 -0
- package/dist/commands/low-family/diagnostics.d.ts +2 -0
- package/dist/commands/low-family/index.d.ts +3 -0
- package/dist/commands/low-family/mesh-ledger.d.ts +10 -0
- package/dist/commands/low-family/mesh-node-logs.d.ts +2 -0
- package/dist/commands/low-family/notification.d.ts +2 -0
- package/dist/commands/low-family/refine-config.d.ts +2 -0
- package/dist/commands/low-family/session-host.d.ts +2 -0
- package/dist/commands/low-family/spec-providerdev.d.ts +11 -0
- package/dist/commands/low-family/status-meta.d.ts +2 -0
- package/dist/commands/low-family/types.d.ts +33 -0
- package/dist/commands/med-family/cli-agent.d.ts +2 -0
- package/dist/commands/med-family/fast-forward.d.ts +2 -0
- package/dist/commands/med-family/ide.d.ts +10 -0
- package/dist/commands/med-family/index.d.ts +3 -0
- package/dist/commands/med-family/mesh-crud.d.ts +29 -0
- package/dist/commands/med-family/mesh-host-pairing.d.ts +2 -0
- package/dist/commands/med-family/mesh-queue.d.ts +2 -0
- package/dist/commands/med-family/mesh-restart.d.ts +2 -0
- package/dist/commands/med-family/types.d.ts +152 -0
- package/dist/commands/mesh-coordinator.d.ts +85 -1
- package/dist/commands/router.d.ts +279 -2
- package/dist/commands/upgrade-helper.d.ts +41 -1
- package/dist/config/chat-history.d.ts +9 -0
- package/dist/config/config.d.ts +5 -0
- package/dist/config/mesh-config.d.ts +129 -1
- package/dist/config/mesh-json-config.d.ts +199 -0
- package/dist/config/repo-settings.d.ts +77 -0
- package/dist/daemon/dev-server.d.ts +0 -2
- package/dist/detection/ide-detector.d.ts +13 -0
- package/dist/detection/win32-ide-version.d.ts +37 -0
- package/dist/git/change-impact-config.d.ts +159 -0
- package/dist/git/git-commands.d.ts +12 -1
- package/dist/git/git-diff.d.ts +6 -0
- package/dist/git/git-executor.d.ts +11 -0
- package/dist/git/git-status.d.ts +57 -0
- package/dist/git/git-types.d.ts +2 -50
- package/dist/git/git-worktree.d.ts +71 -1
- package/dist/git/index.d.ts +3 -1
- package/dist/index.d.ts +66 -14
- package/dist/index.js +48475 -17084
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +49521 -18300
- package/dist/index.mjs.map +1 -1
- package/dist/installer.d.ts +1 -4
- package/dist/ipc/local-ipc-server.d.ts +91 -0
- package/dist/launch.d.ts +1 -1
- package/dist/logging/async-batch-writer.d.ts +10 -0
- package/dist/logging/log-redactor.d.ts +24 -0
- package/dist/logging/log-tail-reader.d.ts +81 -0
- package/dist/logging/logger.d.ts +1 -1
- package/dist/mesh/contracts.d.ts +164 -0
- package/dist/mesh/coordinator-prompt.d.ts +89 -0
- package/dist/mesh/coordinator-registry.d.ts +59 -0
- package/dist/mesh/mesh-active-work.d.ts +174 -0
- package/dist/mesh/mesh-clone-grace.d.ts +8 -0
- package/dist/mesh/mesh-coordinator-config.d.ts +21 -0
- package/dist/mesh/mesh-delivery-policy.d.ts +131 -0
- package/dist/mesh/mesh-event-classify.d.ts +5 -0
- package/dist/mesh/mesh-event-forwarding.d.ts +41 -0
- package/dist/mesh/mesh-event-trace.d.ts +21 -0
- package/dist/mesh/mesh-events-coordinator.d.ts +4 -0
- package/dist/mesh/mesh-events-pending.d.ts +69 -0
- package/dist/mesh/mesh-events-stale.d.ts +61 -0
- package/dist/mesh/mesh-events-utils.d.ts +75 -0
- package/dist/mesh/mesh-events.d.ts +6 -49
- package/dist/mesh/mesh-fast-forward.d.ts +81 -0
- package/dist/mesh/mesh-host-ownership.d.ts +29 -0
- package/dist/mesh/mesh-init.d.ts +128 -0
- package/dist/mesh/mesh-ledger-reconciliation.d.ts +23 -4
- package/dist/mesh/mesh-ledger.d.ts +115 -1
- package/dist/mesh/mesh-magi-status.d.ts +63 -0
- package/dist/mesh/mesh-missions.d.ts +189 -0
- package/dist/mesh/mesh-node-identity.d.ts +306 -0
- package/dist/mesh/mesh-queue-assignment.d.ts +113 -0
- package/dist/mesh/mesh-reconcile-loop.d.ts +79 -0
- package/dist/mesh/mesh-refine-batch.d.ts +68 -0
- package/dist/mesh/mesh-refine-gates.d.ts +428 -0
- package/dist/mesh/mesh-refine-status.d.ts +62 -0
- package/dist/mesh/mesh-review-inbox.d.ts +90 -0
- package/dist/mesh/mesh-routing.d.ts +70 -0
- package/dist/mesh/mesh-runtime-store.d.ts +450 -0
- package/dist/mesh/mesh-scheduling-runtime.d.ts +78 -0
- package/dist/mesh/mesh-task-inflight.d.ts +46 -0
- package/dist/mesh/mesh-task-stats.d.ts +49 -0
- package/dist/mesh/mesh-unresolved-forward-outbox.d.ts +30 -0
- package/dist/mesh/mesh-warmup-deadline.d.ts +68 -0
- package/dist/mesh/mesh-work-queue.d.ts +300 -5
- package/dist/mesh/preview-freshness.d.ts +18 -0
- package/dist/mesh/refine-config.d.ts +216 -0
- package/dist/mesh/worktree-bootstrap-config.d.ts +145 -0
- package/dist/providers/acp-provider-instance.d.ts +5 -0
- package/dist/providers/approval-utils.d.ts +20 -0
- package/dist/providers/chat-message-normalization.d.ts +32 -1
- package/dist/providers/cli-provider-instance.d.ts +254 -3
- package/dist/providers/contracts.d.ts +149 -6
- package/dist/providers/external-sources.d.ts +71 -0
- package/dist/providers/manual-attendance.d.ts +63 -0
- package/dist/providers/native-history/antigravity-cli-transcript.d.ts +126 -0
- package/dist/providers/native-history/claude-cli-transcript.d.ts +70 -0
- package/dist/providers/native-history/codex-cli-transcript.d.ts +73 -0
- package/dist/providers/native-history/constants.d.ts +12 -0
- package/dist/providers/native-history/dispatcher.d.ts +27 -0
- package/dist/providers/native-history/hermes-cli-transcript.d.ts +30 -0
- package/dist/providers/native-history/index.d.ts +13 -0
- package/dist/providers/provider-instance-manager.d.ts +29 -0
- package/dist/providers/provider-instance.d.ts +23 -1
- package/dist/providers/provider-loader.d.ts +26 -4
- package/dist/providers/provider-trust.d.ts +31 -0
- package/dist/providers/read-chat-contract.d.ts +29 -0
- package/dist/providers/sdk/v1/builders/acp/detect-status.d.ts +68 -0
- package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +86 -0
- package/dist/providers/sdk/v1/builders/cli/parse-approval-squash.d.ts +59 -0
- package/dist/providers/sdk/v1/builders/cli/parse-approval.d.ts +65 -0
- package/dist/providers/sdk/v1/builders/cli/parse-session.d.ts +91 -0
- package/dist/providers/sdk/v1/builders/cli/visible-region.d.ts +42 -0
- package/dist/providers/sdk/v1/fixture-tooling/format.d.ts +126 -0
- package/dist/providers/sdk/v1/fixture-tooling/index.d.ts +8 -0
- package/dist/providers/sdk/v1/fixture-tooling/replay.d.ts +38 -0
- package/dist/providers/sdk/v1/index.d.ts +30 -0
- package/dist/providers/sdk/v1/sandbox/README-design.d.ts +193 -0
- package/dist/providers/sdk/v1/sandbox/require-whitelist.d.ts +74 -0
- package/dist/providers/sdk/v1/sandbox/script-runner.d.ts +98 -0
- package/dist/providers/sdk/v1/types/cli/index.d.ts +277 -0
- package/dist/providers/sdk/v1/types/common/index.d.ts +203 -0
- package/dist/providers/sdk/v1/validators/index.d.ts +5 -0
- package/dist/providers/sdk/v1/validators/manifest.d.ts +40 -0
- package/dist/providers/sdk/v1/validators/taint.d.ts +52 -0
- package/dist/providers/spec/adapter.d.ts +98 -0
- package/dist/providers/spec/cli-adapter.d.ts +223 -0
- package/dist/providers/spec/evaluator.d.ts +45 -0
- package/dist/providers/spec/fsm-driver.d.ts +444 -0
- package/dist/providers/spec/fsm-evaluator.d.ts +93 -0
- package/dist/providers/spec/fsm-loader.d.ts +14 -0
- package/dist/providers/spec/fsm-types.d.ts +202 -0
- package/dist/providers/spec/native-history-executor.d.ts +41 -0
- package/dist/providers/spec/pre-launch-trust.d.ts +16 -0
- package/dist/providers/spec/route.d.ts +4 -0
- package/dist/providers/spec/types.d.ts +222 -0
- package/dist/providers/status-monitor.d.ts +7 -7
- package/dist/providers/transcript-v2.d.ts +176 -0
- package/dist/providers/types/interactive-prompt.d.ts +84 -0
- package/dist/providers/working-dir.d.ts +17 -0
- package/dist/repo-mesh-types.d.ts +519 -13
- package/dist/runtime-defaults.d.ts +2 -0
- package/dist/session-host/managed-host.d.ts +64 -0
- package/dist/sessions/registry.d.ts +6 -0
- package/dist/shared-types-extra.d.ts +2 -4
- package/dist/shared-types.d.ts +59 -2
- package/dist/status/normalize.d.ts +1 -1
- package/dist/status/normalize.js +1 -0
- package/dist/status/normalize.js.map +1 -1
- package/dist/status/normalize.mjs +1 -0
- package/dist/status/normalize.mjs.map +1 -1
- package/dist/status/reporter.d.ts +2 -0
- package/dist/status/snapshot.d.ts +26 -0
- package/dist/system/hash.d.ts +8 -0
- package/dist/system/load-better-sqlite3.d.ts +21 -0
- package/dist/types.d.ts +5 -0
- package/package.json +7 -3
- package/src/agent-stream/poller.ts +2 -3
- package/src/agent-stream/provider-adapter.ts +1 -1
- package/src/boot/daemon-lifecycle.ts +63 -12
- package/src/boot/process-hardening.ts +89 -0
- package/src/build-info.ts +73 -0
- package/src/chat/source-machine.ts +534 -0
- package/src/chat/source-resolver.ts +0 -0
- package/src/chat/subscription-updates.ts +20 -1
- package/src/cli-adapter-types.d.ts +3 -1
- package/src/cli-adapter-types.ts +68 -2
- package/src/cli-adapters/cli-script-runner.ts +421 -0
- package/src/cli-adapters/cli-state-engine.ts +1246 -0
- package/src/cli-adapters/provider-cli-adapter.d.ts +1 -1
- package/src/cli-adapters/provider-cli-adapter.ts +914 -1144
- package/src/cli-adapters/provider-cli-parse.d.ts +2 -0
- package/src/cli-adapters/provider-cli-parse.ts +19 -0
- package/src/cli-adapters/provider-cli-runtime.ts +56 -7
- package/src/cli-adapters/provider-cli-shared.d.ts +4 -0
- package/src/cli-adapters/provider-cli-shared.ts +121 -11
- package/src/cli-adapters/pty-transport.ts +2 -1
- package/src/cli-adapters/pty-write-chunking.ts +106 -0
- package/src/cli-adapters/raw-terminal-io.ts +252 -0
- package/src/cli-adapters/resolve-executable.ts +204 -0
- package/src/cli-adapters/session-host-transport.ts +2 -1
- package/src/cli-adapters/terminal-backends/ghostty-vt-backend.ts +63 -29
- package/src/cli-adapters/terminal-backends/types.ts +10 -3
- package/src/cli-adapters/terminal-screen.ts +16 -81
- package/src/commands/WINDOWS-UPGRADE-LOCK-FAILURE.md +198 -0
- package/src/commands/chat-commands-debug-bundle.ts +398 -0
- package/src/commands/chat-commands-read.ts +2488 -0
- package/src/commands/chat-commands-scope.ts +54 -0
- package/src/commands/chat-commands-shared.ts +114 -0
- package/src/commands/chat-commands-write.ts +891 -0
- package/src/commands/chat-commands.ts +19 -1841
- package/src/commands/cli-manager.ts +596 -27
- package/src/commands/handler.ts +841 -2
- package/src/commands/high-family/index.ts +28 -0
- package/src/commands/high-family/mesh-coordinator-launch.ts +680 -0
- package/src/commands/high-family/mesh-events.ts +89 -0
- package/src/commands/high-family/mesh-status.ts +822 -0
- package/src/commands/high-family/types.ts +80 -0
- package/src/commands/low-family/coordinator-prompt.ts +72 -0
- package/src/commands/low-family/daemon-lifecycle.ts +107 -0
- package/src/commands/low-family/diagnostics.ts +57 -0
- package/src/commands/low-family/index.ts +37 -0
- package/src/commands/low-family/mesh-ledger.ts +62 -0
- package/src/commands/low-family/mesh-node-logs.ts +87 -0
- package/src/commands/low-family/notification.ts +116 -0
- package/src/commands/low-family/refine-config.ts +106 -0
- package/src/commands/low-family/session-host.ts +274 -0
- package/src/commands/low-family/spec-providerdev.ts +217 -0
- package/src/commands/low-family/status-meta.ts +112 -0
- package/src/commands/low-family/types.ts +39 -0
- package/src/commands/med-family/cli-agent.ts +238 -0
- package/src/commands/med-family/fast-forward.ts +230 -0
- package/src/commands/med-family/ide.ts +163 -0
- package/src/commands/med-family/index.ts +37 -0
- package/src/commands/med-family/mesh-crud.ts +1253 -0
- package/src/commands/med-family/mesh-host-pairing.ts +234 -0
- package/src/commands/med-family/mesh-queue.ts +167 -0
- package/src/commands/med-family/mesh-restart.ts +92 -0
- package/src/commands/med-family/types.ts +153 -0
- package/src/commands/mesh-coordinator.ts +334 -124
- package/src/commands/router.ts +2893 -3536
- package/src/commands/stream-commands.ts +8 -0
- package/src/commands/upgrade-helper.ts +310 -45
- package/src/config/chat-history.ts +493 -24
- package/src/config/config.ts +12 -0
- package/src/config/mesh-config.ts +619 -23
- package/src/config/mesh-json-config.ts +376 -0
- package/src/config/recent-activity.ts +8 -2
- package/src/config/repo-settings.ts +111 -0
- package/src/daemon/dev-auto-implement.ts +3 -2
- package/src/daemon/dev-cli-debug.ts +10 -1
- package/src/daemon/dev-server.ts +0 -541
- package/src/detection/cli-detector.ts +28 -9
- package/src/detection/ide-detector.ts +55 -16
- package/src/detection/win32-ide-version.ts +106 -0
- package/src/git/change-impact-config.ts +354 -0
- package/src/git/git-commands.ts +78 -15
- package/src/git/git-diff.ts +81 -11
- package/src/git/git-executor.ts +12 -0
- package/src/git/git-status.ts +767 -48
- package/src/git/git-types.ts +14 -62
- package/src/git/git-worktree.ts +261 -4
- package/src/git/index.ts +17 -0
- package/src/index.ts +198 -12
- package/src/installer.d.ts +1 -1
- package/src/installer.ts +8 -6
- package/src/ipc/local-ipc-server.ts +278 -0
- package/src/launch.d.ts +1 -1
- package/src/launch.ts +37 -28
- package/src/logging/async-batch-writer.ts +55 -0
- package/src/logging/command-log.ts +7 -5
- package/src/logging/log-redactor.ts +100 -0
- package/src/logging/log-tail-reader.ts +341 -0
- package/src/logging/logger.ts +14 -7
- package/src/mesh/contracts.ts +338 -0
- package/src/mesh/coordinator-prompt.ts +416 -34
- package/src/mesh/coordinator-registry.ts +121 -0
- package/src/mesh/mesh-active-work.ts +645 -0
- package/src/mesh/mesh-clone-grace.ts +68 -0
- package/src/mesh/mesh-coordinator-config.ts +97 -0
- package/src/mesh/mesh-delivery-policy.ts +315 -0
- package/src/mesh/mesh-event-classify.ts +51 -0
- package/src/mesh/mesh-event-forwarding.ts +1897 -0
- package/src/mesh/mesh-event-trace.ts +67 -0
- package/src/mesh/mesh-events-coordinator.ts +32 -0
- package/src/mesh/mesh-events-pending.ts +656 -0
- package/src/mesh/mesh-events-stale.ts +389 -0
- package/src/mesh/mesh-events-utils.ts +443 -0
- package/src/mesh/mesh-events.ts +33 -1035
- package/src/mesh/mesh-fast-forward.ts +843 -0
- package/src/mesh/mesh-host-ownership.ts +111 -0
- package/src/mesh/mesh-init.ts +350 -0
- package/src/mesh/mesh-ledger-reconciliation.ts +12 -7
- package/src/mesh/mesh-ledger.ts +841 -102
- package/src/mesh/mesh-magi-status.ts +223 -0
- package/src/mesh/mesh-missions.ts +449 -0
- package/src/mesh/mesh-node-identity.ts +1985 -0
- package/src/mesh/mesh-queue-assignment.ts +1924 -0
- package/src/mesh/mesh-reconcile-loop.ts +2018 -0
- package/src/mesh/mesh-refine-batch.ts +205 -0
- package/src/mesh/mesh-refine-gates.ts +1673 -0
- package/src/mesh/mesh-refine-status.ts +231 -0
- package/src/mesh/mesh-review-inbox.ts +307 -0
- package/src/mesh/mesh-routing.ts +291 -0
- package/src/mesh/mesh-runtime-store.ts +1853 -0
- package/src/mesh/mesh-scheduling-runtime.ts +198 -0
- package/src/mesh/mesh-task-inflight.ts +70 -0
- package/src/mesh/mesh-task-stats.ts +161 -0
- package/src/mesh/mesh-unresolved-forward-outbox.ts +185 -0
- package/src/mesh/mesh-warmup-deadline.ts +152 -0
- package/src/mesh/mesh-work-queue.ts +1254 -141
- package/src/mesh/preview-freshness.ts +118 -0
- package/src/mesh/refine-config.ts +423 -0
- package/src/mesh/worktree-bootstrap-config.ts +450 -0
- package/src/providers/acp-provider-instance.ts +43 -10
- package/src/providers/approval-utils.d.ts +5 -0
- package/src/providers/approval-utils.ts +57 -5
- package/src/providers/chat-message-normalization.ts +92 -4
- package/src/providers/cli-provider-instance.ts +2177 -107
- package/src/providers/contracts.d.ts +55 -0
- package/src/providers/contracts.ts +160 -6
- package/src/providers/extension-provider-instance.ts +12 -7
- package/src/providers/external-sources.ts +218 -0
- package/src/providers/ide-provider-instance.ts +35 -12
- package/src/providers/manual-attendance.ts +85 -0
- package/src/providers/native-history/antigravity-cli-transcript.ts +1043 -0
- package/src/providers/native-history/claude-cli-transcript.ts +396 -0
- package/src/providers/native-history/codex-cli-transcript.ts +479 -0
- package/src/providers/native-history/constants.ts +19 -0
- package/src/providers/native-history/dispatcher.ts +373 -0
- package/src/providers/native-history/hermes-cli-transcript.ts +260 -0
- package/src/providers/native-history/index.ts +30 -0
- package/src/providers/provider-instance-manager.ts +71 -0
- package/src/providers/provider-instance.ts +19 -1
- package/src/providers/provider-loader.ts +668 -50
- package/src/providers/provider-schema.ts +87 -14
- package/src/providers/provider-trust.ts +114 -0
- package/src/providers/read-chat-contract.ts +76 -16
- package/src/providers/sdk/README.md +49 -0
- package/src/providers/sdk/v1/builders/acp/detect-status.ts +144 -0
- package/src/providers/sdk/v1/builders/cli/detect-status.ts +313 -0
- package/src/providers/sdk/v1/builders/cli/parse-approval-squash.ts +158 -0
- package/src/providers/sdk/v1/builders/cli/parse-approval.ts +250 -0
- package/src/providers/sdk/v1/builders/cli/parse-session.ts +276 -0
- package/src/providers/sdk/v1/builders/cli/visible-region.ts +143 -0
- package/src/providers/sdk/v1/fixture-tooling/format.ts +130 -0
- package/src/providers/sdk/v1/fixture-tooling/index.ts +22 -0
- package/src/providers/sdk/v1/fixture-tooling/replay.ts +352 -0
- package/src/providers/sdk/v1/index.ts +152 -0
- package/src/providers/sdk/v1/sandbox/README-design.ts +195 -0
- package/src/providers/sdk/v1/sandbox/require-whitelist.ts +472 -0
- package/src/providers/sdk/v1/sandbox/script-runner.ts +150 -0
- package/src/providers/sdk/v1/schemas/cli/provider.schema.json +509 -0
- package/src/providers/sdk/v1/schemas/primitives/acp-session-protocol-v1.json +131 -0
- package/src/providers/sdk/v1/schemas/primitives/native-history-codex-rollout-v1.json +66 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-approval-squash-v1.json +91 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-assistant-block-v1.json +91 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-cue-ordering-v1.json +47 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-dispatch-order-v1.json +32 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-footer-chrome-v1.json +42 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-index-finder-v1.json +27 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-modal-v1.json +125 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-prompt-marker-v1.json +45 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-session-id-extraction-v1.json +46 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-settled-prompt-v1.json +71 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-spinner-v1.json +83 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-transcript-pty-v1.json +83 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-visible-region-v1.json +57 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-welcome-screen-v1.json +35 -0
- package/src/providers/sdk/v1/types/cli/index.ts +365 -0
- package/src/providers/sdk/v1/types/common/index.ts +229 -0
- package/src/providers/sdk/v1/validators/index.ts +19 -0
- package/src/providers/sdk/v1/validators/manifest.ts +110 -0
- package/src/providers/sdk/v1/validators/taint.ts +309 -0
- package/src/providers/spec/adapter.ts +246 -0
- package/src/providers/spec/cli-adapter.ts +1064 -0
- package/src/providers/spec/evaluator.ts +407 -0
- package/src/providers/spec/fsm-driver.ts +1481 -0
- package/src/providers/spec/fsm-evaluator.ts +290 -0
- package/src/providers/spec/fsm-loader.ts +128 -0
- package/src/providers/spec/fsm-types.ts +301 -0
- package/src/providers/spec/native-history-executor.ts +1174 -0
- package/src/providers/spec/pre-launch-trust.ts +104 -0
- package/src/providers/spec/route.ts +51 -0
- package/src/providers/spec/types.ts +262 -0
- package/src/providers/status-monitor.d.ts +7 -7
- package/src/providers/status-monitor.ts +37 -22
- package/src/providers/transcript-v2.ts +567 -0
- package/src/providers/types/interactive-prompt.ts +536 -0
- package/src/providers/version-archive.ts +64 -24
- package/src/providers/working-dir.ts +23 -0
- package/src/repo-mesh-types.ts +758 -14
- package/src/runtime-defaults.ts +39 -0
- package/src/session-host/managed-host.ts +218 -0
- package/src/sessions/registry.ts +6 -0
- package/src/shared-types-extra.ts +2 -4
- package/src/shared-types.d.ts +8 -0
- package/src/shared-types.ts +64 -1
- package/src/status/builders.ts +26 -6
- package/src/status/normalize.ts +2 -0
- package/src/status/reporter.ts +19 -1
- package/src/status/snapshot.ts +95 -26
- package/src/system/hash.ts +23 -0
- package/src/system/host-memory.ts +29 -12
- package/src/system/load-better-sqlite3.ts +68 -0
- package/src/types.ts +5 -0
- package/dist/cli-adapters/terminal-backends/xterm-backend.d.ts +0 -17
- package/dist/mesh/mesh-sync.d.ts +0 -53
- package/src/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +0 -16
- package/src/cli-adapters/terminal-backends/xterm-backend.ts +0 -97
- package/src/mesh/mesh-sync.ts +0 -111
|
@@ -0,0 +1,1897 @@
|
|
|
1
|
+
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
|
+
import { loadConfig } from '../config/config.js';
|
|
3
|
+
import { getMesh, getMeshByRepo, listMeshes } from '../config/mesh-config.js';
|
|
4
|
+
import { LOG } from '../logging/logger.js';
|
|
5
|
+
import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
|
|
6
|
+
import type { SessionRecoveryContext } from './mesh-ledger.js';
|
|
7
|
+
import { updateSessionTaskStatus, enqueueTask, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents, getQueue } from './mesh-work-queue.js';
|
|
8
|
+
import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
|
|
9
|
+
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
10
|
+
import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents, type PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
11
|
+
import type { ProviderInstance } from '../providers/provider-instance.js';
|
|
12
|
+
import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
|
|
13
|
+
import { resolveMeshHostStatus } from './mesh-host-ownership.js';
|
|
14
|
+
import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
|
|
15
|
+
import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
|
|
16
|
+
import { getLastDisplayMessage } from '../status/snapshot.js';
|
|
17
|
+
import { resolveDelegatedWorkerAutoApprove } from '../repo-mesh-types.js';
|
|
18
|
+
import { meshNodeIdMatches, daemonIdsEquivalent, expandDaemonIdForms, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
19
|
+
import {
|
|
20
|
+
findRecentTerminalLedgerEvidence,
|
|
21
|
+
hasDispatchAfterTerminal,
|
|
22
|
+
hasUnterminalDirectDispatchLedgerEntry,
|
|
23
|
+
buildNoProgressCompletionReconciliation,
|
|
24
|
+
} from './mesh-events-stale.js';
|
|
25
|
+
import {
|
|
26
|
+
buildMeshSystemMessage,
|
|
27
|
+
readNonEmptyString,
|
|
28
|
+
resolveEventSessionId,
|
|
29
|
+
readRefineJobId,
|
|
30
|
+
readWorkerResultMetadata,
|
|
31
|
+
resolveMeshSurfacedSessionPreview,
|
|
32
|
+
isFalseIdleCompletion,
|
|
33
|
+
isWeakCompletionEvidence,
|
|
34
|
+
} from './mesh-events-utils.js';
|
|
35
|
+
import { isMeshCoordinatorEvent, shouldForceInjectMeshEvent, EVENT_TO_LEDGER_KIND } from './mesh-event-classify.js';
|
|
36
|
+
import {
|
|
37
|
+
getMeshWithCache,
|
|
38
|
+
tryAssignQueueTask,
|
|
39
|
+
triggerMeshQueue,
|
|
40
|
+
runIdleMaintenanceThenAssignQueue,
|
|
41
|
+
maybeAutoFastForwardIdleNode,
|
|
42
|
+
sessionHasActiveAssignment,
|
|
43
|
+
} from './mesh-queue-assignment.js';
|
|
44
|
+
|
|
45
|
+
// The set of coordinator-daemon ids this daemon answers to when draining the
|
|
46
|
+
// pending-events queue. Mirrors resolveCoordinatorDaemonIds in mesh-reconcile-loop:
|
|
47
|
+
// a unicast event may be stamped with the status id, the bare machineId, OR the
|
|
48
|
+
// config-form node daemonId (`daemon_<machineId>`) depending on which dispatch path
|
|
49
|
+
// created the worker. We expand to EVERY equivalent form so a `daemon_<machineId>`
|
|
50
|
+
// completion matches a coordinator that knows itself as bare `<machineId>` (the
|
|
51
|
+
// base-node completion-surface bug) and vice versa.
|
|
52
|
+
function resolveCoordinatorDrainDaemonIds(components: DaemonComponents): string[] {
|
|
53
|
+
const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
|
|
54
|
+
const machineId = readNonEmptyString(loadConfig().machineId);
|
|
55
|
+
return expandDaemonIdForms([statusInstanceId, machineId]);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
// Remote Node Idle Session Tracking
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
const REMOTE_IDLE_SESSION_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
|
62
|
+
|
|
63
|
+
// ---------------------------------------------------------------------------
|
|
64
|
+
// Workspace-to-mesh lookup cache
|
|
65
|
+
// ---------------------------------------------------------------------------
|
|
66
|
+
const meshByWorkspaceCache = new Map<string, { mesh: any; cachedAt: number }>();
|
|
67
|
+
const MESH_WORKSPACE_CACHE_TTL_MS = 5_000;
|
|
68
|
+
|
|
69
|
+
function getCachedMeshByWorkspace(workspace: string): any {
|
|
70
|
+
const now = Date.now();
|
|
71
|
+
const cached = meshByWorkspaceCache.get(workspace);
|
|
72
|
+
if (cached && now - cached.cachedAt < MESH_WORKSPACE_CACHE_TTL_MS) return cached.mesh;
|
|
73
|
+
const mesh = getMeshByRepo(workspace);
|
|
74
|
+
meshByWorkspaceCache.set(workspace, { mesh, cachedAt: now });
|
|
75
|
+
return mesh;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// Deterministic meshId recovery for a forwarded worker event that carries no meshId.
|
|
79
|
+
// An unresolved-mesh worker (forwardUnresolvedDelegateEvent) cannot resolve its own
|
|
80
|
+
// meshId locally, so it pushes the event with nodeId + workspace only and relies on
|
|
81
|
+
// the coordinator — which hosts the mesh — to recover the id. Workspace recovery
|
|
82
|
+
// (getCachedMeshByWorkspace → getMeshByRepo) is the fast path but can miss (a worktree
|
|
83
|
+
// clone whose repoIdentity differs, or a transient cache state), which left the retry
|
|
84
|
+
// permanently rejected with "meshId required". The node-id IS a stable, coordinator-side
|
|
85
|
+
// fact: scan the hosted meshes for the one whose node matches the forwarded nodeId
|
|
86
|
+
// (3-form normalizer). This is timing-independent and never depends on repo lookup.
|
|
87
|
+
function recoverMeshIdByNodeId(nodeId: string): string {
|
|
88
|
+
if (!nodeId) return '';
|
|
89
|
+
for (const mesh of listMeshes()) {
|
|
90
|
+
if (Array.isArray(mesh.nodes) && mesh.nodes.some((n: any) => meshNodeIdMatches(n, nodeId))) {
|
|
91
|
+
return readNonEmptyString(mesh.id);
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
return '';
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// MESHID-DROP coordinator-anchor recovery (Fix B): the last-resort meshId recovery for an
|
|
98
|
+
// unresolved-delegate forward whose payload carries the worker's coordinator anchor
|
|
99
|
+
// (meshCoordinatorDaemonId) but no resolvable meshId — neither workspace nor nodeId scan
|
|
100
|
+
// matched (a freshly-cloned worktree node not yet registered under the forwarded id form, or
|
|
101
|
+
// an empty payload nodeId). The receiving daemon IS the coordinator/host, so scope to the
|
|
102
|
+
// meshes IT hosts whose host daemon matches the anchor (daemonIdsEquivalent — never a raw
|
|
103
|
+
// compare, so daemon_mach_/mach_/standalone_ forms all match the same machine). Among those:
|
|
104
|
+
// • a nodeId → the mesh whose nodes contain it (meshNodeIdMatches 3-form) wins;
|
|
105
|
+
// • no nodeId → fall back to the anchor's SINGLE hosted mesh only. Ambiguity (the anchor
|
|
106
|
+
// hosts >1 mesh and no node disambiguates) returns '' rather than guess — a wrong meshId
|
|
107
|
+
// would inject the completion into an unrelated mesh's ledger, worse than the retry-cap drop.
|
|
108
|
+
export function recoverMeshIdByCoordinatorAndNode(coordinatorDaemonId: string, nodeId: string): string {
|
|
109
|
+
if (!coordinatorDaemonId) return '';
|
|
110
|
+
const hosted = listMeshes().filter(mesh => {
|
|
111
|
+
const host = resolveMeshHostStatus(mesh);
|
|
112
|
+
return host.role === 'host'
|
|
113
|
+
&& (!host.hostDaemonId || daemonIdsEquivalent(host.hostDaemonId, coordinatorDaemonId));
|
|
114
|
+
});
|
|
115
|
+
if (hosted.length === 0) return '';
|
|
116
|
+
if (nodeId) {
|
|
117
|
+
const byNode = hosted.find(mesh =>
|
|
118
|
+
Array.isArray(mesh.nodes) && mesh.nodes.some((n: any) => meshNodeIdMatches(n, nodeId)));
|
|
119
|
+
if (byNode) return readNonEmptyString(byNode.id);
|
|
120
|
+
// nodeId present but matched no hosted mesh — do NOT fall through to the single-mesh
|
|
121
|
+
// guess; the node belongs to a mesh we don't host (or under a different id), and
|
|
122
|
+
// guessing would misroute. Stay unresolved.
|
|
123
|
+
return '';
|
|
124
|
+
}
|
|
125
|
+
// No nodeId to disambiguate: only safe when the anchor hosts exactly one mesh.
|
|
126
|
+
return hosted.length === 1 ? readNonEmptyString(hosted[0].id) : '';
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// RECONCILE-MESHID-DROP: WORKER-side meshId resolution for an unresolved-delegate
|
|
130
|
+
// forward payload. forwardUnresolvedDelegateEvent omits meshId by design (the worker
|
|
131
|
+
// "can't resolve it") and relies on the COORDINATOR recovering it from workspace/nodeId.
|
|
132
|
+
// That recovery fails when the no_node_binding session's payload has an empty nodeId AND
|
|
133
|
+
// the coordinator's workspace→mesh lookup misses (a worktree clone whose repoIdentity
|
|
134
|
+
// differs / a cache miss) — leaving the reconcile retry rejected with "meshId required"
|
|
135
|
+
// every 4s forever. The worker actually has MORE context than the stripped payload gives
|
|
136
|
+
// the coordinator: it hosts the node as a member and holds the LIVE session, whose
|
|
137
|
+
// settings.meshNodeFor / meshNodeId are authoritative even when they were not stamped
|
|
138
|
+
// onto the original event. Resolve here (worker side) and stamp meshId onto the payload so
|
|
139
|
+
// the coordinator accepts it. Mirrors the receiver's recovery order, then adds the live-
|
|
140
|
+
// session fallback. Returns '' when even the worker cannot resolve it (truly unresolvable —
|
|
141
|
+
// the retry cap then drops it instead of looping). No side effects; safe to call per retry.
|
|
142
|
+
export function resolveForwardEventMeshId(
|
|
143
|
+
components: DaemonComponents,
|
|
144
|
+
payload: Record<string, unknown>,
|
|
145
|
+
): string {
|
|
146
|
+
const direct = readNonEmptyString(payload.meshId);
|
|
147
|
+
if (direct) return direct;
|
|
148
|
+
const workspace = readNonEmptyString(payload.workspace);
|
|
149
|
+
const byWorkspace = workspace ? readNonEmptyString(getCachedMeshByWorkspace(workspace)?.id) : '';
|
|
150
|
+
if (byWorkspace) return byWorkspace;
|
|
151
|
+
const byNode = recoverMeshIdByNodeId(readNonEmptyString(payload.nodeId));
|
|
152
|
+
if (byNode) return byNode;
|
|
153
|
+
// Live-session fallback: the worker session may carry meshNodeFor / meshNodeId now even
|
|
154
|
+
// though the original event didn't (a late stamp, or an event that fired before binding).
|
|
155
|
+
const sessionId = readNonEmptyString(payload.targetSessionId)
|
|
156
|
+
|| readNonEmptyString(payload.sessionId)
|
|
157
|
+
|| readNonEmptyString(payload.instanceId);
|
|
158
|
+
if (sessionId) {
|
|
159
|
+
try {
|
|
160
|
+
const state = components.instanceManager?.getInstance?.(sessionId)?.getState?.();
|
|
161
|
+
const settings = (state?.settings as Record<string, unknown>) || {};
|
|
162
|
+
const meshNodeFor = readNonEmptyString(settings.meshNodeFor);
|
|
163
|
+
if (meshNodeFor) return meshNodeFor;
|
|
164
|
+
const byStamp = recoverMeshIdByNodeId(readNonEmptyString(settings.meshNodeId));
|
|
165
|
+
if (byStamp) return byStamp;
|
|
166
|
+
const sessionWorkspace = readNonEmptyString(state?.workspace);
|
|
167
|
+
const bySessionWorkspace = sessionWorkspace ? readNonEmptyString(getCachedMeshByWorkspace(sessionWorkspace)?.id) : '';
|
|
168
|
+
if (bySessionWorkspace) return bySessionWorkspace;
|
|
169
|
+
} catch { /* best-effort — fall through to unresolved */ }
|
|
170
|
+
}
|
|
171
|
+
return '';
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
export function __resetMeshWorkspaceCacheForTests(): void {
|
|
176
|
+
meshByWorkspaceCache.clear();
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function sweepExpiredRemoteIdleSessions(): void {
|
|
180
|
+
try {
|
|
181
|
+
MeshRuntimeStore.getInstance().pruneExpiredRemoteIdleSessions();
|
|
182
|
+
} catch { /* best-effort */ }
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS = 30 * 60 * 1000;
|
|
186
|
+
|
|
187
|
+
function isIntentionalCleanupStopMetadata(event: Record<string, unknown>): boolean {
|
|
188
|
+
return event.intentional === true
|
|
189
|
+
|| event.intentionalStop === true
|
|
190
|
+
|| event.operatorCleanup === true
|
|
191
|
+
|| event.reason === 'operator_cleanup'
|
|
192
|
+
|| event.stopReason === 'operator_cleanup'
|
|
193
|
+
|| event.cleanupReason === 'operator_cleanup'
|
|
194
|
+
|| event.source === 'mesh_cleanup_sessions'
|
|
195
|
+
|| event.source === 'mesh_remove_node';
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function hasRecentIntentionalCleanupStop(meshId: string, sessionId?: string, nodeId?: string): boolean {
|
|
199
|
+
if (!sessionId && !nodeId) return false;
|
|
200
|
+
const cutoff = Date.now() - INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS;
|
|
201
|
+
const entries = readLedgerEntries(meshId, { tail: 200 });
|
|
202
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
203
|
+
const entry = entries[i];
|
|
204
|
+
const timestamp = new Date(entry.timestamp).getTime();
|
|
205
|
+
if (!Number.isNaN(timestamp) && timestamp < cutoff) break;
|
|
206
|
+
if (!isIntentionalCleanupStopEntry(entry)) continue;
|
|
207
|
+
// SESSION-ID IS SINGLE-FORM: a session id is one canonical UUID (crypto.randomUUID
|
|
208
|
+
// in the provider instance), carried verbatim across daemons — no serialization
|
|
209
|
+
// variants like node/daemon ids. Exact `===` is correct; no equivalence helper.
|
|
210
|
+
if (sessionId && entry.sessionId === sessionId) return true;
|
|
211
|
+
// Normalized node-id match (P4): the cleanup-stop entry's node id may be stored as
|
|
212
|
+
// `nodeId` or `node_id` and the `nodeId` arg can be in either form — a raw `===`
|
|
213
|
+
// would miss a genuine intentional-cleanup entry and fail to suppress the stop event.
|
|
214
|
+
if (!sessionId && nodeId && meshNodeIdMatches(entry as unknown as MeshNodeIdentified, nodeId)) return true;
|
|
215
|
+
}
|
|
216
|
+
return false;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function shouldSuppressIntentionalCleanupStop(args: {
|
|
220
|
+
event: string;
|
|
221
|
+
meshId: string;
|
|
222
|
+
metadataEvent: Record<string, unknown>;
|
|
223
|
+
sessionId?: string;
|
|
224
|
+
nodeId?: string;
|
|
225
|
+
}): boolean {
|
|
226
|
+
if (args.event !== 'agent:stopped' && args.event !== 'monitor:no_progress') return false;
|
|
227
|
+
if (isIntentionalCleanupStopMetadata(args.metadataEvent)) return true;
|
|
228
|
+
return hasRecentIntentionalCleanupStop(args.meshId, args.sessionId, args.nodeId);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
const RECENT_COMPLETION_FINGERPRINT_TTL_MS = 10 * 60 * 1000;
|
|
232
|
+
|
|
233
|
+
function hasFingerprintSeen(meshId: string, fingerprint: string): boolean {
|
|
234
|
+
try {
|
|
235
|
+
return MeshRuntimeStore.getInstance().hasCompletionFingerprint(meshId, fingerprint);
|
|
236
|
+
} catch {
|
|
237
|
+
return false;
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function recordFingerprintSeen(meshId: string, fingerprint: string): void {
|
|
242
|
+
try {
|
|
243
|
+
const db = MeshRuntimeStore.getInstance();
|
|
244
|
+
db.recordCompletionFingerprint(meshId, fingerprint, RECENT_COMPLETION_FINGERPRINT_TTL_MS);
|
|
245
|
+
db.sweepExpiredFingerprints();
|
|
246
|
+
} catch { /* best-effort; duplicate events are preferable to a crash */ }
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
function readEventTimestamp(value: unknown): number | null {
|
|
250
|
+
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
|
251
|
+
if (typeof value === 'string' && value.trim()) {
|
|
252
|
+
const numeric = Number(value);
|
|
253
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
254
|
+
const parsed = Date.parse(value);
|
|
255
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
256
|
+
}
|
|
257
|
+
return null;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function buildMeshCompletionFingerprint(args: {
|
|
261
|
+
meshId: string;
|
|
262
|
+
event: string;
|
|
263
|
+
sessionId: string;
|
|
264
|
+
providerType?: string;
|
|
265
|
+
providerSessionId?: string;
|
|
266
|
+
timestamp?: number | null;
|
|
267
|
+
finalSummary?: string;
|
|
268
|
+
coordinatorDaemonId?: string;
|
|
269
|
+
}): string {
|
|
270
|
+
const timestampPart = Number.isFinite(args.timestamp)
|
|
271
|
+
? String(args.timestamp)
|
|
272
|
+
: readNonEmptyString(args.finalSummary).slice(0, 200);
|
|
273
|
+
return [
|
|
274
|
+
args.meshId,
|
|
275
|
+
args.event,
|
|
276
|
+
args.sessionId,
|
|
277
|
+
args.providerType || '',
|
|
278
|
+
args.providerSessionId || '',
|
|
279
|
+
timestampPart,
|
|
280
|
+
args.coordinatorDaemonId || '',
|
|
281
|
+
].join('::');
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
function isDuplicateMeshCompletionEvent(args: {
|
|
285
|
+
meshId: string;
|
|
286
|
+
event: string;
|
|
287
|
+
sessionId: string;
|
|
288
|
+
providerType?: string;
|
|
289
|
+
providerSessionId?: string;
|
|
290
|
+
timestamp?: number | null;
|
|
291
|
+
finalSummary?: string;
|
|
292
|
+
coordinatorDaemonId?: string;
|
|
293
|
+
taskId?: string;
|
|
294
|
+
nodeId?: string;
|
|
295
|
+
}): boolean {
|
|
296
|
+
const fingerprint = buildMeshCompletionFingerprint(args);
|
|
297
|
+
if (!fingerprint) return false;
|
|
298
|
+
if (hasFingerprintSeen(args.meshId, fingerprint)) {
|
|
299
|
+
if (args.taskId) {
|
|
300
|
+
recordCompletionConflict({
|
|
301
|
+
meshId: args.meshId,
|
|
302
|
+
fingerprint,
|
|
303
|
+
conflictingTaskId: args.taskId,
|
|
304
|
+
conflictingSessionId: args.sessionId,
|
|
305
|
+
event: args.event,
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
return true;
|
|
309
|
+
}
|
|
310
|
+
recordFingerprintSeen(args.meshId, fingerprint);
|
|
311
|
+
return false;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function isDuplicateMeshApprovalEvent(args: {
|
|
315
|
+
meshId: string;
|
|
316
|
+
sessionId: string;
|
|
317
|
+
providerType?: string;
|
|
318
|
+
timestamp?: number | null;
|
|
319
|
+
modalMessage?: string;
|
|
320
|
+
modalButtons?: unknown;
|
|
321
|
+
}): boolean {
|
|
322
|
+
const modalButtons = Array.isArray(args.modalButtons)
|
|
323
|
+
? args.modalButtons.map(button => String(button).trim()).filter(Boolean)
|
|
324
|
+
: [];
|
|
325
|
+
const approvalIdentity = Number.isFinite(args.timestamp)
|
|
326
|
+
? String(args.timestamp)
|
|
327
|
+
: JSON.stringify({ message: args.modalMessage || '', buttons: modalButtons });
|
|
328
|
+
if (!approvalIdentity || approvalIdentity === '{"message":"","buttons":[]}') return false;
|
|
329
|
+
const fingerprint = [
|
|
330
|
+
args.meshId,
|
|
331
|
+
'agent:waiting_approval',
|
|
332
|
+
args.sessionId,
|
|
333
|
+
args.providerType || '',
|
|
334
|
+
approvalIdentity,
|
|
335
|
+
].join('::');
|
|
336
|
+
if (hasFingerprintSeen(args.meshId, fingerprint)) return true;
|
|
337
|
+
recordFingerprintSeen(args.meshId, fingerprint);
|
|
338
|
+
return false;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
function isDuplicateRefineTerminalEvent(meshId: string, eventName: string, metadataEvent: Record<string, unknown>): boolean {
|
|
342
|
+
const jobId = readRefineJobId({ metadataEvent });
|
|
343
|
+
const fingerprint = jobId && new Set(['refine:completed', 'refine:failed']).has(eventName) ? `${meshId}::${eventName}::${jobId}` : '';
|
|
344
|
+
if (!fingerprint) return false;
|
|
345
|
+
if (hasFingerprintSeen(meshId, fingerprint)) return true;
|
|
346
|
+
recordFingerprintSeen(meshId, fingerprint);
|
|
347
|
+
return false;
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
// NOTIF-MISS (FIX 2): the set of `source` tags the transcript-reconcile synthesis path stamps
|
|
351
|
+
// onto the terminal ledger entry it writes (mesh-events-stale.reconcileDirectDispatchCompletion
|
|
352
|
+
// FromTranscript). A terminal carrying one of these was NOT produced by an actual provider
|
|
353
|
+
// agent_status_event — it is a coordinator-side reconstruction from the worker's transcript. The
|
|
354
|
+
// authoritative real completion must never be permanently masked by such a synthesized record.
|
|
355
|
+
const RECONCILED_COMPLETION_SOURCES = new Set([
|
|
356
|
+
'direct_task_transcript_reconciliation',
|
|
357
|
+
'daemon_reconcile_transcript_completion',
|
|
358
|
+
'mcp_mesh_status_transcript_reconciliation',
|
|
359
|
+
'no_progress_reconciliation',
|
|
360
|
+
]);
|
|
361
|
+
|
|
362
|
+
// True when the recorded terminal ledger entry was SYNTHESIZED by transcript reconciliation
|
|
363
|
+
// rather than emitted by the real provider event. The reconcile path tags its ledger payload
|
|
364
|
+
// with a reconcile `source`; a genuine provider completion carries no such tag (or a normal
|
|
365
|
+
// completionDiagnostic.reason). Both the explicit source AND the diagnostic reason are checked so
|
|
366
|
+
// either carrier identifies the synthesized record.
|
|
367
|
+
function isSynthesizedReconciledTerminal(terminalPayload: Record<string, unknown>): boolean {
|
|
368
|
+
const source = readNonEmptyString(terminalPayload.source);
|
|
369
|
+
if (source && RECONCILED_COMPLETION_SOURCES.has(source)) return true;
|
|
370
|
+
const diagnostic = terminalPayload.completionDiagnostic;
|
|
371
|
+
if (diagnostic && typeof diagnostic === 'object' && !Array.isArray(diagnostic)) {
|
|
372
|
+
const reason = readNonEmptyString((diagnostic as Record<string, unknown>).reason);
|
|
373
|
+
if (reason && RECONCILED_COMPLETION_SOURCES.has(reason)) return true;
|
|
374
|
+
}
|
|
375
|
+
return false;
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
// True when the INCOMING completion event is a REAL provider agent_status_event (an actual
|
|
379
|
+
// agent:generating_completed from the live session) rather than itself a synthesized/reconciled
|
|
380
|
+
// re-injection. We must only let a real event override a synthesized terminal — a second
|
|
381
|
+
// synthesized re-arrival should still dedup normally. A real provider event carries no reconcile
|
|
382
|
+
// `source` tag; a re-injected reconciliation does.
|
|
383
|
+
function isRealProviderCompletionEvent(metadataEvent: Record<string, unknown>): boolean {
|
|
384
|
+
const source = readNonEmptyString(metadataEvent.source);
|
|
385
|
+
return !source || !RECONCILED_COMPLETION_SOURCES.has(source);
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// The genuine-completion counterpart: a real final summary / worker result is present and
|
|
389
|
+
// the completion is not flagged as a missing-final-assistant false idle. Used to decide
|
|
390
|
+
// whether a new completion may supersede a prior WEAK (false-idle) terminal. NOTE this gates
|
|
391
|
+
// only on isFalseIdleCompletion (the completionDiagnostic subset), NOT the broader
|
|
392
|
+
// isWeakCompletionEvidence — preserving the original semantics where a self-declared
|
|
393
|
+
// weak/insufficient event still counts as genuine if it carries a real final summary.
|
|
394
|
+
function isGenuineCompletionEvidence(metadataEvent: Record<string, unknown>): boolean {
|
|
395
|
+
if (isFalseIdleCompletion(metadataEvent)) return false;
|
|
396
|
+
return !!readWorkerResultMetadata(metadataEvent) || !!readNonEmptyString(metadataEvent.finalSummary);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
// (FALSEIDLE-BGCHILD-b) A later genuine completion of the SAME task that carries a
|
|
400
|
+
// substantively different — and fuller — final summary than the recorded terminal is the REAL
|
|
401
|
+
// final that an earlier (false-idle) completion pre-empted, not a duplicate. The background-child
|
|
402
|
+
// false idle is the nasty case the plain isWeakCompletionEvidence supersession misses: the
|
|
403
|
+
// early completion's screen parser DID see a prior/intermediate standard assistant, so it is
|
|
404
|
+
// recorded as a STRONG terminal with a non-empty (but truncated) finalSummary. Without this the
|
|
405
|
+
// providerSessionId/finalSummary dedup below swallows the genuine final and the coordinator is
|
|
406
|
+
// stuck with the truncated mid-turn text forever (the one-shot-consumption symptom). Same-task,
|
|
407
|
+
// new event is genuine, prior terminal summary is a strict prefix of (or otherwise shorter than)
|
|
408
|
+
// the new one → treat as the corrected final and let it through. Conservative: requires the new
|
|
409
|
+
// summary to be genuine evidence AND meaningfully longer, so an identical re-arrival or a SHORTER
|
|
410
|
+
// later summary is still deduped.
|
|
411
|
+
function supersedesTruncatedTerminalSummary(args: {
|
|
412
|
+
terminalPayload: Record<string, unknown>;
|
|
413
|
+
metadataEvent: Record<string, unknown>;
|
|
414
|
+
terminalTaskId: string;
|
|
415
|
+
eventTaskId: string;
|
|
416
|
+
}): boolean {
|
|
417
|
+
// Only applies when both name the SAME task (a distinct task is handled by distinctTaskCompletion).
|
|
418
|
+
if (!args.terminalTaskId || !args.eventTaskId || args.terminalTaskId !== args.eventTaskId) return false;
|
|
419
|
+
if (!isGenuineCompletionEvidence(args.metadataEvent)) return false;
|
|
420
|
+
const terminalSummary = readNonEmptyString(args.terminalPayload.finalSummary);
|
|
421
|
+
const eventSummary = readNonEmptyString(args.metadataEvent.finalSummary);
|
|
422
|
+
if (!eventSummary) return false;
|
|
423
|
+
// Identical text → genuine duplicate, keep deduping.
|
|
424
|
+
if (terminalSummary === eventSummary) return false;
|
|
425
|
+
// The recorded terminal was a known-weak (false-idle) one → already handled by the weak
|
|
426
|
+
// supersession path; nothing extra to do here.
|
|
427
|
+
if (isWeakCompletionEvidence(args.terminalPayload)) return false;
|
|
428
|
+
// No prior summary at all, or the new summary strictly extends / is meaningfully longer than
|
|
429
|
+
// the recorded one → the recorded terminal was the truncated pre-emption; supersede it.
|
|
430
|
+
if (!terminalSummary) return true;
|
|
431
|
+
if (eventSummary.startsWith(terminalSummary)) return true;
|
|
432
|
+
return eventSummary.length > terminalSummary.length + 32;
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
// The latest still-active direct-dispatch taskId for a session, resolved BEFORE the
|
|
436
|
+
// completion flips the dispatch row terminal. Direct dispatches (mesh_send_task) have no
|
|
437
|
+
// work-queue row, so this is the only taskId available to attribute the terminal ledger
|
|
438
|
+
// entry (and thus mesh task-stats) to — without it the terminal carries no taskId and the
|
|
439
|
+
// task surfaces as status='unknown' / terminalKind=null in computeMeshTaskStats.
|
|
440
|
+
function resolveActiveDirectDispatchTaskId(meshId: string, sessionId: string): string | undefined {
|
|
441
|
+
try {
|
|
442
|
+
// SESSION-ID IS SINGLE-FORM (canonical crypto.randomUUID, carried verbatim across
|
|
443
|
+
// daemons) — exact `===` filter is correct; no node-id-style form normalization.
|
|
444
|
+
const matches = getActiveDirectDispatches(meshId).filter(d => d.sessionId === sessionId);
|
|
445
|
+
if (!matches.length) return undefined;
|
|
446
|
+
// getActiveDirectDispatches returns rows ordered by dispatched_at ASC; the last is
|
|
447
|
+
// the most recent dispatch (the re-dispatch / nudge whose completion this is).
|
|
448
|
+
return readNonEmptyString(matches[matches.length - 1].taskId) || undefined;
|
|
449
|
+
} catch {
|
|
450
|
+
return undefined;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
// Coordinator-side suppression/reconcile gate for an incoming mesh event. Each clause is a
|
|
455
|
+
// closed dedup/suppression concern that only inspects the event + already-resolved context and
|
|
456
|
+
// either (a) returns a `suppress` result the caller forwards verbatim, (b) returns a `reconcile`
|
|
457
|
+
// signal carrying the rewritten metadataEvent for the caller to re-inject as
|
|
458
|
+
// agent:generating_completed, or (c) returns null to let the event fall through to the
|
|
459
|
+
// terminal/ledger machinery. Extracted verbatim from injectMeshSystemMessage — no behavior
|
|
460
|
+
// change; the only side effects (best-effort remote-idle cleanup, LOG, trace) fire on the same
|
|
461
|
+
// paths as before.
|
|
462
|
+
function evaluateMeshEventSuppression(
|
|
463
|
+
args: {
|
|
464
|
+
meshId: string;
|
|
465
|
+
sourceInstanceId?: string;
|
|
466
|
+
nodeId?: string;
|
|
467
|
+
nodeLabel: string;
|
|
468
|
+
event: string;
|
|
469
|
+
metadataEvent: Record<string, unknown>;
|
|
470
|
+
},
|
|
471
|
+
ctx: {
|
|
472
|
+
traceCtx: Parameters<typeof traceMeshEventDrop>[1];
|
|
473
|
+
eventSessionId: string;
|
|
474
|
+
eventNodeId: string;
|
|
475
|
+
eventTimestamp: number | null;
|
|
476
|
+
workerCoordinatorDaemonId: string | undefined;
|
|
477
|
+
},
|
|
478
|
+
):
|
|
479
|
+
| { kind: 'suppress'; result: { success: true; forwarded: 0; suppressed: true; [extra: string]: unknown } }
|
|
480
|
+
| { kind: 'reconcile'; metadataEvent: Record<string, unknown> }
|
|
481
|
+
| null {
|
|
482
|
+
const { traceCtx, eventSessionId, eventNodeId, eventTimestamp, workerCoordinatorDaemonId } = ctx;
|
|
483
|
+
|
|
484
|
+
const intentionalCleanupStop = shouldSuppressIntentionalCleanupStop({
|
|
485
|
+
event: args.event,
|
|
486
|
+
meshId: args.meshId,
|
|
487
|
+
metadataEvent: args.metadataEvent,
|
|
488
|
+
sessionId: eventSessionId || undefined,
|
|
489
|
+
nodeId: eventNodeId || undefined,
|
|
490
|
+
});
|
|
491
|
+
if (intentionalCleanupStop) {
|
|
492
|
+
if (eventSessionId && eventNodeId) {
|
|
493
|
+
try {
|
|
494
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(args.meshId, eventNodeId, eventSessionId);
|
|
495
|
+
} catch { /* best-effort */ }
|
|
496
|
+
}
|
|
497
|
+
LOG.info('MeshEvents', `Suppressed ${args.event} for intentionally cleanup-stopped session ${eventSessionId || '(unknown session)'}`);
|
|
498
|
+
traceMeshEventDrop('intentional_cleanup_stop', traceCtx);
|
|
499
|
+
return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, intentionalCleanupStop: true } };
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
if (args.event === 'monitor:no_progress') {
|
|
503
|
+
const reconciledCompletion = buildNoProgressCompletionReconciliation({
|
|
504
|
+
meshId: args.meshId,
|
|
505
|
+
nodeId: args.nodeId,
|
|
506
|
+
nodeLabel: args.nodeLabel,
|
|
507
|
+
metadataEvent: args.metadataEvent,
|
|
508
|
+
sourceInstanceId: args.sourceInstanceId,
|
|
509
|
+
});
|
|
510
|
+
if (reconciledCompletion?.source === 'no_progress_reconciliation') {
|
|
511
|
+
LOG.info('MeshEvents', `Reconciled no-progress monitor to completion for session ${eventSessionId || '(unknown session)'}`);
|
|
512
|
+
return { kind: 'reconcile', metadataEvent: reconciledCompletion };
|
|
513
|
+
}
|
|
514
|
+
if (reconciledCompletion?.source === 'no_progress_terminal_ledger_suppression') {
|
|
515
|
+
LOG.info('MeshEvents', `Suppressed no-progress monitor because terminal ledger evidence already exists for session ${eventSessionId || '(unknown session)'}`);
|
|
516
|
+
traceMeshEventDrop('no_progress_terminal_ledger_suppression', traceCtx, `terminalKind=${reconciledCompletion.terminalLedgerKind}`);
|
|
517
|
+
return {
|
|
518
|
+
kind: 'suppress',
|
|
519
|
+
result: {
|
|
520
|
+
success: true,
|
|
521
|
+
forwarded: 0,
|
|
522
|
+
suppressed: true,
|
|
523
|
+
terminalLedgerEvidence: true,
|
|
524
|
+
terminalLedgerKind: reconciledCompletion.terminalLedgerKind,
|
|
525
|
+
},
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
if (isDuplicateRefineTerminalEvent(args.meshId, args.event, args.metadataEvent)) {
|
|
531
|
+
LOG.info('MeshEvents', `Suppressed duplicate ${args.event} for refine job ${readRefineJobId({ metadataEvent: args.metadataEvent })}`);
|
|
532
|
+
traceMeshEventDrop('duplicate_refine_terminal', traceCtx);
|
|
533
|
+
return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, duplicateRefineTerminalEvent: true } };
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
if (args.event === 'agent:waiting_approval' && eventSessionId) {
|
|
537
|
+
const duplicateApproval = isDuplicateMeshApprovalEvent({
|
|
538
|
+
meshId: args.meshId,
|
|
539
|
+
sessionId: eventSessionId,
|
|
540
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
541
|
+
timestamp: eventTimestamp,
|
|
542
|
+
modalMessage: readNonEmptyString(args.metadataEvent.modalMessage) || undefined,
|
|
543
|
+
modalButtons: args.metadataEvent.modalButtons,
|
|
544
|
+
});
|
|
545
|
+
if (duplicateApproval) {
|
|
546
|
+
LOG.info('MeshEvents', `Suppressed duplicate approval event for mesh ${args.meshId} session ${eventSessionId}`);
|
|
547
|
+
traceMeshEventDrop('duplicate_approval', traceCtx);
|
|
548
|
+
return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, duplicateApproval: true } };
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
if (args.event === 'agent:generating_completed' && eventSessionId) {
|
|
552
|
+
const terminal = findRecentTerminalLedgerEvidence({
|
|
553
|
+
meshId: args.meshId,
|
|
554
|
+
sessionId: eventSessionId,
|
|
555
|
+
nodeId: eventNodeId || undefined,
|
|
556
|
+
});
|
|
557
|
+
if (terminal?.kind === 'task_completed' && !sessionHasActiveAssignment(args.meshId, eventSessionId)) {
|
|
558
|
+
const newDispatchAfterTerminal = hasDispatchAfterTerminal(args.meshId, eventSessionId, terminal.id);
|
|
559
|
+
// Fix B (re-dispatch 2nd-completion routing): a prior terminal recorded from a FALSE
|
|
560
|
+
// idle (weak evidence / no confirmed final assistant) must NOT permanently suppress a
|
|
561
|
+
// later GENUINE completion of the same session. providerSessionId is stable across a
|
|
562
|
+
// session's turns, so the providerSessionId/finalSummary dedup below would otherwise
|
|
563
|
+
// swallow the real 2nd-turn completion that a coordinator nudge (direct re-dispatch)
|
|
564
|
+
// drove — exactly the missed-event bug. When the prior terminal was weak and the new
|
|
565
|
+
// event carries genuine completion evidence, let it through so it is recorded and
|
|
566
|
+
// re-attributed to the latest task (the normal task_completed path below).
|
|
567
|
+
const supersedesWeakTerminal = isWeakCompletionEvidence(terminal.payload)
|
|
568
|
+
&& isGenuineCompletionEvidence(args.metadataEvent);
|
|
569
|
+
// CANON-B (direct-dispatch completion race): a FAST direct dispatch (mesh_send_task)
|
|
570
|
+
// to an already-idle, previously-used session can have its genuine completion reach
|
|
571
|
+
// this coordinator handler BEFORE the dispatching side records the new task's dispatch
|
|
572
|
+
// row / task_dispatched ledger entry — insertDirectDispatch + appendLedgerEntry both run
|
|
573
|
+
// AFTER the agent_command await resolves, while the worker may already be done. In that
|
|
574
|
+
// window sessionHasActiveAssignment is false (no active dispatch row, no unterminal
|
|
575
|
+
// ledger entry yet), so this prior-terminal dedup engages; and because providerSessionId
|
|
576
|
+
// is STABLE across a reused session's turns, the providerSessionId/finalSummary match
|
|
577
|
+
// below would suppress the NEW task's completion as a duplicate of the PRIOR task —
|
|
578
|
+
// silently losing it (the observed intermittent miss; fresh enqueue/autoLaunch is immune
|
|
579
|
+
// because a fresh session has no prior same-providerSessionId terminal and the queue row
|
|
580
|
+
// is claimed atomically before dispatch). The echoed taskId is the authoritative
|
|
581
|
+
// discriminator: when the completion names a DIFFERENT task than the recorded terminal,
|
|
582
|
+
// it is a genuinely new task's completion, never a duplicate — let it through so it is
|
|
583
|
+
// attributed to its own taskId. A same-task re-arrival (taskId equal) or a taskId-less
|
|
584
|
+
// legacy event still falls through to the providerSessionId/finalSummary dedup.
|
|
585
|
+
const terminalTaskId = readNonEmptyString(terminal.payload.taskId);
|
|
586
|
+
const eventTaskId = readNonEmptyString(args.metadataEvent.taskId);
|
|
587
|
+
const distinctTaskCompletion = !!eventTaskId && !!terminalTaskId && eventTaskId !== terminalTaskId;
|
|
588
|
+
// (FALSEIDLE-BGCHILD-b) Same-task genuine completion carrying a fuller summary than the
|
|
589
|
+
// recorded (truncated, false-idle-pre-empted) terminal supersedes it — see helper.
|
|
590
|
+
const supersedesTruncatedTerminal = supersedesTruncatedTerminalSummary({
|
|
591
|
+
terminalPayload: terminal.payload,
|
|
592
|
+
metadataEvent: args.metadataEvent,
|
|
593
|
+
terminalTaskId,
|
|
594
|
+
eventTaskId,
|
|
595
|
+
});
|
|
596
|
+
// NOTIF-MISS (FIX 2): the recorded terminal was SYNTHESIZED by transcript reconciliation
|
|
597
|
+
// (no real provider event), and THIS incoming event is the authoritative REAL provider
|
|
598
|
+
// completion. The reconcile path may fire ~1s after a direct dispatch from a reused
|
|
599
|
+
// session's stale transcript tail, writing a synthesized terminal whose providerSessionId
|
|
600
|
+
// (stable across a session's turns) and/or finalSummary then match the real completion —
|
|
601
|
+
// so the providerSessionId/finalSummary dedup below would drop the genuine event and the
|
|
602
|
+
// coordinator would never learn the task finished. A synthesized record must NEVER
|
|
603
|
+
// permanently mask the real completion: let the real event through (it is re-recorded by
|
|
604
|
+
// the normal task_completed path, superseding the synthesized one). A second SYNTHESIZED
|
|
605
|
+
// re-arrival is NOT a real event and still dedups normally.
|
|
606
|
+
// RECONCILE-SYNTH-PREEMPTS-COMPLETION: the bar here is deliberately LOWER than the
|
|
607
|
+
// weak/truncated supersessions above — a synthesized terminal is a coordinator-side
|
|
608
|
+
// reconstruction, never a real provider event, so ANY genuine real provider completion
|
|
609
|
+
// for the session must win over it. Requiring the full isGenuineCompletionEvidence
|
|
610
|
+
// (finalSummary OR workerResult present) dropped the real event whenever the relay did
|
|
611
|
+
// not re-populate finalSummary at this layer — exactly the observed 71s task whose real
|
|
612
|
+
// generating_completed hit drop:duplicate_completion_terminal_ledger after a premature
|
|
613
|
+
// synth. We require only that the incoming event is a REAL provider completion that is
|
|
614
|
+
// not itself a false idle (missing-final-assistant); a real-but-false-idle event still
|
|
615
|
+
// does not supersede (it is not trustworthy terminal evidence either).
|
|
616
|
+
const supersedesSynthesizedTerminal = isSynthesizedReconciledTerminal(terminal.payload)
|
|
617
|
+
&& isRealProviderCompletionEvent(args.metadataEvent)
|
|
618
|
+
&& !isFalseIdleCompletion(args.metadataEvent);
|
|
619
|
+
if (!newDispatchAfterTerminal && !supersedesWeakTerminal && !distinctTaskCompletion && !supersedesTruncatedTerminal && !supersedesSynthesizedTerminal) {
|
|
620
|
+
const terminalProviderSessionId = readNonEmptyString(terminal.payload.providerSessionId);
|
|
621
|
+
const terminalFinalSummary = readNonEmptyString(terminal.payload.finalSummary);
|
|
622
|
+
const eventProviderSessionId = readNonEmptyString(args.metadataEvent.providerSessionId);
|
|
623
|
+
const eventFinalSummary = readNonEmptyString(args.metadataEvent.finalSummary);
|
|
624
|
+
if (
|
|
625
|
+
(terminalProviderSessionId && terminalProviderSessionId === eventProviderSessionId)
|
|
626
|
+
|| (terminalFinalSummary && terminalFinalSummary === eventFinalSummary)
|
|
627
|
+
|| args.metadataEvent.source === 'no_progress_reconciliation'
|
|
628
|
+
) {
|
|
629
|
+
LOG.info('MeshEvents', `Suppressed duplicate completion with existing terminal ledger evidence for mesh ${args.meshId} session ${eventSessionId}`);
|
|
630
|
+
traceMeshEventDrop('duplicate_completion_terminal_ledger', traceCtx);
|
|
631
|
+
return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, duplicateCompletion: true, terminalLedgerEvidence: true } };
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
const duplicateCompletion = isDuplicateMeshCompletionEvent({
|
|
636
|
+
meshId: args.meshId,
|
|
637
|
+
event: args.event,
|
|
638
|
+
sessionId: eventSessionId,
|
|
639
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
640
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
641
|
+
timestamp: eventTimestamp,
|
|
642
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
643
|
+
coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
|
|
644
|
+
taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
|
|
645
|
+
nodeId: eventNodeId || undefined,
|
|
646
|
+
});
|
|
647
|
+
if (duplicateCompletion) {
|
|
648
|
+
LOG.info('MeshEvents', `Suppressed duplicate completion for mesh ${args.meshId} session ${eventSessionId}`);
|
|
649
|
+
traceMeshEventDrop('duplicate_completion', traceCtx);
|
|
650
|
+
return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, duplicateCompletion: true } };
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
if (args.event === 'agent:stopped' && eventSessionId) {
|
|
654
|
+
const duplicateStopped = isDuplicateMeshCompletionEvent({
|
|
655
|
+
meshId: args.meshId,
|
|
656
|
+
event: args.event,
|
|
657
|
+
sessionId: eventSessionId,
|
|
658
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
659
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
660
|
+
timestamp: eventTimestamp,
|
|
661
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
662
|
+
coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
|
|
663
|
+
taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
|
|
664
|
+
nodeId: eventNodeId || undefined,
|
|
665
|
+
});
|
|
666
|
+
if (duplicateStopped) {
|
|
667
|
+
LOG.info('MeshEvents', `Suppressed duplicate stopped event for mesh ${args.meshId} session ${eventSessionId}`);
|
|
668
|
+
traceMeshEventDrop('duplicate_stopped', traceCtx);
|
|
669
|
+
return { kind: 'suppress', result: { success: true, forwarded: 0, suppressed: true, duplicateStopped: true } };
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
return null;
|
|
674
|
+
}
|
|
675
|
+
|
|
676
|
+
function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
677
|
+
meshId: string;
|
|
678
|
+
sourceInstanceId?: string;
|
|
679
|
+
nodeId?: string;
|
|
680
|
+
nodeLabel: string;
|
|
681
|
+
event: string;
|
|
682
|
+
metadataEvent: Record<string, unknown>;
|
|
683
|
+
}) {
|
|
684
|
+
const eventSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
685
|
+
const eventNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
686
|
+
|
|
687
|
+
// EVTTRACE correlation context for this event's coordinator-side lifecycle (queue /
|
|
688
|
+
// dedup / suppress). Observation only — never read by any decision below.
|
|
689
|
+
const traceCtx = {
|
|
690
|
+
taskId: args.metadataEvent.taskId,
|
|
691
|
+
sessionId: eventSessionId,
|
|
692
|
+
nodeId: eventNodeId,
|
|
693
|
+
meshId: args.meshId,
|
|
694
|
+
event: args.event,
|
|
695
|
+
};
|
|
696
|
+
|
|
697
|
+
const sourceSession = args.sourceInstanceId
|
|
698
|
+
? components.instanceManager.getInstance(args.sourceInstanceId)
|
|
699
|
+
: undefined;
|
|
700
|
+
const workerCoordinatorDaemonId = readNonEmptyString(
|
|
701
|
+
(sourceSession?.getState()?.settings as Record<string, unknown>)?.meshCoordinatorDaemonId,
|
|
702
|
+
);
|
|
703
|
+
// Session-level routing anchor (multi-coordinator). Prefer the LIVE worker session's
|
|
704
|
+
// stamp; fall back to a relayed value carried in metadataEvent.meshCoordinatorSessionId
|
|
705
|
+
// (a remote worker's completion arrives via handleMeshForwardEvent with no local
|
|
706
|
+
// sourceSession, so the stamp can only ride in the relayed metadata). Empty on legacy /
|
|
707
|
+
// version-skewed dispatches → the event stays daemon-broadcast (no regression).
|
|
708
|
+
const workerCoordinatorSessionId = readNonEmptyString(
|
|
709
|
+
(sourceSession?.getState()?.settings as Record<string, unknown>)?.meshCoordinatorSessionId,
|
|
710
|
+
) || readNonEmptyString(args.metadataEvent.meshCoordinatorSessionId);
|
|
711
|
+
|
|
712
|
+
// T2: a summary-less completion (and any non-completion status-sync event) carries no
|
|
713
|
+
// assistant text on the event, so resolveMeshSurfacedSessionPreview had nothing to surface
|
|
714
|
+
// and the coordinator's inbox mirror stayed stuck on the first dispatched user task. When
|
|
715
|
+
// THIS daemon hosts the live worker instance (sourceSession present), derive the worker's
|
|
716
|
+
// latest display message straight from its transcript and attach it to the event as
|
|
717
|
+
// lastMessagePreview/lastMessageRole/lastMessageAt. resolveMeshSurfacedSessionPreview reads
|
|
718
|
+
// these as an assistant-only fallback; they also ride the pending-queue + P2P relay
|
|
719
|
+
// (handleMeshForwardEvent whitelist) so a remote coordinator can surface them. A remote
|
|
720
|
+
// coordinator has no local instance and keeps relying on the relayed fields — unchanged.
|
|
721
|
+
const enrichedMetadataEvent = ((): Record<string, unknown> => {
|
|
722
|
+
const last = sourceSession ? getLastDisplayMessage(sourceSession.getState()) : null;
|
|
723
|
+
const base = (!last || !last.preview)
|
|
724
|
+
? args.metadataEvent
|
|
725
|
+
: {
|
|
726
|
+
...args.metadataEvent,
|
|
727
|
+
lastMessagePreview: last.preview,
|
|
728
|
+
lastMessageRole: last.role,
|
|
729
|
+
...(last.receivedAt > 0 ? { lastMessageAt: last.receivedAt } : {}),
|
|
730
|
+
};
|
|
731
|
+
// MAGI: stamp the queue task's consensusGroupId onto the completion metadata
|
|
732
|
+
// so the intentional-fan-out dedup exemption (buildPendingEventFingerprint)
|
|
733
|
+
// can see it. The work queue is owned by THIS host/coordinator daemon — where
|
|
734
|
+
// both the lookup and the dedup run — so the local lookup covers local and
|
|
735
|
+
// relayed workers alike. Best-effort: never fail the event path on a miss, and
|
|
736
|
+
// never clobber a consensusGroupId the worker already relayed.
|
|
737
|
+
if (readNonEmptyString((base as Record<string, unknown>).consensusGroupId)) return base;
|
|
738
|
+
const eventTaskId = readNonEmptyString(args.metadataEvent.taskId);
|
|
739
|
+
if (!eventTaskId) return base;
|
|
740
|
+
try {
|
|
741
|
+
const entry = MeshRuntimeStore.getInstance().findQueueEntryById(args.meshId, eventTaskId);
|
|
742
|
+
const consensusGroupId = readNonEmptyString((entry as { consensusGroupId?: unknown } | null)?.consensusGroupId);
|
|
743
|
+
if (consensusGroupId) return { ...base, consensusGroupId };
|
|
744
|
+
} catch { /* queue lookup is best-effort; absence just falls back to the generic fingerprint */ }
|
|
745
|
+
return base;
|
|
746
|
+
})();
|
|
747
|
+
|
|
748
|
+
// R2: cloud P2P dashboard metadata sync. The cloud daemon used to do this from its own
|
|
749
|
+
// relay listener; now the single core forwarder invokes the injected hook (no-op on
|
|
750
|
+
// standalone) so the event path stays single-listener and the local code path is identical
|
|
751
|
+
// across standalone and cloud.
|
|
752
|
+
if (components.onMeshCoordinatorEventForwarded) {
|
|
753
|
+
try {
|
|
754
|
+
// T: the coordinator surfaces a remote worker's session but holds no local
|
|
755
|
+
// instance for it, so the status snapshot can't derive a preview and the
|
|
756
|
+
// mirror would stay stuck on the first dispatched user task. Resolve the
|
|
757
|
+
// worker's latest assistant reply (carried on the completion event's
|
|
758
|
+
// finalSummary / workerResult) into a preview the mirror can stamp, so the
|
|
759
|
+
// mobile inbox reflects the assistant response. Completion events carry assistant
|
|
760
|
+
// text as finalSummary; a summary-less completion / status sync falls back to the
|
|
761
|
+
// worker's latest assistant display message (enrichedMetadataEvent.lastMessage*).
|
|
762
|
+
// For a mid-turn user-only event this is undefined and the prior surfaced preview
|
|
763
|
+
// is preserved downstream (no clobber).
|
|
764
|
+
const surfacedPreview = resolveMeshSurfacedSessionPreview(enrichedMetadataEvent);
|
|
765
|
+
components.onMeshCoordinatorEventForwarded({
|
|
766
|
+
event: args.event,
|
|
767
|
+
meshId: args.meshId,
|
|
768
|
+
nodeId: eventNodeId || undefined,
|
|
769
|
+
...enrichedMetadataEvent,
|
|
770
|
+
// Ensure a `workspace` field reaches updateMeshOwnedSession even when the
|
|
771
|
+
// worker provider event only carried `workspaceName`. The merge spread of
|
|
772
|
+
// metadataEvent above wins when it already has a non-empty `workspace`.
|
|
773
|
+
workspace: readNonEmptyString(args.metadataEvent.workspace)
|
|
774
|
+
|| readNonEmptyString(args.metadataEvent.workspaceName)
|
|
775
|
+
|| undefined,
|
|
776
|
+
...(surfacedPreview ? {
|
|
777
|
+
meshSessionLastMessagePreview: surfacedPreview.preview,
|
|
778
|
+
meshSessionLastMessageRole: surfacedPreview.role,
|
|
779
|
+
meshSessionLastMessageAt: surfacedPreview.receivedAt || undefined,
|
|
780
|
+
} : {}),
|
|
781
|
+
});
|
|
782
|
+
} catch { /* dashboard metadata sync is best-effort */ }
|
|
783
|
+
}
|
|
784
|
+
|
|
785
|
+
const eventTimestamp = readEventTimestamp(args.metadataEvent.timestamp);
|
|
786
|
+
// Coordinator-side dedup/suppression gate (extracted, behavior-preserving). A non-null
|
|
787
|
+
// outcome either short-circuits with a forwarded result or signals a no-progress→completion
|
|
788
|
+
// reconciliation that we re-inject; null lets the event fall through to the ledger machinery.
|
|
789
|
+
const suppression = evaluateMeshEventSuppression(args, {
|
|
790
|
+
traceCtx,
|
|
791
|
+
eventSessionId,
|
|
792
|
+
eventNodeId,
|
|
793
|
+
eventTimestamp,
|
|
794
|
+
workerCoordinatorDaemonId,
|
|
795
|
+
});
|
|
796
|
+
if (suppression) {
|
|
797
|
+
if (suppression.kind === 'reconcile') {
|
|
798
|
+
return injectMeshSystemMessage(components, {
|
|
799
|
+
...args,
|
|
800
|
+
event: 'agent:generating_completed',
|
|
801
|
+
metadataEvent: suppression.metadataEvent,
|
|
802
|
+
});
|
|
803
|
+
}
|
|
804
|
+
return suppression.result;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
function markSessionTerminal(sessionId: string, outcome: 'completed' | 'failed', occurredAtMs?: number | null, opts?: { tentativeIfDirect?: boolean }): { id?: string } | null {
|
|
808
|
+
// C2: prefer an exact taskId match when the completion event carries one —
|
|
809
|
+
// it's immune to coordinator↔worker clock skew that can hide the assigned row.
|
|
810
|
+
const eventTaskId = readNonEmptyString(args.metadataEvent.taskId) || undefined;
|
|
811
|
+
const task = updateSessionTaskStatus(args.meshId, sessionId, outcome, {
|
|
812
|
+
occurredAt: occurredAtMs != null ? new Date(occurredAtMs).toISOString() : undefined,
|
|
813
|
+
taskId: eventTaskId,
|
|
814
|
+
});
|
|
815
|
+
// Fix A (early-terminal prevention): a false-idle completion (no confirmed final
|
|
816
|
+
// assistant) for a DIRECT dispatch — i.e. no work-queue row matched — must not flip the
|
|
817
|
+
// dispatch row terminal. Leaving it active lets the reconcile loop (PHASE 4) re-read the
|
|
818
|
+
// transcript and record the genuine completion once the worker truly finishes (commonly
|
|
819
|
+
// after a coordinator nudge / re-dispatch). A matched queue task, or a completion with
|
|
820
|
+
// genuine evidence, is marked terminal as before.
|
|
821
|
+
// WARMUPGAP: a no-taskId completion from a session that holds no active assignment is a
|
|
822
|
+
// pre-assignment warmup / ghost event (a worker spawns, idles, and emits idle→generating→
|
|
823
|
+
// completed before any task is dispatched, with meshActiveTaskId unset so the event carries
|
|
824
|
+
// no taskId). Letting it through would hit the session_id fallback in updateDirectDispatchStatus
|
|
825
|
+
// and flip a sibling/stale dispatch row this event does not own — the real task later lands on
|
|
826
|
+
// a corrupted row and never reaches completed. Skip the dispatch update for that case. A
|
|
827
|
+
// taskId-carrying completion (real task), or any completion whose session currently holds an
|
|
828
|
+
// active assignment (legacy/relayed worker), still flips as before.
|
|
829
|
+
const leaveDirectDispatchActive = (!task && opts?.tentativeIfDirect === true)
|
|
830
|
+
|| (!eventTaskId && !sessionHasActiveAssignment(args.meshId, sessionId));
|
|
831
|
+
if (!leaveDirectDispatchActive) {
|
|
832
|
+
// CANON-B: flip the exact dispatch row the completion echoed its taskId for; the
|
|
833
|
+
// session_id fallback (no echoed taskId) still covers legacy/relayed workers.
|
|
834
|
+
updateDirectDispatchStatus(args.meshId, sessionId, outcome, eventTaskId);
|
|
835
|
+
}
|
|
836
|
+
markSessionDeliveriesTerminal(args.meshId, sessionId, outcome);
|
|
837
|
+
setImmediate(() => cleanupTerminalDirectDispatches());
|
|
838
|
+
return task ? { id: task.id } : null;
|
|
839
|
+
}
|
|
840
|
+
|
|
841
|
+
let completedTaskForLedger: { id?: string } | null = null;
|
|
842
|
+
// Fix B: direct-dispatch taskId used to attribute the terminal ledger entry when no
|
|
843
|
+
// work-queue row matches (resolved BEFORE markSessionTerminal flips the dispatch terminal).
|
|
844
|
+
let directDispatchTaskIdForLedger: string | undefined;
|
|
845
|
+
// BOOTSTRAP-MSG: whether a queued task already targets this node, so the
|
|
846
|
+
// worktree_bootstrap_complete [System] message reflects the auto-claim instead of
|
|
847
|
+
// advising a manual mesh_launch_session (which would spawn a duplicate session).
|
|
848
|
+
let worktreeHasQueuedTask = false;
|
|
849
|
+
if (args.event === 'agent:generating_completed') {
|
|
850
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
851
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
852
|
+
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
853
|
+
|
|
854
|
+
if (sessionId) {
|
|
855
|
+
// CANON-B / ARCH-REFACTOR R1: trust the taskId the completion echoed. With R1's
|
|
856
|
+
// per-turn identity binding the worker stamps the COMPLETING turn's own taskId
|
|
857
|
+
// (not the racy session scalar), so the echoed id is authoritative and this is
|
|
858
|
+
// the path that should always be taken for an R1+ worker. The most-recent-by-
|
|
859
|
+
// session heuristic (resolveActiveDirectDispatchTaskId) is retained ONLY as a
|
|
860
|
+
// backward-compat fallback for legacy / version-skewed workers that carry no
|
|
861
|
+
// taskId — it is the very re-derive R1 exists to make unnecessary, and must not
|
|
862
|
+
// override a present echoed id, hence the `||` short-circuit order.
|
|
863
|
+
directDispatchTaskIdForLedger = readNonEmptyString(args.metadataEvent.taskId)
|
|
864
|
+
|| resolveActiveDirectDispatchTaskId(args.meshId, sessionId);
|
|
865
|
+
// A false-idle completion of a direct dispatch is recorded but kept tentative (the
|
|
866
|
+
// dispatch row stays active for the reconcile fallback); a genuine completion is terminal.
|
|
867
|
+
const isFalseIdle = isFalseIdleCompletion(args.metadataEvent);
|
|
868
|
+
completedTaskForLedger = markSessionTerminal(sessionId, 'completed', eventTimestamp, { tentativeIfDirect: isFalseIdle });
|
|
869
|
+
if (nodeId && providerType) {
|
|
870
|
+
// OVEREAGER-REMOTE-IDLE (Defect A+B): re-register the now-idle remote session into
|
|
871
|
+
// the remote-idle store, symmetric with the agent:ready branch. Previously
|
|
872
|
+
// setRemoteIdleSession ran ONLY on agent:ready, while agent:generating_started
|
|
873
|
+
// DELETES the entry — so the FIRST turn a remote worker runs permanently evicts it
|
|
874
|
+
// from the store, and generating_completed never re-added it. A later
|
|
875
|
+
// mesh_enqueue_task's triggerMeshQueue then read getRemoteIdleSessions() == 0
|
|
876
|
+
// (remoteIdleSessionsChecked:0) for a session that is genuinely live-idle, needlessly
|
|
877
|
+
// auto-launching a second worker (Defect A). Worse, the two idle-session sources then
|
|
878
|
+
// disagreed: the enqueue drain saw 0 (store empty) and auto-launched + left the queue
|
|
879
|
+
// task pending, while THIS completing session's runIdleMaintenanceThenAssignQueue
|
|
880
|
+
// claimed the same still-pending row straight from SQL — so the task body injected into
|
|
881
|
+
// BOTH the reused idle session and the auto-launched one (Defect B). Re-registering here
|
|
882
|
+
// unifies the source: the next triggerMeshQueue drain sees the live idle session, reuses
|
|
883
|
+
// it (claimed:true, no auto-launch), and injects exactly once. Skip a false-idle
|
|
884
|
+
// (mid-turn / no-final-assistant) completion — that session is NOT genuinely idle.
|
|
885
|
+
if (!isFalseIdle) {
|
|
886
|
+
sweepExpiredRemoteIdleSessions();
|
|
887
|
+
try {
|
|
888
|
+
MeshRuntimeStore.getInstance().setRemoteIdleSession(args.meshId, nodeId, sessionId, providerType, Date.now() + REMOTE_IDLE_SESSION_TTL_MS);
|
|
889
|
+
} catch { /* best-effort */ }
|
|
890
|
+
setImmediate(() => {
|
|
891
|
+
maybeAutoFastForwardIdleNode(components, { meshId: args.meshId, nodeId, sessionId, providerType })
|
|
892
|
+
.finally(() => {
|
|
893
|
+
try {
|
|
894
|
+
// Claim for THIS session first; on success drop the just-registered
|
|
895
|
+
// idle entry so the enqueue drain doesn't re-pick an already-busy session.
|
|
896
|
+
const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
|
|
897
|
+
if (assigned) MeshRuntimeStore.getInstance().deleteRemoteIdleSession(args.meshId, nodeId, sessionId);
|
|
898
|
+
} catch (e: any) {
|
|
899
|
+
LOG.warn('MeshQueue', `Failed to assign idle queue task after completion for ${nodeId}: ${e?.message || e}`);
|
|
900
|
+
}
|
|
901
|
+
});
|
|
902
|
+
});
|
|
903
|
+
} else {
|
|
904
|
+
runIdleMaintenanceThenAssignQueue(components, { meshId: args.meshId, nodeId, sessionId, providerType });
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
// M1-3: wake dependents of the completed task. The maintenance path above
|
|
908
|
+
// only assigns to the completing session; dependents may be claimable by
|
|
909
|
+
// other idle sessions, so run a full queue trigger when any are waiting.
|
|
910
|
+
const completedTaskId = completedTaskForLedger?.id;
|
|
911
|
+
if (completedTaskId && hasPendingDependents(args.meshId, completedTaskId)) {
|
|
912
|
+
setImmediate(() => {
|
|
913
|
+
triggerMeshQueue(components, args.meshId).catch((e: any) => {
|
|
914
|
+
LOG.warn('MeshQueue', `Dependent wake after task ${completedTaskId} failed: ${e?.message || e}`);
|
|
915
|
+
});
|
|
916
|
+
});
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
} else if (args.event === 'agent:ready') {
|
|
920
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
921
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
922
|
+
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
923
|
+
const providerSessionId = readNonEmptyString(args.metadataEvent.providerSessionId) || undefined;
|
|
924
|
+
const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary) || undefined;
|
|
925
|
+
const workerResult = readWorkerResultMetadata(args.metadataEvent);
|
|
926
|
+
const hasCompletionEvidence = !!finalSummary || !!workerResult;
|
|
927
|
+
if (sessionId && hasCompletionEvidence) {
|
|
928
|
+
completedTaskForLedger = markSessionTerminal(sessionId, 'completed');
|
|
929
|
+
if (completedTaskForLedger) {
|
|
930
|
+
try {
|
|
931
|
+
appendLedgerEntry(args.meshId, {
|
|
932
|
+
kind: 'task_completed',
|
|
933
|
+
nodeId: nodeId || undefined,
|
|
934
|
+
sessionId,
|
|
935
|
+
providerType: providerType || undefined,
|
|
936
|
+
payload: {
|
|
937
|
+
event: args.event,
|
|
938
|
+
nodeLabel: args.nodeLabel,
|
|
939
|
+
taskId: completedTaskForLedger.id,
|
|
940
|
+
completedViaReady: true,
|
|
941
|
+
providerSessionId,
|
|
942
|
+
finalSummary,
|
|
943
|
+
workerResult,
|
|
944
|
+
evidence: buildTaskCompletionEvidence({
|
|
945
|
+
event: 'agent:ready',
|
|
946
|
+
nodeId,
|
|
947
|
+
sessionId,
|
|
948
|
+
providerType: providerType || undefined,
|
|
949
|
+
providerSessionId,
|
|
950
|
+
finalSummary,
|
|
951
|
+
workerResult,
|
|
952
|
+
}),
|
|
953
|
+
},
|
|
954
|
+
});
|
|
955
|
+
} catch (e: any) {
|
|
956
|
+
LOG.warn('MeshLedger', `Failed to record task_completed from ready: ${e?.message || e}`);
|
|
957
|
+
}
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
|
|
961
|
+
if (sessionId && nodeId && providerType) {
|
|
962
|
+
// WORKTREE-BOOTSTRAP-DISPATCH-RACE: a freshly cloned worktree session emits
|
|
963
|
+
// agent:ready as soon as the CLI process reaches its idle prompt — which happens
|
|
964
|
+
// BEFORE the worktree bootstrap (npm install + native-addon repair: node-datachannel,
|
|
965
|
+
// better-sqlite3, ghostty-vt-node) has finished. Claiming a queued task on that early
|
|
966
|
+
// ready dispatches work into a session whose runtime is half-built: the child daemon
|
|
967
|
+
// dies loading the absent native addon → the worker session boots empty (no task ever
|
|
968
|
+
// injected), the queue row reports assigned-but-dead, and the work silently leaks to /
|
|
969
|
+
// gets re-routed onto the base node. Live evidence (the OPSRULES dispatch): node
|
|
970
|
+
// node_6df455dd emitted agent:ready at 10:36:28 but worktree_bootstrap_complete only at
|
|
971
|
+
// 10:36:56 — a 28s window in which a claim produced an empty session.
|
|
972
|
+
//
|
|
973
|
+
// Gate the claim on bootstrap state: if this node is a worktree whose bootstrap is still
|
|
974
|
+
// 'running', register the idle session (so it stays a claim candidate) but DEFER the
|
|
975
|
+
// claim. The claim re-fires on the next agent:ready / reconcile drain tick once bootstrap
|
|
976
|
+
// reaches a terminal state. 'failed' is left to flow through unchanged — a failed
|
|
977
|
+
// bootstrap surfaces its own coordinator event and a dispatch there fails loudly rather
|
|
978
|
+
// than silently, which is the correct, visible behavior (deferring forever would hide it).
|
|
979
|
+
let worktreeBootstrapPending = false;
|
|
980
|
+
try {
|
|
981
|
+
const mesh = getMeshWithCache(components, args.meshId);
|
|
982
|
+
const node = mesh?.nodes?.find((n: any) => meshNodeIdMatches(n, nodeId)) as { worktreeBootstrap?: { status?: string } } | undefined;
|
|
983
|
+
worktreeBootstrapPending = node?.worktreeBootstrap?.status === 'running';
|
|
984
|
+
} catch { /* best-effort: unknown bootstrap state → do not defer (prior behavior) */ }
|
|
985
|
+
|
|
986
|
+
sweepExpiredRemoteIdleSessions();
|
|
987
|
+
try {
|
|
988
|
+
MeshRuntimeStore.getInstance().setRemoteIdleSession(args.meshId, nodeId, sessionId, providerType, Date.now() + REMOTE_IDLE_SESSION_TTL_MS);
|
|
989
|
+
} catch { /* best-effort */ }
|
|
990
|
+
if (worktreeBootstrapPending) {
|
|
991
|
+
LOG.info('MeshQueue', `Deferring queue claim for worktree node ${nodeId} (${sessionId}): worktree bootstrap still running — early agent:ready precedes bootstrap completion; the idle session is registered and the claim will re-fire once bootstrap finishes (guards against dispatching into a half-built worktree → empty session / work leaking to the base node)`);
|
|
992
|
+
} else {
|
|
993
|
+
setImmediate(() => {
|
|
994
|
+
maybeAutoFastForwardIdleNode(components, { meshId: args.meshId, nodeId, sessionId, providerType })
|
|
995
|
+
.finally(() => {
|
|
996
|
+
try {
|
|
997
|
+
const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
|
|
998
|
+
if (assigned) MeshRuntimeStore.getInstance().deleteRemoteIdleSession(args.meshId, nodeId, sessionId);
|
|
999
|
+
} catch (e: any) {
|
|
1000
|
+
LOG.warn('MeshQueue', `Failed to assign idle queue task after maintenance for ${nodeId}: ${e?.message || e}`);
|
|
1001
|
+
}
|
|
1002
|
+
});
|
|
1003
|
+
});
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
} else if (args.event === 'agent:generating_started') {
|
|
1007
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
1008
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1009
|
+
if (sessionId && nodeId) {
|
|
1010
|
+
try {
|
|
1011
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(args.meshId, nodeId, sessionId);
|
|
1012
|
+
} catch { /* best-effort */ }
|
|
1013
|
+
}
|
|
1014
|
+
if (sessionId) {
|
|
1015
|
+
// CANON-B: a generating_started that echoes its taskId acks exactly the dispatch
|
|
1016
|
+
// and the delivery for THAT task — not every in-flight dispatch/delivery on the
|
|
1017
|
+
// session. A session that already holds a freshly-dispatched (still 'dispatched')
|
|
1018
|
+
// sibling must keep that row 'dispatched' so its own confirm can match it; acking
|
|
1019
|
+
// by session would mark it 'acked' prematurely and hide a genuine non-delivery.
|
|
1020
|
+
const startedTaskId = readNonEmptyString(args.metadataEvent.taskId) || undefined;
|
|
1021
|
+
// WARMUPGAP: only ack a dispatch row when the event names its task, or the session
|
|
1022
|
+
// currently holds an active assignment. A no-taskId generating_started from an
|
|
1023
|
+
// unassigned session is a pre-assignment warmup — the session_id fallback would ack a
|
|
1024
|
+
// sibling/stale dispatch row this event does not own, marking it 'acked' prematurely and
|
|
1025
|
+
// hiding a genuine non-delivery. Skip the dispatch ack for that ghost case (the delivery
|
|
1026
|
+
// acks below are bound to actual deliveries and stay a no-op for a warmup session).
|
|
1027
|
+
//
|
|
1028
|
+
// MESH-DISPATCH-MISROUTE (fix 3, consumer residual): when the event carries no taskId
|
|
1029
|
+
// (a legacy/relayed worker whose producer never stamped meshActiveTaskId) but the
|
|
1030
|
+
// session owns EXACTLY ONE active dispatch, resolve that row's taskId and flip it by PK
|
|
1031
|
+
// instead of the session_id sweep — the sweep flips every non-terminal row for the
|
|
1032
|
+
// session ("may flip a sibling dispatch row"). With ≥2 active rows the owner is
|
|
1033
|
+
// ambiguous, so resolvedAckTaskId stays undefined and we DROP the ack rather than
|
|
1034
|
+
// mis-flip a sibling (the genuine ack arrives once the producer/reconcile names a task).
|
|
1035
|
+
if (startedTaskId) {
|
|
1036
|
+
updateDirectDispatchStatus(args.meshId, sessionId, 'acked', startedTaskId);
|
|
1037
|
+
} else if (sessionHasActiveAssignment(args.meshId, sessionId)) {
|
|
1038
|
+
const soleTaskId = (() => {
|
|
1039
|
+
try { return MeshRuntimeStore.getInstance().getSoleActiveDirectDispatchTaskId(args.meshId, sessionId); }
|
|
1040
|
+
catch { return null; }
|
|
1041
|
+
})();
|
|
1042
|
+
if (soleTaskId) {
|
|
1043
|
+
updateDirectDispatchStatus(args.meshId, sessionId, 'acked', soleTaskId);
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
const activeDeliveries = ((): { id: string; taskId: string | null }[] => {
|
|
1047
|
+
try { return MeshRuntimeStore.getInstance().getActiveSessionDeliveries(args.meshId, sessionId); }
|
|
1048
|
+
catch { return []; }
|
|
1049
|
+
})();
|
|
1050
|
+
const deliveriesToAck = startedTaskId
|
|
1051
|
+
? activeDeliveries.filter(d => d.taskId === startedTaskId)
|
|
1052
|
+
: activeDeliveries;
|
|
1053
|
+
for (const d of deliveriesToAck) {
|
|
1054
|
+
updateSessionDeliveryStatus(d.id, 'acked');
|
|
1055
|
+
}
|
|
1056
|
+
}
|
|
1057
|
+
} else if (args.event === 'agent:stopped') {
|
|
1058
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
1059
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1060
|
+
if (sessionId && nodeId) {
|
|
1061
|
+
try {
|
|
1062
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(args.meshId, nodeId, sessionId);
|
|
1063
|
+
} catch { /* best-effort */ }
|
|
1064
|
+
}
|
|
1065
|
+
if (sessionId) {
|
|
1066
|
+
// CANON-B: prefer the echoed taskId; session heuristic is the fallback.
|
|
1067
|
+
directDispatchTaskIdForLedger = readNonEmptyString(args.metadataEvent.taskId)
|
|
1068
|
+
|| resolveActiveDirectDispatchTaskId(args.meshId, sessionId);
|
|
1069
|
+
completedTaskForLedger = markSessionTerminal(sessionId, 'failed');
|
|
1070
|
+
}
|
|
1071
|
+
} else if (args.event === 'worktree_bootstrap_complete' || args.event === 'worktree_bootstrap_failed') {
|
|
1072
|
+
// WORKTREE-BOOTSTRAP-REFIRE: the agent:ready branch above DEFERS the queue claim while
|
|
1073
|
+
// worktreeBootstrap.status === 'running' — a freshly cloned worktree emits agent:ready at
|
|
1074
|
+
// its idle prompt BEFORE bootstrap finishes, and dispatching then produces a half-built
|
|
1075
|
+
// session. The defer registers the idle session (setRemoteIdleSession) but schedules NO
|
|
1076
|
+
// retry; it relies on "the next agent:ready / reconcile drain tick" to re-fire. But
|
|
1077
|
+
// agent:ready is a one-shot (emitAgentReadyOnce, guarded by agentReadyEmitted) and the
|
|
1078
|
+
// session stays idle→idle after bootstrap, so no new agent:ready edge ever fires → the
|
|
1079
|
+
// deferred claim is stranded → the worker session boots empty (totalMessages=0) and the
|
|
1080
|
+
// coordinator relaunch/stop-loops. Re-fire the queue drain on the terminal bootstrap
|
|
1081
|
+
// transition: the idle session registered at defer time is now claimable (tryAssignQueueTask
|
|
1082
|
+
// has no bootstrap gate of its own). This runs on whichever daemon processes the event —
|
|
1083
|
+
// the local worker (when it co-hosts the coordinator) at emit time, or the remote
|
|
1084
|
+
// coordinator after it pulls the queued event — so the registered local/remote idle session
|
|
1085
|
+
// is drained in every topology. 'failed' is re-fired too so a deferred-then-failed bootstrap
|
|
1086
|
+
// dispatches and fails loudly/visibly rather than stranding silently. Falls through to the
|
|
1087
|
+
// coordinator broadcast below — the bootstrap event is still delivered to the coordinator.
|
|
1088
|
+
// WORKTREE-BOOTSTRAP-COORD-STATE: stamp the terminal bootstrap status onto the
|
|
1089
|
+
// COORDINATOR's mesh view BEFORE re-firing the queue. The clone+bootstrap ran on
|
|
1090
|
+
// the worker daemon (clone_mesh_node forwards to the source node's machine), so
|
|
1091
|
+
// persistWorktreeSetupState only flipped status→'complete' on the worker's mesh
|
|
1092
|
+
// object — the coordinator still holds the 'running' state it stamped from the
|
|
1093
|
+
// forwarded clone reply. Without this, the claim gate (agent:ready defer above +
|
|
1094
|
+
// mesh-queue-assignment) reads getMeshWithCache, sees 'running' forever, and
|
|
1095
|
+
// defers every claim — so this very re-fire loops against a gate that never opens
|
|
1096
|
+
// (claim never lands; idle session re-registered each tick; auto-launch spawns a
|
|
1097
|
+
// fresh session every cycle → runaway worktree-session multiplication). Stamping
|
|
1098
|
+
// the terminal state opens the gate so the deferred claim lands on this re-fire.
|
|
1099
|
+
const bootstrapNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1100
|
+
if (bootstrapNodeId) {
|
|
1101
|
+
try {
|
|
1102
|
+
// Fix (3): pass the worktree path so a hydrate-on-miss upsert (when this
|
|
1103
|
+
// coordinator never received the clone reply) can seed an addressable node.
|
|
1104
|
+
const bootstrapWorkspace = readNonEmptyString(args.metadataEvent.worktreePath)
|
|
1105
|
+
|| readNonEmptyString(args.metadataEvent.workspace);
|
|
1106
|
+
(components.router as any)?.markWorktreeBootstrapTerminalState?.(
|
|
1107
|
+
args.meshId,
|
|
1108
|
+
bootstrapNodeId,
|
|
1109
|
+
args.event === 'worktree_bootstrap_failed' ? 'failed' : 'complete',
|
|
1110
|
+
bootstrapWorkspace ? { workspace: bootstrapWorkspace } : undefined,
|
|
1111
|
+
);
|
|
1112
|
+
} catch (e: any) {
|
|
1113
|
+
LOG.warn('MeshQueue', `Failed to stamp terminal bootstrap state for ${bootstrapNodeId} (mesh ${args.meshId}): ${e?.message || e}`);
|
|
1114
|
+
}
|
|
1115
|
+
}
|
|
1116
|
+
// BOOTSTRAP-MSG: detect whether the queue re-fire below has a task to auto-claim for
|
|
1117
|
+
// this node BEFORE setImmediate(triggerMeshQueue) runs — at this point a deferred task
|
|
1118
|
+
// is still 'pending' (it has not been claimed yet); 'assigned' covers a task already
|
|
1119
|
+
// claimed by an earlier tick. Either case means a session is being/has been spun up by
|
|
1120
|
+
// the queue, so the completion message must NOT advise a manual mesh_launch_session.
|
|
1121
|
+
// Only meaningful for the 'complete' transition (a failed bootstrap claims nothing).
|
|
1122
|
+
if (args.event === 'worktree_bootstrap_complete' && bootstrapNodeId) {
|
|
1123
|
+
try {
|
|
1124
|
+
worktreeHasQueuedTask = getQueue(args.meshId, { status: ['pending', 'assigned'] })
|
|
1125
|
+
.some((task) => meshNodeIdMatches({ id: task.targetNodeId } as MeshNodeIdentified, bootstrapNodeId));
|
|
1126
|
+
} catch (e: any) {
|
|
1127
|
+
LOG.warn('MeshQueue', `Failed to check queued task for ${bootstrapNodeId} (mesh ${args.meshId}): ${e?.message || e}`);
|
|
1128
|
+
}
|
|
1129
|
+
}
|
|
1130
|
+
setImmediate(() => {
|
|
1131
|
+
triggerMeshQueue(components, args.meshId).catch((e: any) => {
|
|
1132
|
+
LOG.warn('MeshQueue', `Queue re-fire after ${args.event} failed (mesh ${args.meshId}): ${e?.message || e}`);
|
|
1133
|
+
});
|
|
1134
|
+
});
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
const ledgerKind = EVENT_TO_LEDGER_KIND[args.event];
|
|
1138
|
+
if (ledgerKind) {
|
|
1139
|
+
try {
|
|
1140
|
+
const ledgerNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined;
|
|
1141
|
+
const ledgerSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined;
|
|
1142
|
+
const ledgerProviderType = readNonEmptyString(args.metadataEvent.providerType) || undefined;
|
|
1143
|
+
const providerSessionId = readNonEmptyString(args.metadataEvent.providerSessionId) || undefined;
|
|
1144
|
+
const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary) || undefined;
|
|
1145
|
+
const workerResult = readWorkerResultMetadata(args.metadataEvent);
|
|
1146
|
+
const completionEvidence = ledgerKind === 'task_completed' && ledgerNodeId && ledgerSessionId
|
|
1147
|
+
? buildTaskCompletionEvidence({
|
|
1148
|
+
event: 'agent:generating_completed',
|
|
1149
|
+
nodeId: ledgerNodeId,
|
|
1150
|
+
sessionId: ledgerSessionId,
|
|
1151
|
+
providerType: ledgerProviderType,
|
|
1152
|
+
providerSessionId,
|
|
1153
|
+
finalSummary,
|
|
1154
|
+
workerResult,
|
|
1155
|
+
})
|
|
1156
|
+
: undefined;
|
|
1157
|
+
appendLedgerEntry(args.meshId, {
|
|
1158
|
+
kind: ledgerKind,
|
|
1159
|
+
nodeId: ledgerNodeId,
|
|
1160
|
+
sessionId: ledgerSessionId,
|
|
1161
|
+
providerType: ledgerProviderType,
|
|
1162
|
+
payload: {
|
|
1163
|
+
event: args.event,
|
|
1164
|
+
nodeLabel: args.nodeLabel,
|
|
1165
|
+
// Fix B: fall back to the direct-dispatch taskId when no work-queue row
|
|
1166
|
+
// matched, so the terminal entry is attributable in mesh task-stats
|
|
1167
|
+
// (otherwise the direct task shows status='unknown' / terminalKind=null).
|
|
1168
|
+
taskId: completedTaskForLedger?.id || directDispatchTaskIdForLedger || undefined,
|
|
1169
|
+
providerSessionId,
|
|
1170
|
+
finalSummary,
|
|
1171
|
+
workerResult,
|
|
1172
|
+
completionDiagnostic: args.metadataEvent.completionDiagnostic && typeof args.metadataEvent.completionDiagnostic === 'object'
|
|
1173
|
+
? args.metadataEvent.completionDiagnostic
|
|
1174
|
+
: undefined,
|
|
1175
|
+
evidence: completionEvidence,
|
|
1176
|
+
// B2: evidenceLevel lets coordinator know when completion evidence is insufficient.
|
|
1177
|
+
// NOTIF Defect-2b: ONLY source==='default' (no parseable answer at all) is
|
|
1178
|
+
// 'insufficient'. 'parseable_answer' (a real JSON answer that just isn't
|
|
1179
|
+
// worker-result-shaped, e.g. a MAGI envelope) is concrete evidence and must
|
|
1180
|
+
// resolve to 'sufficient' — resolveWorkerResult now upgrades that case so a
|
|
1181
|
+
// complete, valid answer is no longer mislabelled insufficient/reviewRecommended.
|
|
1182
|
+
...(completionEvidence
|
|
1183
|
+
? completionEvidence.workerResult.source === 'default'
|
|
1184
|
+
? { evidenceLevel: 'insufficient', reviewRecommended: true }
|
|
1185
|
+
: { evidenceLevel: 'sufficient' }
|
|
1186
|
+
: {}),
|
|
1187
|
+
},
|
|
1188
|
+
});
|
|
1189
|
+
} catch (e: any) {
|
|
1190
|
+
LOG.warn('MeshLedger', `Failed to record ${ledgerKind}: ${e?.message || e}`);
|
|
1191
|
+
}
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
let recoveryContext: SessionRecoveryContext | null = null;
|
|
1195
|
+
if (args.event === 'agent:stopped') {
|
|
1196
|
+
try {
|
|
1197
|
+
const mesh = getMesh(args.meshId);
|
|
1198
|
+
const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
|
|
1199
|
+
|
|
1200
|
+
recoveryContext = getSessionRecoveryContext(args.meshId, {
|
|
1201
|
+
sessionId: resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined,
|
|
1202
|
+
nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
|
|
1203
|
+
maxRetries,
|
|
1204
|
+
});
|
|
1205
|
+
recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
|
|
1206
|
+
|
|
1207
|
+
if (recoveryContext.retryRecommended && recoveryContext.consecutiveNodeFailures > 0) {
|
|
1208
|
+
appendLedgerEntry(args.meshId, {
|
|
1209
|
+
kind: 'recovery_attempted',
|
|
1210
|
+
nodeId: recoveryContext.failedNodeId || undefined,
|
|
1211
|
+
sessionId: recoveryContext.failedSessionId || undefined,
|
|
1212
|
+
providerType: recoveryContext.failedProviderType || undefined,
|
|
1213
|
+
payload: {
|
|
1214
|
+
consecutiveFailures: recoveryContext.consecutiveNodeFailures,
|
|
1215
|
+
taskAttemptCount: recoveryContext.taskAttemptCount,
|
|
1216
|
+
retryRecommended: recoveryContext.retryRecommended,
|
|
1217
|
+
advice: recoveryContext.advice,
|
|
1218
|
+
},
|
|
1219
|
+
});
|
|
1220
|
+
|
|
1221
|
+
if (recoveryContext.lastTaskMessage && recoveryContext.failedNodeId && recoveryContext.failedProviderType) {
|
|
1222
|
+
const autoNodeId = recoveryContext.failedNodeId;
|
|
1223
|
+
try {
|
|
1224
|
+
const task = enqueueTask(args.meshId, recoveryContext.lastTaskMessage, {
|
|
1225
|
+
targetNodeId: autoNodeId
|
|
1226
|
+
});
|
|
1227
|
+
LOG.info('MeshRecovery', `Auto-requeued failed task: ${task.id} for node ${autoNodeId}`);
|
|
1228
|
+
|
|
1229
|
+
const node = mesh?.nodes.find((n: any) => meshNodeIdMatches(n, autoNodeId));
|
|
1230
|
+
if (node) {
|
|
1231
|
+
components.cliManager.handleCliCommand('launch_cli', {
|
|
1232
|
+
cliType: recoveryContext.failedProviderType,
|
|
1233
|
+
dir: node.workspace,
|
|
1234
|
+
settings: {
|
|
1235
|
+
role: 'worker',
|
|
1236
|
+
meshNodeFor: args.meshId,
|
|
1237
|
+
meshNodeId: node.id,
|
|
1238
|
+
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
1239
|
+
// Coordinator-dispatched recovery relaunch: same auto-approve
|
|
1240
|
+
// policy as the primary worker launch path.
|
|
1241
|
+
autoApprove: resolveDelegatedWorkerAutoApprove(mesh?.policy, node?.policy),
|
|
1242
|
+
launchedByCoordinator: true,
|
|
1243
|
+
}
|
|
1244
|
+
}).catch((e: any) => LOG.error('MeshRecovery', `Failed to auto-relaunch session for ${node.id}: ${e?.message}`));
|
|
1245
|
+
}
|
|
1246
|
+
} catch (e: any) {
|
|
1247
|
+
LOG.warn('MeshRecovery', `Failed to execute auto-recovery: ${e?.message}`);
|
|
1248
|
+
}
|
|
1249
|
+
}
|
|
1250
|
+
}
|
|
1251
|
+
|
|
1252
|
+
LOG.info('MeshRecovery', `Recovery context for ${args.nodeLabel}: ${recoveryContext.advice}`);
|
|
1253
|
+
} catch (e: any) {
|
|
1254
|
+
LOG.warn('MeshRecovery', `Failed to build recovery context: ${e?.message || e}`);
|
|
1255
|
+
}
|
|
1256
|
+
}
|
|
1257
|
+
|
|
1258
|
+
const messageText = buildMeshSystemMessage({
|
|
1259
|
+
event: args.event,
|
|
1260
|
+
nodeLabel: args.nodeLabel,
|
|
1261
|
+
metadataEvent: args.metadataEvent,
|
|
1262
|
+
recoveryContext,
|
|
1263
|
+
worktreeHasQueuedTask,
|
|
1264
|
+
});
|
|
1265
|
+
if (!messageText) {
|
|
1266
|
+
// Lifecycle events that carry no coordinator-facing message (agent:ready /
|
|
1267
|
+
// agent:generating_started) still drive the remote-claim state machine: the
|
|
1268
|
+
// coordinator's agent:ready branch above runs setRemoteIdleSession +
|
|
1269
|
+
// tryAssignQueueTask, and agent:generating_started clears the remote-idle entry.
|
|
1270
|
+
// For a LOCAL worker whose coordinator is a REMOTE daemon those side effects ran
|
|
1271
|
+
// on the wrong daemon (this worker's empty queue / store), so the coordinator never
|
|
1272
|
+
// learns the auto-launched session went idle and re-auto-launches it forever
|
|
1273
|
+
// (queue task stuck pending). Queue the silent event so the coordinator pulls it
|
|
1274
|
+
// (PHASE 1 pullRemoteNodeQueues → handleMeshForwardEvent) and re-runs the claim on
|
|
1275
|
+
// the daemon that actually owns the queue. Gate strictly on a present, REMOTE
|
|
1276
|
+
// coordinator daemon id: a co-located worker already ran the claim on the right
|
|
1277
|
+
// daemon, and a coordinator processing a *pulled* event has no sourceSession so
|
|
1278
|
+
// workerCoordinatorDaemonId is empty — neither re-queues, so there is no loop.
|
|
1279
|
+
const isSilentClaimRelevantEvent = args.event === 'agent:ready' || args.event === 'agent:generating_started';
|
|
1280
|
+
const coordinatorIsRemote = !!workerCoordinatorDaemonId
|
|
1281
|
+
&& !resolveCoordinatorDrainDaemonIds(components).includes(workerCoordinatorDaemonId);
|
|
1282
|
+
if (!(isSilentClaimRelevantEvent && coordinatorIsRemote)) {
|
|
1283
|
+
return { success: false, error: 'unsupported mesh event' };
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
// ── Queue-only delivery (single-model: queue + periodic poll) ──────────────
|
|
1288
|
+
// Every mesh coordinator event — terminal or not, local-coordinator or
|
|
1289
|
+
// remote — is persisted to the pending-events queue (SQLite + JSONL) and
|
|
1290
|
+
// NOTHING is pushed here. The old spontaneous-forward paths were removed:
|
|
1291
|
+
// - F1 remote P2P `mesh_forward_event` dispatch (network/stamp-dependent,
|
|
1292
|
+
// silently dropped on P2P failure or missing meshCoordinatorDaemonId)
|
|
1293
|
+
// - F3 live-CLI PTY `send_message` fire-and-forget inject (silently
|
|
1294
|
+
// dropped when the coordinator was generating)
|
|
1295
|
+
// Delivery to a live CLI coordinator now happens via setupMeshReconcileLoop,
|
|
1296
|
+
// which drains this queue on a fixed interval and injects into the coordinator
|
|
1297
|
+
// only when it is idle. A pure stdio MCP (LLM) coordinator — which has no live
|
|
1298
|
+
// CLI session to inject into — drains the queue itself when it calls a mesh
|
|
1299
|
+
// tool (mesh_status / mesh_read_chat). Either way the queue is the single
|
|
1300
|
+
// source of truth and the only thing this function writes to.
|
|
1301
|
+
//
|
|
1302
|
+
// targetCoordinatorDaemonId scopes the event to a specific coordinator daemon
|
|
1303
|
+
// (unicast) when the worker carries one, so the reconcile loop on the right
|
|
1304
|
+
// daemon drains it and other daemons skip it. Absent → broadcast/backfill.
|
|
1305
|
+
const pendingEvent = {
|
|
1306
|
+
event: args.event,
|
|
1307
|
+
meshId: args.meshId,
|
|
1308
|
+
nodeLabel: args.nodeLabel,
|
|
1309
|
+
nodeId: args.nodeId || undefined,
|
|
1310
|
+
workspace: readNonEmptyString(args.metadataEvent.workspace)
|
|
1311
|
+
|| readNonEmptyString(args.metadataEvent.workspaceName),
|
|
1312
|
+
metadataEvent: {
|
|
1313
|
+
...enrichedMetadataEvent,
|
|
1314
|
+
...(recoveryContext ? { recoveryContext } : {}),
|
|
1315
|
+
// Stash the coordinator session id INSIDE metadataEvent too, so it survives the
|
|
1316
|
+
// P2P relay serialization (buildForwardPayloadFromPending spreads metadata; the
|
|
1317
|
+
// handleMeshForwardEvent whitelist reads it back) — a top-level field alone would
|
|
1318
|
+
// be dropped when the event crosses a machine boundary.
|
|
1319
|
+
...(workerCoordinatorSessionId ? { meshCoordinatorSessionId: workerCoordinatorSessionId } : {}),
|
|
1320
|
+
},
|
|
1321
|
+
// Silent lifecycle events (agent:ready / agent:generating_started) carry no
|
|
1322
|
+
// coordinator message; they are queued only so the coordinator re-runs the
|
|
1323
|
+
// remote-claim state machine on pull. injectPendingIntoCoordinator skips
|
|
1324
|
+
// entries without a coordinatorMessage, so a live CLI coordinator is not spammed.
|
|
1325
|
+
...(messageText ? { coordinatorMessage: messageText } : {}),
|
|
1326
|
+
queuedAt: Date.now(),
|
|
1327
|
+
...(workerCoordinatorDaemonId ? { targetCoordinatorDaemonId: workerCoordinatorDaemonId } : {}),
|
|
1328
|
+
// Top-level session anchor for the local PHASE 2 strict-match on the coordinator
|
|
1329
|
+
// daemon. Absent → daemon-level broadcast (legacy / single-coordinator path).
|
|
1330
|
+
...(workerCoordinatorSessionId ? { targetCoordinatorSessionId: workerCoordinatorSessionId } : {}),
|
|
1331
|
+
};
|
|
1332
|
+
if (queuePendingMeshCoordinatorEvent(pendingEvent)) {
|
|
1333
|
+
LOG.info('MeshEvents', `Queued ${args.event} for coordinator (mesh ${args.meshId}${workerCoordinatorDaemonId ? `, coordinator daemon ${workerCoordinatorDaemonId}` : ''}${workerCoordinatorSessionId ? `, coordinator session ${workerCoordinatorSessionId}` : ''})`);
|
|
1334
|
+
// EVTTRACE: event persisted to the coordinator pending queue (awaiting reconcile drain).
|
|
1335
|
+
traceMeshEventStage('queued', traceCtx, workerCoordinatorDaemonId ? `coordinatorDaemon=${workerCoordinatorDaemonId}` : 'broadcast');
|
|
1336
|
+
} else {
|
|
1337
|
+
// EVTTRACE: queue rejected the event (dedup at queue time / persistence guard).
|
|
1338
|
+
traceMeshEventDrop('queue_dedup', traceCtx);
|
|
1339
|
+
}
|
|
1340
|
+
return { success: true, forwarded: 0 };
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1343
|
+
// Reconstruct the metadataEvent that injectMeshSystemMessage consumes from a forwarded
|
|
1344
|
+
// (cross-machine) mesh event. The remote relay hop arrives as a flat payload, NOT the
|
|
1345
|
+
// original provider event object, so this whitelists the fields the coordinator-side
|
|
1346
|
+
// pipeline reads and re-projects them. Kept pure + exported so the relay-path field
|
|
1347
|
+
// preservation (esp. taskId) is unit-testable without driving injectMeshSystemMessage.
|
|
1348
|
+
//
|
|
1349
|
+
// IMPORTANT asymmetry: the LOCAL in-process forward path (onMeshCoordinatorEventForwarded)
|
|
1350
|
+
// passes the whole event through as metadataEvent, so every field on the event survives
|
|
1351
|
+
// there for free. This remote-only path must explicitly mirror each field it needs.
|
|
1352
|
+
export function buildRelayMetadataEvent(payload: Record<string, unknown>): Record<string, unknown> {
|
|
1353
|
+
const relayModalMessage = readNonEmptyString(payload.modalMessage);
|
|
1354
|
+
const relayModalButtons = Array.isArray(payload.modalButtons)
|
|
1355
|
+
? (payload.modalButtons as unknown[]).filter((b): b is string => typeof b === 'string' && b.trim().length > 0)
|
|
1356
|
+
: null;
|
|
1357
|
+
return {
|
|
1358
|
+
// Preserve the dispatch task id across the machine boundary. The `received` trace
|
|
1359
|
+
// stage reads payload.taskId; without mirroring it here the rebuilt metadataEvent
|
|
1360
|
+
// loses it, so injectMeshSystemMessage's traceCtx.taskId and the
|
|
1361
|
+
// updateDirectDispatchStatus(eventTaskId) call go undefined — the EvtTrace
|
|
1362
|
+
// queued/surfaced stages show task=- and the direct-dispatch ledger falls back to a
|
|
1363
|
+
// session_id match (which can flip a sibling row). The local in-process forward path
|
|
1364
|
+
// keeps event.taskId/meshActiveTaskId for free; this mirrors it for the remote relay.
|
|
1365
|
+
// Same taskId/meshActiveTaskId ordering the local unroutable trace uses.
|
|
1366
|
+
taskId: readNonEmptyString(payload.taskId) || readNonEmptyString(payload.meshActiveTaskId),
|
|
1367
|
+
targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.instanceId),
|
|
1368
|
+
providerType: readNonEmptyString(payload.providerType),
|
|
1369
|
+
providerSessionId: readNonEmptyString(payload.providerSessionId),
|
|
1370
|
+
// Preserve the originating coordinator SESSION id across the machine boundary so
|
|
1371
|
+
// the completion routes back to the exact coordinator session (multi-coordinator).
|
|
1372
|
+
// buildForwardPayloadFromPending spreads the worker event's metadata, so the id
|
|
1373
|
+
// arrives as payload.meshCoordinatorSessionId; the top-level targetCoordinatorSessionId
|
|
1374
|
+
// is also accepted as a fallback. injectMeshSystemMessage re-derives the routing
|
|
1375
|
+
// anchors from this. Absent → daemon-level fallback (version-skew safe).
|
|
1376
|
+
meshCoordinatorSessionId: readNonEmptyString(payload.meshCoordinatorSessionId) || readNonEmptyString(payload.targetCoordinatorSessionId),
|
|
1377
|
+
// Carry the session identity fields the worker provider event emits so the
|
|
1378
|
+
// coordinator's mirror (updateMeshOwnedSession) gets a real workspace/title/
|
|
1379
|
+
// settings. Without these the remote-relay hop reconstructs metadataEvent with
|
|
1380
|
+
// an empty workspace, and the dashboard flaps to the generic
|
|
1381
|
+
// "Terminal (Mesh Node)" title (and degrades the provider label) between live
|
|
1382
|
+
// events and the periodic get_status_metadata snapshot. The local in-process
|
|
1383
|
+
// forward path (onMeshCoordinatorEventForwarded) already preserves these; this
|
|
1384
|
+
// mirrors them for the remote-only relay path.
|
|
1385
|
+
workspace: readNonEmptyString(payload.workspace) || readNonEmptyString(payload.workspaceName),
|
|
1386
|
+
workspaceName: readNonEmptyString(payload.workspaceName) || readNonEmptyString(payload.workspace),
|
|
1387
|
+
sessionTitle: readNonEmptyString(payload.sessionTitle),
|
|
1388
|
+
sessionStatus: readNonEmptyString(payload.sessionStatus),
|
|
1389
|
+
sessionChatStatus: readNonEmptyString(payload.sessionChatStatus),
|
|
1390
|
+
providerName: readNonEmptyString(payload.providerName),
|
|
1391
|
+
...(payload.sessionSettings && typeof payload.sessionSettings === 'object' && !Array.isArray(payload.sessionSettings) ? { sessionSettings: payload.sessionSettings } : {}),
|
|
1392
|
+
finalSummary: readNonEmptyString(payload.finalSummary) || readNonEmptyString(payload.summary),
|
|
1393
|
+
// T2: carry the worker's status-snapshot last-message preview across the machine
|
|
1394
|
+
// boundary so a summary-less completion still surfaces the assistant reply in the
|
|
1395
|
+
// coordinator's inbox mirror. resolveMeshSurfacedSessionPreview reads these
|
|
1396
|
+
// (assistant-role only) when finalSummary is absent.
|
|
1397
|
+
lastMessagePreview: readNonEmptyString(payload.lastMessagePreview),
|
|
1398
|
+
lastMessageRole: readNonEmptyString(payload.lastMessageRole),
|
|
1399
|
+
...(payload.lastMessageAt !== undefined ? { lastMessageAt: payload.lastMessageAt } : {}),
|
|
1400
|
+
jobId: readNonEmptyString(payload.jobId),
|
|
1401
|
+
interactionId: readNonEmptyString(payload.interactionId),
|
|
1402
|
+
status: readNonEmptyString(payload.status),
|
|
1403
|
+
targetDaemonId: readNonEmptyString(payload.targetDaemonId),
|
|
1404
|
+
startedAt: readNonEmptyString(payload.startedAt),
|
|
1405
|
+
completedAt: readNonEmptyString(payload.completedAt),
|
|
1406
|
+
retryOfJobId: readNonEmptyString(payload.retryOfJobId),
|
|
1407
|
+
...(relayModalMessage ? { modalMessage: relayModalMessage } : {}),
|
|
1408
|
+
...(relayModalButtons && relayModalButtons.length > 0 ? { modalButtons: relayModalButtons } : {}),
|
|
1409
|
+
...(payload.result && typeof payload.result === 'object' && !Array.isArray(payload.result) ? { result: payload.result } : {}),
|
|
1410
|
+
...(payload.completionDiagnostic && typeof payload.completionDiagnostic === 'object' && !Array.isArray(payload.completionDiagnostic) ? { completionDiagnostic: payload.completionDiagnostic } : {}),
|
|
1411
|
+
...(payload.workerResult && typeof payload.workerResult === 'object' && !Array.isArray(payload.workerResult) ? { workerResult: payload.workerResult } : {}),
|
|
1412
|
+
...(payload.meshWorkerResult && typeof payload.meshWorkerResult === 'object' && !Array.isArray(payload.meshWorkerResult) ? { meshWorkerResult: payload.meshWorkerResult } : {}),
|
|
1413
|
+
...(payload.structuredResult && typeof payload.structuredResult === 'object' && !Array.isArray(payload.structuredResult) ? { structuredResult: payload.structuredResult } : {}),
|
|
1414
|
+
...(payload.timestamp !== undefined ? { timestamp: payload.timestamp } : {}),
|
|
1415
|
+
intentional: payload.intentional === true,
|
|
1416
|
+
intentionalStop: payload.intentionalStop === true,
|
|
1417
|
+
operatorCleanup: payload.operatorCleanup === true,
|
|
1418
|
+
reason: readNonEmptyString(payload.reason),
|
|
1419
|
+
stopReason: readNonEmptyString(payload.stopReason),
|
|
1420
|
+
cleanupReason: readNonEmptyString(payload.cleanupReason),
|
|
1421
|
+
source: readNonEmptyString(payload.source),
|
|
1422
|
+
};
|
|
1423
|
+
}
|
|
1424
|
+
|
|
1425
|
+
export function handleMeshForwardEvent(components: DaemonComponents, payload: Record<string, unknown>) {
|
|
1426
|
+
const eventName = readNonEmptyString(payload.event);
|
|
1427
|
+
if (!isMeshCoordinatorEvent(eventName)) {
|
|
1428
|
+
return { success: false, error: 'unsupported mesh event' };
|
|
1429
|
+
}
|
|
1430
|
+
const nodeId = readNonEmptyString(payload.nodeId);
|
|
1431
|
+
const workspace = readNonEmptyString(payload.workspace);
|
|
1432
|
+
|
|
1433
|
+
// The fallback worker-forward path (forwardUnresolvedDelegateEvent) cannot resolve a
|
|
1434
|
+
// mesh id locally on the remote worker, so it forwards the event with nodeId +
|
|
1435
|
+
// workspace only. The coordinator hosting the mesh CAN resolve it. Two recovery
|
|
1436
|
+
// paths, in order:
|
|
1437
|
+
// 1) workspace → mesh (fast path; cached repoIdentity lookup), then
|
|
1438
|
+
// 2) nodeId → mesh (deterministic backstop; scans hosted meshes for the node).
|
|
1439
|
+
// Workspace recovery alone was unreliable — a worktree clone whose repoIdentity
|
|
1440
|
+
// differs, or a transient cache miss, left the reconcile retry permanently rejected
|
|
1441
|
+
// ("meshId required") so the worker's completion never surfaced to the coordinator.
|
|
1442
|
+
// The nodeId is a stable coordinator-side fact and resolves timing-independently.
|
|
1443
|
+
const meshId = readNonEmptyString(payload.meshId)
|
|
1444
|
+
|| (workspace ? readNonEmptyString(getCachedMeshByWorkspace(workspace)?.id) : '')
|
|
1445
|
+
|| recoverMeshIdByNodeId(nodeId)
|
|
1446
|
+
// Fix B last resort: workspace + nodeId scan both missed, but the worker stamped
|
|
1447
|
+
// its coordinator anchor onto the forward (forwardUnresolvedDelegateEvent). Recover
|
|
1448
|
+
// via the hosted mesh that anchor owns (+ node disambiguation), guarding ambiguity.
|
|
1449
|
+
|| recoverMeshIdByCoordinatorAndNode(
|
|
1450
|
+
readNonEmptyString(payload.meshCoordinatorDaemonId) || readNonEmptyString(payload.coordinatorDaemonId),
|
|
1451
|
+
nodeId,
|
|
1452
|
+
);
|
|
1453
|
+
if (!meshId) {
|
|
1454
|
+
// EVTTRACE: forwarded event rejected at receive — no meshId could be resolved
|
|
1455
|
+
// (no payload.meshId, no workspace→mesh, no nodeId→mesh). Observation only.
|
|
1456
|
+
traceMeshEventDrop('meshId_required', {
|
|
1457
|
+
taskId: payload.taskId,
|
|
1458
|
+
sessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId),
|
|
1459
|
+
nodeId,
|
|
1460
|
+
event: eventName,
|
|
1461
|
+
}, workspace ? `workspace=${workspace} unresolved` : 'no workspace/nodeId');
|
|
1462
|
+
return { success: false, error: 'meshId required' };
|
|
1463
|
+
}
|
|
1464
|
+
// EVTTRACE: forwarded event accepted at receive (meshId resolved).
|
|
1465
|
+
// NOTIF-MISS (FIX 3): mirror buildRelayMetadataEvent's taskId fallback. A worker provider
|
|
1466
|
+
// completion stamps its task id as `meshActiveTaskId` (settings → event), not always the
|
|
1467
|
+
// top-level `taskId`, so reading payload.taskId alone rendered `task=-` at the received stage
|
|
1468
|
+
// (the [stage:queued] task=<id> → [stage:received] task=- loss). Falling back to
|
|
1469
|
+
// meshActiveTaskId keeps the trace task-scoped end-to-end so same-task redelivery is
|
|
1470
|
+
// distinguishable from a different task's real completion.
|
|
1471
|
+
traceMeshEventStage('received', {
|
|
1472
|
+
taskId: readNonEmptyString(payload.taskId) || readNonEmptyString(payload.meshActiveTaskId),
|
|
1473
|
+
sessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId),
|
|
1474
|
+
nodeId,
|
|
1475
|
+
meshId,
|
|
1476
|
+
event: eventName,
|
|
1477
|
+
});
|
|
1478
|
+
const nodeLabel = nodeId ? `Node '${nodeId}'` : workspace ? `Agent at ${workspace}` : 'Remote agent';
|
|
1479
|
+
|
|
1480
|
+
return injectMeshSystemMessage(components, {
|
|
1481
|
+
meshId,
|
|
1482
|
+
nodeId,
|
|
1483
|
+
nodeLabel,
|
|
1484
|
+
event: eventName,
|
|
1485
|
+
metadataEvent: buildRelayMetadataEvent(payload),
|
|
1486
|
+
});
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1489
|
+
// ---------------------------------------------------------------------------
|
|
1490
|
+
// Per-coordinator forward serialization (P2P send-backpressure relief).
|
|
1491
|
+
//
|
|
1492
|
+
// When several workers finish at once, each completion runs forwardUnresolvedDelegate
|
|
1493
|
+
// Event and fires its own `mesh_forward_event` push. Firing the whole burst
|
|
1494
|
+
// concurrently dumps it into the single per-peer P2P DataChannel buffer in one tick,
|
|
1495
|
+
// which starves the rpc_ack/rpc_res replies the same channel must carry — a
|
|
1496
|
+
// coordinator's inbound `git_status` then times out even though the worker's own
|
|
1497
|
+
// forward acks return in ~1s. To cap the concurrent burst we serialize the immediate
|
|
1498
|
+
// pushes per coordinator: at most one push is in flight to a given coordinator at a
|
|
1499
|
+
// time, the rest run in arrival order behind it. A lone event (idle lane) still
|
|
1500
|
+
// dispatches immediately — only a genuine burst is paced. Durability is unchanged:
|
|
1501
|
+
// every event is already persisted to the outbox before the push runs, so serializing
|
|
1502
|
+
// only delays the best-effort fast path; PHASE 0 retry still covers any gap. This pairs
|
|
1503
|
+
// with the DataChannel send-buffer gate in daemon-cloud's mesh manager (writeRequest),
|
|
1504
|
+
// which is the hard guarantee; this throttle keeps the burst from piling up there.
|
|
1505
|
+
interface CoordinatorForwardLane { tail: Promise<unknown>; depth: number; }
|
|
1506
|
+
const coordinatorForwardLanes = new Map<string, CoordinatorForwardLane>();
|
|
1507
|
+
function enqueueCoordinatorForwardPush(coordinatorDaemonId: string, run: () => Promise<unknown>): void {
|
|
1508
|
+
let lane = coordinatorForwardLanes.get(coordinatorDaemonId);
|
|
1509
|
+
if (!lane) { lane = { tail: Promise.resolve(), depth: 0 }; coordinatorForwardLanes.set(coordinatorDaemonId, lane); }
|
|
1510
|
+
const wasIdle = lane.depth === 0;
|
|
1511
|
+
lane.depth += 1;
|
|
1512
|
+
const dec = (): void => { lane!.depth -= 1; };
|
|
1513
|
+
if (wasIdle) {
|
|
1514
|
+
// Idle lane → dispatch synchronously, so a lone completion (the common case) has
|
|
1515
|
+
// ZERO added latency and the push call happens in-line. Only a genuine burst —
|
|
1516
|
+
// events arriving while a push is still in flight — is paced (else branch).
|
|
1517
|
+
lane.tail = Promise.resolve(run()).catch(() => {}).then(dec, dec);
|
|
1518
|
+
} else {
|
|
1519
|
+
// Burst: queue behind the in-flight push(es) in arrival order so the whole burst
|
|
1520
|
+
// is not dumped into the shared DataChannel buffer at once. The tail is guarded
|
|
1521
|
+
// so one rejecting push never wedges the lane for the next.
|
|
1522
|
+
lane.tail = lane.tail.then(() => run()).catch(() => {}).then(dec, dec);
|
|
1523
|
+
}
|
|
1524
|
+
}
|
|
1525
|
+
|
|
1526
|
+
// ---------------------------------------------------------------------------
|
|
1527
|
+
// Worker-side fallback forward for unresolved-mesh delegates.
|
|
1528
|
+
//
|
|
1529
|
+
// A REMOTE worker daemon that is being P2P-remote-controlled by a coordinator is
|
|
1530
|
+
// NOT a member of the coordinator's mesh — it has no local mesh record. So when its
|
|
1531
|
+
// completion event reaches the forwarder, resolveWorkerDelegateRouting() resolves the
|
|
1532
|
+
// coordinator anchor (meshCoordinatorDaemonId) from the worker envelope but cannot
|
|
1533
|
+
// resolve the mesh id (neither meshNodeFor nor a workspace→mesh lookup yields one) and
|
|
1534
|
+
// returns isDelegate=false / mesh_unresolved. Before this fallback the event was dropped
|
|
1535
|
+
// (delivery_unroutable) and only recovered later when the coordinator happened to pull
|
|
1536
|
+
// the worker's queue — which it can't, because the worker never queued an unroutable
|
|
1537
|
+
// event. Live symptom: `WARN [MeshEvents] delivery_unroutable: ... mesh unresolved`.
|
|
1538
|
+
//
|
|
1539
|
+
// The fix: the routing object still carries coordinatorDaemonId. Forward the raw event
|
|
1540
|
+
// straight to that coordinator daemon over P2P (mesh_forward_event). The coordinator
|
|
1541
|
+
// hosts the mesh, so it recovers the mesh id by workspace in handleMeshForwardEvent and
|
|
1542
|
+
// injects/queues it normally. meshId is intentionally omitted from the payload (the
|
|
1543
|
+
// worker has none); workspace is the routing anchor the coordinator resolves from.
|
|
1544
|
+
//
|
|
1545
|
+
// No loop / no double-delivery:
|
|
1546
|
+
// - This only fires on the WORKER (the coordinator-own session is rejected by the
|
|
1547
|
+
// resolver before reaching here), and the coordinator merely injects — it does not
|
|
1548
|
+
// re-enter this forwarder for the relayed event.
|
|
1549
|
+
// - It fires only when the normal queue path did NOT run (isDelegate=false), so the
|
|
1550
|
+
// event is never both queued locally and forwarded.
|
|
1551
|
+
//
|
|
1552
|
+
// Returns true when the event was durably accepted for delivery to the coordinator
|
|
1553
|
+
// daemon (so the caller skips the delivery_unroutable diagnostic); false when no
|
|
1554
|
+
// fallback was possible (no coordinator anchor / no dispatch transport).
|
|
1555
|
+
//
|
|
1556
|
+
// Durability: the directed push to the coordinator is the ONLY delivery route for an
|
|
1557
|
+
// unresolved-mesh worker (it is in no mesh.node the coordinator can pull). So instead
|
|
1558
|
+
// of a fire-and-forget push that drops on one transient P2P failure, the event is
|
|
1559
|
+
// persisted to the worker-side outbox FIRST and only acked after a successful push.
|
|
1560
|
+
// A best-effort immediate push keeps latency low on the happy path; a failed or
|
|
1561
|
+
// un-acked push leaves the durable row for setupMeshReconcileLoop's PHASE 0 to retry.
|
|
1562
|
+
function forwardUnresolvedDelegateEvent(
|
|
1563
|
+
components: DaemonComponents,
|
|
1564
|
+
routing: ReturnType<typeof resolveWorkerDelegateRouting>,
|
|
1565
|
+
event: Record<string, unknown>,
|
|
1566
|
+
): boolean {
|
|
1567
|
+
const coordinatorDaemonId = readNonEmptyString(routing.coordinatorDaemonId);
|
|
1568
|
+
if (!coordinatorDaemonId) return false;
|
|
1569
|
+
if (!components.dispatchMeshCommand) return false;
|
|
1570
|
+
|
|
1571
|
+
const eventName = readNonEmptyString(event.event);
|
|
1572
|
+
if (!eventName) return false;
|
|
1573
|
+
|
|
1574
|
+
// Flat payload mirroring buildForwardPayloadFromPending / what handleMeshForwardEvent
|
|
1575
|
+
// reads. nodeId/workspace come from the worker envelope so the coordinator can name and
|
|
1576
|
+
// locate the node.
|
|
1577
|
+
const payload: Record<string, unknown> = {
|
|
1578
|
+
...event,
|
|
1579
|
+
event: eventName,
|
|
1580
|
+
nodeId: readNonEmptyString(routing.nodeId) || readNonEmptyString(event.meshNodeId) || undefined,
|
|
1581
|
+
workspace: readNonEmptyString(routing.workspace) || readNonEmptyString(event.workspace) || undefined,
|
|
1582
|
+
// Fix B: carry the resolved coordinator anchor so the coordinator's receive-side
|
|
1583
|
+
// recovery (recoverMeshIdByCoordinatorAndNode) can match this forward to one of the
|
|
1584
|
+
// meshes it hosts when workspace + nodeId both miss. routing.coordinatorDaemonId is
|
|
1585
|
+
// the same anchor this forward is addressed to (coordinatorDaemonId below).
|
|
1586
|
+
meshCoordinatorDaemonId: coordinatorDaemonId,
|
|
1587
|
+
};
|
|
1588
|
+
// RECONCILE-MESHID-DROP: stamp meshId when the WORKER can resolve it (member node /
|
|
1589
|
+
// live-session meshNodeFor). Historically omitted "because the worker can't resolve
|
|
1590
|
+
// it", but for a member-hosted node a no_node_binding session's coordinator-side
|
|
1591
|
+
// recovery (empty payload nodeId + workspace cache miss) fails and the retry is
|
|
1592
|
+
// rejected "meshId required" forever. Resolving here makes the forward self-sufficient;
|
|
1593
|
+
// when unresolvable even here it stays absent and the coordinator's own workspace/nodeId
|
|
1594
|
+
// recovery still runs (unchanged), with the retry cap as the loop backstop.
|
|
1595
|
+
const resolvedMeshId = resolveForwardEventMeshId(components, payload);
|
|
1596
|
+
if (resolvedMeshId) payload.meshId = resolvedMeshId;
|
|
1597
|
+
|
|
1598
|
+
// Self-addressed fallback: the resolved coordinator IS this daemon (a self-
|
|
1599
|
+
// coordinating / single-node mesh, or a delegate whose coordinator anchor resolved
|
|
1600
|
+
// to our own id). A cross-daemon mesh_forward_event to our own id is REFUSED by the
|
|
1601
|
+
// dispatch self-dial guard ("route via the local router instead"), so persisting it
|
|
1602
|
+
// to the outbox would only loop forever in PHASE 0's retry, never acked. Honour the
|
|
1603
|
+
// guard's advice: route the event straight through the local receiver — the exact
|
|
1604
|
+
// path the coordinator runs on receiving a remote push — and skip the outbox entirely.
|
|
1605
|
+
const selfDaemonIds = resolveCoordinatorDrainDaemonIds(components);
|
|
1606
|
+
if (selfDaemonIds.some(self => daemonIdsEquivalent(self, coordinatorDaemonId))) {
|
|
1607
|
+
try {
|
|
1608
|
+
handleMeshForwardEvent(components, payload);
|
|
1609
|
+
LOG.info('MeshEvents', `Self-addressed unresolved-delegate ${eventName} routed via local router (coordinator ${coordinatorDaemonId} is self) — outbox skipped`);
|
|
1610
|
+
} catch (e: any) {
|
|
1611
|
+
LOG.warn('MeshEvents', `Local route of self-addressed unresolved-delegate ${eventName} failed: ${e?.message || e}`);
|
|
1612
|
+
}
|
|
1613
|
+
return true;
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
// 1) Persist durably FIRST. Idempotent on fingerprint, so a re-fired completion
|
|
1617
|
+
// does not duplicate the outbox row. If persistence fails we still attempt the
|
|
1618
|
+
// push below (degrades to the old at-most-once behaviour rather than dropping
|
|
1619
|
+
// the chance entirely).
|
|
1620
|
+
const persisted = enqueueUnresolvedDelegateForward(coordinatorDaemonId, eventName, payload);
|
|
1621
|
+
// EVTTRACE: unresolved-mesh worker persisted its completion to the outbox (no meshId
|
|
1622
|
+
// available locally; coordinator will recover it on receive).
|
|
1623
|
+
const fwdTraceCtx = {
|
|
1624
|
+
taskId: (payload as Record<string, unknown>).taskId,
|
|
1625
|
+
sessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId),
|
|
1626
|
+
nodeId: readNonEmptyString(routing.nodeId) || readNonEmptyString(event.meshNodeId),
|
|
1627
|
+
event: eventName,
|
|
1628
|
+
};
|
|
1629
|
+
traceMeshEventStage('outbox_enqueue', fwdTraceCtx, `coordinatorDaemon=${coordinatorDaemonId} meshId=absent`);
|
|
1630
|
+
|
|
1631
|
+
// 2) Best-effort immediate push for low latency. On success, ack the outbox row so
|
|
1632
|
+
// the retry loop won't re-send it. On failure, leave it queued — PHASE 0 retries.
|
|
1633
|
+
traceMeshEventStage('forward_send', fwdTraceCtx, 'immediate push');
|
|
1634
|
+
// Serialize per coordinator so a multi-worker completion burst is paced rather than
|
|
1635
|
+
// dumped concurrently into the shared P2P DataChannel buffer (see coordinator
|
|
1636
|
+
// ForwardLanes). dispatchMeshCommand was null-checked above; capture it for the
|
|
1637
|
+
// deferred closure.
|
|
1638
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
1639
|
+
enqueueCoordinatorForwardPush(coordinatorDaemonId, () =>
|
|
1640
|
+
Promise.resolve(dispatchMeshCommand(coordinatorDaemonId, 'mesh_forward_event', payload))
|
|
1641
|
+
.then((result: any) => {
|
|
1642
|
+
if (result && result.success === false) {
|
|
1643
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued for retry`);
|
|
1644
|
+
traceMeshEventDrop('immediate_forward_rejected', fwdTraceCtx, readNonEmptyString(result.error) || 'no reason');
|
|
1645
|
+
return;
|
|
1646
|
+
}
|
|
1647
|
+
// Acked. Mark the durable copy delivered so the retry loop skips it.
|
|
1648
|
+
if (persisted) ackUnresolvedDelegateForwardByFingerprint(coordinatorDaemonId, eventName, payload);
|
|
1649
|
+
})
|
|
1650
|
+
.catch((e: any) => {
|
|
1651
|
+
// Coordinator momentarily unreachable; the durable row stays queued and the
|
|
1652
|
+
// reconcile loop retries it. Trace so the relay attempt is visible.
|
|
1653
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} failed: ${e?.message || e} — left queued for retry`);
|
|
1654
|
+
}));
|
|
1655
|
+
LOG.info('MeshEvents', `Durably forwarded ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId}`);
|
|
1656
|
+
return true;
|
|
1657
|
+
}
|
|
1658
|
+
|
|
1659
|
+
// Ack a just-pushed outbox entry by re-deriving its row from the same coordinator +
|
|
1660
|
+
// event + payload. We don't thread the row id back from enqueue (the immediate push is
|
|
1661
|
+
// fire-then-ack), so locate it among the undrained entries by matching coordinator and
|
|
1662
|
+
// the flat payload's forward identity. A miss is harmless — the retry loop's own
|
|
1663
|
+
// receiver-side dedup suppresses a duplicate delivery.
|
|
1664
|
+
function ackUnresolvedDelegateForwardByFingerprint(
|
|
1665
|
+
coordinatorDaemonId: string,
|
|
1666
|
+
eventName: string,
|
|
1667
|
+
payload: Record<string, unknown>,
|
|
1668
|
+
): void {
|
|
1669
|
+
const match = peekUnresolvedDelegateForwards().find(entry =>
|
|
1670
|
+
daemonIdsEquivalent(entry.coordinatorDaemonId, coordinatorDaemonId)
|
|
1671
|
+
&& readNonEmptyString(entry.payload.event) === eventName
|
|
1672
|
+
&& readNonEmptyString(entry.payload.targetSessionId || entry.payload.sessionId || entry.payload.instanceId)
|
|
1673
|
+
=== readNonEmptyString(payload.targetSessionId || payload.sessionId || payload.instanceId)
|
|
1674
|
+
&& readNonEmptyString(entry.payload.workspace) === readNonEmptyString(payload.workspace),
|
|
1675
|
+
);
|
|
1676
|
+
if (match) ackUnresolvedDelegateForward(match.id);
|
|
1677
|
+
}
|
|
1678
|
+
|
|
1679
|
+
/**
|
|
1680
|
+
* NOTIF-HELD-DRAIN (Fix 2): event-driven coordinator drain. The reconcile loop delivers a
|
|
1681
|
+
* worker's queued completion to an IDLE local coordinator only on its periodic poll. When a
|
|
1682
|
+
* coordinator is sitting idle awaiting exactly that completion, waiting up to a full poll
|
|
1683
|
+
* interval is the avoidable delivery latency the RCA flags — and combined with the (now-fixed)
|
|
1684
|
+
* modal-park false-positive it stretched into the multi-minute notification stall. So the
|
|
1685
|
+
* MOMENT a worker delegate event is persisted for a mesh, attempt the same idle-coordinator
|
|
1686
|
+
* drain immediately, mirroring the event-driven worker-claim path (agent:ready /
|
|
1687
|
+
* agent:generating_completed → triggerMeshQueue).
|
|
1688
|
+
*
|
|
1689
|
+
* Safety:
|
|
1690
|
+
* - drainPendingMeshCoordinatorEvents marks rows drained=1 atomically, so this races the
|
|
1691
|
+
* reconcile poll and the coordinator's own idle auto-flush harmlessly — exactly one consumes
|
|
1692
|
+
* each row.
|
|
1693
|
+
* - Only IDLE, non-modal-parked coordinators are delivery targets (never a generating /
|
|
1694
|
+
* consent-modal PTY).
|
|
1695
|
+
* - Strict session routing is honoured: an event naming an originating coordinator session is
|
|
1696
|
+
* delivered only to that live idle session; anything not currently deliverable here
|
|
1697
|
+
* (wrong/absent session, or a message-less lifecycle event) is RE-QUEUED — never dropped —
|
|
1698
|
+
* so the reconcile loop's strict hold/expire path remains the single authority for it.
|
|
1699
|
+
*/
|
|
1700
|
+
export function flushPendingForMeshIdleCoordinators(components: DaemonComponents, meshId: string): void {
|
|
1701
|
+
// O(1) gate: skip the (relatively expensive) per-instance getState scan when the queue is
|
|
1702
|
+
// empty for this mesh.
|
|
1703
|
+
try {
|
|
1704
|
+
const store = MeshRuntimeStore.getInstance();
|
|
1705
|
+
if (store.pendingEventCount(meshId) === 0) return;
|
|
1706
|
+
} catch { /* store unavailable — fall through and let the drain decide */ }
|
|
1707
|
+
|
|
1708
|
+
const idleCoordinators: { instance: ProviderInstance; sessionId: string }[] = [];
|
|
1709
|
+
try {
|
|
1710
|
+
for (const inst of components.instanceManager.getByCategory('cli')) {
|
|
1711
|
+
const state = inst.getState();
|
|
1712
|
+
const settings = state.settings && typeof state.settings === 'object'
|
|
1713
|
+
? state.settings as Record<string, unknown>
|
|
1714
|
+
: {};
|
|
1715
|
+
if (readNonEmptyString(settings.meshCoordinatorFor) !== meshId) continue;
|
|
1716
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
1717
|
+
const modalParked = typeof (inst as any).isModalParked === 'function'
|
|
1718
|
+
? (inst as any).isModalParked() === true
|
|
1719
|
+
: (status === 'waiting_choice' || status === 'waiting_approval');
|
|
1720
|
+
// PTY-OVERTRUST-DRAIN (Defect B): decide idle on the RAW adapter turn-state
|
|
1721
|
+
// (getDrainStatus, mask-stripped) to match the reconcile loop — getState().status
|
|
1722
|
+
// overlays the auto-approve hold-idle mask that paints a genuinely-idle coordinator
|
|
1723
|
+
// `generating`, which would make this opportunistic flush skip a real drain target.
|
|
1724
|
+
// Fall back to the masked literal for any instance without getDrainStatus().
|
|
1725
|
+
const drainStatus: string | null = typeof (inst as any).getDrainStatus === 'function'
|
|
1726
|
+
? (inst as any).getDrainStatus()
|
|
1727
|
+
: null;
|
|
1728
|
+
const idle = drainStatus !== null ? drainStatus === 'idle' : (status === 'idle');
|
|
1729
|
+
if (idle && !modalParked) {
|
|
1730
|
+
idleCoordinators.push({ instance: inst, sessionId: readNonEmptyString(state.instanceId) });
|
|
1731
|
+
}
|
|
1732
|
+
}
|
|
1733
|
+
} catch { return; }
|
|
1734
|
+
if (idleCoordinators.length === 0) return; // no idle target now → leave for the reconcile poll
|
|
1735
|
+
|
|
1736
|
+
const drainDaemonIds = resolveCoordinatorDrainDaemonIds(components);
|
|
1737
|
+
let pendingEvents: PendingMeshCoordinatorEvent[];
|
|
1738
|
+
try {
|
|
1739
|
+
pendingEvents = drainPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
|
|
1740
|
+
} catch (e: any) {
|
|
1741
|
+
LOG.warn('MeshEvents', `Event-driven coordinator drain failed for mesh ${meshId}: ${e?.message || e}`);
|
|
1742
|
+
return;
|
|
1743
|
+
}
|
|
1744
|
+
if (pendingEvents.length === 0) return;
|
|
1745
|
+
|
|
1746
|
+
let delivered = 0;
|
|
1747
|
+
for (const pending of pendingEvents) {
|
|
1748
|
+
const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
|
|
1749
|
+
const targets = wantSession
|
|
1750
|
+
? idleCoordinators.filter(c => c.sessionId === wantSession)
|
|
1751
|
+
: idleCoordinators;
|
|
1752
|
+
// Not deliverable into an idle target here (wrong/absent session), or a message-less
|
|
1753
|
+
// lifecycle event (agent:ready / generating_started carry no coordinatorMessage and
|
|
1754
|
+
// must not be injected): re-queue so the reconcile loop owns it (lazy-synth / strict
|
|
1755
|
+
// hold/expire). Re-queue preserves queuedAt so the strict TTL measures true age.
|
|
1756
|
+
if (targets.length === 0 || !pending.coordinatorMessage) {
|
|
1757
|
+
try { queuePendingMeshCoordinatorEvent(pending); } catch { /* best-effort re-queue */ }
|
|
1758
|
+
continue;
|
|
1759
|
+
}
|
|
1760
|
+
const message = pending.coordinatorMessage;
|
|
1761
|
+
const force = shouldForceInjectMeshEvent(pending.event);
|
|
1762
|
+
for (const c of targets) {
|
|
1763
|
+
c.instance.onEvent('send_message', {
|
|
1764
|
+
input: { text: message, textFallback: message },
|
|
1765
|
+
...(force ? { force: true } : {}),
|
|
1766
|
+
});
|
|
1767
|
+
delivered++;
|
|
1768
|
+
}
|
|
1769
|
+
}
|
|
1770
|
+
if (delivered > 0) {
|
|
1771
|
+
LOG.info('MeshEvents', `Event-driven drain delivered ${delivered} pending event(s) to ${idleCoordinators.length} idle coordinator(s) for mesh ${meshId}`);
|
|
1772
|
+
}
|
|
1773
|
+
}
|
|
1774
|
+
|
|
1775
|
+
export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
1776
|
+
components.instanceManager.onEvent((event) => {
|
|
1777
|
+
// --- Coordinator idle auto-flush (fast path) ---
|
|
1778
|
+
// When a coordinator session becomes idle, immediately flush any pending
|
|
1779
|
+
// coordinator events that accumulated while it was generating, rather than
|
|
1780
|
+
// waiting up to one reconcile interval for setupMeshReconcileLoop to do it.
|
|
1781
|
+
// Both paths drain the SAME queue via drainPendingMeshCoordinatorEvents,
|
|
1782
|
+
// whose SQLite drained=1 marking is atomic — whichever fires first consumes
|
|
1783
|
+
// the events and the other gets nothing, so there is no double-delivery.
|
|
1784
|
+
// This runs before the delegate routing below so that coordinator-own idle
|
|
1785
|
+
// transitions are handled first.
|
|
1786
|
+
// Exception: a coordinator that is itself a direct-dispatch target still needs
|
|
1787
|
+
// to go through delegate routing so that the dispatching coordinator receives a
|
|
1788
|
+
// pendingCoordinatorEvents entry for the completion.
|
|
1789
|
+
if (event.event === 'agent:ready' || event.event === 'agent:generating_completed') {
|
|
1790
|
+
const flushInstanceId = readNonEmptyString(event.instanceId);
|
|
1791
|
+
if (flushInstanceId) {
|
|
1792
|
+
const flushSource = components.instanceManager.getInstance(flushInstanceId);
|
|
1793
|
+
if (flushSource && flushSource.category === 'cli') {
|
|
1794
|
+
const flushState = flushSource.getState();
|
|
1795
|
+
const flushSettings = flushState.settings && typeof flushState.settings === 'object' ? flushState.settings as Record<string, unknown> : {};
|
|
1796
|
+
const coordinatorMeshId = readNonEmptyString(flushSettings.meshCoordinatorFor);
|
|
1797
|
+
if (coordinatorMeshId) {
|
|
1798
|
+
const status = readNonEmptyString(flushState.status).toLowerCase();
|
|
1799
|
+
if (status === 'idle') {
|
|
1800
|
+
try {
|
|
1801
|
+
// Drain with the daemon's full coordinator-id set (status id + machineId).
|
|
1802
|
+
// The MCP layer stamps the prefixed status id (`standalone_<machineId>` /
|
|
1803
|
+
// `daemon_<machineId>`) as the worker's meshCoordinatorDaemonId; draining
|
|
1804
|
+
// with bare machineId alone would miss those unicast events. Mirrors
|
|
1805
|
+
// resolveCoordinatorDaemonIds in mesh-reconcile-loop.
|
|
1806
|
+
const drainDaemonIds = resolveCoordinatorDrainDaemonIds(components);
|
|
1807
|
+
const pendingEvents = drainPendingMeshCoordinatorEvents(coordinatorMeshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
|
|
1808
|
+
if (pendingEvents.length > 0) {
|
|
1809
|
+
LOG.info('MeshEvents', `Auto-flushing ${pendingEvents.length} pending coordinator event(s) for mesh ${coordinatorMeshId} on coordinator idle`);
|
|
1810
|
+
for (const pending of pendingEvents) {
|
|
1811
|
+
if (!pending.coordinatorMessage) continue;
|
|
1812
|
+
const forcePending = shouldForceInjectMeshEvent(pending.event);
|
|
1813
|
+
flushSource.onEvent('send_message', {
|
|
1814
|
+
input: { text: pending.coordinatorMessage, textFallback: pending.coordinatorMessage },
|
|
1815
|
+
...(forcePending ? { force: true } : {}),
|
|
1816
|
+
});
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
} catch (e: any) {
|
|
1820
|
+
LOG.warn('MeshEvents', `Failed to auto-flush pending coordinator events: ${e?.message || e}`);
|
|
1821
|
+
}
|
|
1822
|
+
}
|
|
1823
|
+
// Skip delegate routing unless this coordinator session is itself
|
|
1824
|
+
// a direct-dispatch target — in that case fall through so the
|
|
1825
|
+
// dispatching coordinator gets a pendingCoordinatorEvents entry.
|
|
1826
|
+
let hasDirectDispatch = false;
|
|
1827
|
+
try {
|
|
1828
|
+
hasDirectDispatch =
|
|
1829
|
+
getActiveDirectDispatches(coordinatorMeshId).some(d => d.sessionId === flushInstanceId)
|
|
1830
|
+
|| hasUnterminalDirectDispatchLedgerEntry(coordinatorMeshId, flushInstanceId);
|
|
1831
|
+
} catch { /* best-effort */ }
|
|
1832
|
+
if (!hasDirectDispatch) return;
|
|
1833
|
+
}
|
|
1834
|
+
}
|
|
1835
|
+
}
|
|
1836
|
+
}
|
|
1837
|
+
|
|
1838
|
+
// --- Delegate event routing ---
|
|
1839
|
+
if (!isMeshCoordinatorEvent(event.event)) return;
|
|
1840
|
+
|
|
1841
|
+
const instanceId = readNonEmptyString(event.instanceId);
|
|
1842
|
+
if (!instanceId) return;
|
|
1843
|
+
|
|
1844
|
+
// R1: all session→node→mesh→coordinator interpretation is folded into the single
|
|
1845
|
+
// resolveWorkerDelegateRouting() resolver. No stamp (meshNodeFor / meshNodeId /
|
|
1846
|
+
// meshCoordinatorDaemonId / meshCoordinatorNodeId / launchedByCoordinator) is read
|
|
1847
|
+
// here to make a routing decision — the resolver is the one authority, and the
|
|
1848
|
+
// forwarder consumes its typed result only.
|
|
1849
|
+
const routing = resolveWorkerDelegateRouting(components, instanceId, {
|
|
1850
|
+
getMeshById: (meshId) => getMeshWithCache(components, meshId),
|
|
1851
|
+
getMeshByWorkspace: (workspace) => getCachedMeshByWorkspace(workspace),
|
|
1852
|
+
});
|
|
1853
|
+
if (!routing.isDelegate) {
|
|
1854
|
+
// Fallback: a REMOTE worker that isn't a member of the coordinator's mesh can't
|
|
1855
|
+
// resolve a mesh id locally (mesh_unresolved), but it still carries the coordinator
|
|
1856
|
+
// daemon anchor. Forward the event straight to that coordinator over P2P instead of
|
|
1857
|
+
// dropping it — the coordinator hosts the mesh and recovers the id by workspace.
|
|
1858
|
+
if (isUnroutableDelegateRejection(routing)
|
|
1859
|
+
&& forwardUnresolvedDelegateEvent(components, routing, event)) {
|
|
1860
|
+
return;
|
|
1861
|
+
}
|
|
1862
|
+
// R4: a worker that presented a valid envelope but resolved to no mesh (and could
|
|
1863
|
+
// not be fallback-forwarded — e.g. no coordinator anchor) used to be dropped
|
|
1864
|
+
// silently. Leave a fail-loud diagnostic so the missing completion is traceable.
|
|
1865
|
+
// Benign non-delegate rejections (not_cli / no_workspace / etc.) are no-ops inside
|
|
1866
|
+
// recordUnroutableDelegateEvent.
|
|
1867
|
+
// EVTTRACE: a delegate event that could not be routed AND could not be
|
|
1868
|
+
// fallback-forwarded (no coordinator anchor). Only mesh_unresolved is a real
|
|
1869
|
+
// drop; the benign non-delegate rejections are ordinary non-mesh traffic.
|
|
1870
|
+
if (isUnroutableDelegateRejection(routing)) {
|
|
1871
|
+
traceMeshEventDrop('unroutable', {
|
|
1872
|
+
taskId: (event as Record<string, unknown>).meshActiveTaskId ?? (event as Record<string, unknown>).taskId,
|
|
1873
|
+
sessionId: routing.sessionId,
|
|
1874
|
+
nodeId: routing.nodeId,
|
|
1875
|
+
event: event.event,
|
|
1876
|
+
}, 'no coordinator anchor / mesh_unresolved');
|
|
1877
|
+
}
|
|
1878
|
+
recordUnroutableDelegateEvent(routing, event.event);
|
|
1879
|
+
return;
|
|
1880
|
+
}
|
|
1881
|
+
|
|
1882
|
+
injectMeshSystemMessage(components, {
|
|
1883
|
+
meshId: routing.meshId,
|
|
1884
|
+
sourceInstanceId: instanceId,
|
|
1885
|
+
nodeId: routing.nodeId,
|
|
1886
|
+
nodeLabel: routing.nodeLabel,
|
|
1887
|
+
event: event.event,
|
|
1888
|
+
metadataEvent: event,
|
|
1889
|
+
});
|
|
1890
|
+
|
|
1891
|
+
// NOTIF-HELD-DRAIN (Fix 2): the worker's event is now persisted in the pending queue.
|
|
1892
|
+
// If a local coordinator for this mesh is sitting idle awaiting it, deliver immediately
|
|
1893
|
+
// instead of waiting up to a full reconcile interval (event-driven, mirrors the
|
|
1894
|
+
// worker-claim path). No-op when no idle coordinator is present (held for reconcile).
|
|
1895
|
+
flushPendingForMeshIdleCoordinators(components, routing.meshId);
|
|
1896
|
+
});
|
|
1897
|
+
}
|