@adhdev/daemon-core 0.9.82-rc.38 → 0.9.82-rc.381
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/daemon-lifecycle.d.ts +9 -0
- package/dist/boot/process-hardening.d.ts +50 -0
- package/dist/build-info.d.ts +37 -0
- package/dist/chat/source-machine.d.ts +166 -0
- package/dist/chat/source-resolver.d.ts +104 -0
- package/dist/chat/subscription-updates.d.ts +1 -0
- package/dist/cli-adapter-types.d.ts +55 -2
- package/dist/cli-adapters/cli-script-runner.d.ts +117 -0
- package/dist/cli-adapters/cli-state-engine.d.ts +198 -0
- package/dist/cli-adapters/provider-cli-adapter.d.ts +121 -64
- package/dist/cli-adapters/provider-cli-parse.d.ts +4 -0
- package/dist/cli-adapters/provider-cli-runtime.d.ts +34 -0
- package/dist/cli-adapters/provider-cli-shared.d.ts +63 -0
- package/dist/cli-adapters/pty-write-chunking.d.ts +34 -0
- package/dist/cli-adapters/raw-terminal-io.d.ts +37 -0
- package/dist/cli-adapters/resolve-executable.d.ts +14 -0
- package/dist/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +6 -3
- package/dist/cli-adapters/terminal-backends/types.d.ts +10 -2
- package/dist/cli-adapters/terminal-screen.d.ts +4 -11
- package/dist/commands/chat-commands-debug-bundle.d.ts +14 -0
- package/dist/commands/chat-commands-read.d.ts +7 -0
- package/dist/commands/chat-commands-scope.d.ts +39 -0
- package/dist/commands/chat-commands-shared.d.ts +33 -0
- package/dist/commands/chat-commands-write.d.ts +14 -0
- package/dist/commands/chat-commands.d.ts +10 -23
- package/dist/commands/cli-manager.d.ts +15 -1
- package/dist/commands/handler.d.ts +125 -0
- package/dist/commands/high-family/index.d.ts +3 -0
- package/dist/commands/high-family/mesh-coordinator-launch.d.ts +2 -0
- package/dist/commands/high-family/mesh-events.d.ts +2 -0
- package/dist/commands/high-family/mesh-status.d.ts +2 -0
- package/dist/commands/high-family/types.d.ts +60 -0
- package/dist/commands/low-family/coordinator-prompt.d.ts +9 -0
- package/dist/commands/low-family/daemon-lifecycle.d.ts +2 -0
- package/dist/commands/low-family/diagnostics.d.ts +2 -0
- package/dist/commands/low-family/index.d.ts +3 -0
- package/dist/commands/low-family/mesh-ledger.d.ts +10 -0
- package/dist/commands/low-family/mesh-node-logs.d.ts +2 -0
- package/dist/commands/low-family/notification.d.ts +2 -0
- package/dist/commands/low-family/refine-config.d.ts +2 -0
- package/dist/commands/low-family/session-host.d.ts +2 -0
- package/dist/commands/low-family/spec-providerdev.d.ts +11 -0
- package/dist/commands/low-family/status-meta.d.ts +2 -0
- package/dist/commands/low-family/types.d.ts +33 -0
- package/dist/commands/med-family/cli-agent.d.ts +2 -0
- package/dist/commands/med-family/fast-forward.d.ts +2 -0
- package/dist/commands/med-family/ide.d.ts +10 -0
- package/dist/commands/med-family/index.d.ts +3 -0
- package/dist/commands/med-family/mesh-crud.d.ts +2 -0
- package/dist/commands/med-family/mesh-host-pairing.d.ts +2 -0
- package/dist/commands/med-family/mesh-queue.d.ts +2 -0
- package/dist/commands/med-family/mesh-restart.d.ts +2 -0
- package/dist/commands/med-family/types.d.ts +116 -0
- package/dist/commands/mesh-coordinator.d.ts +85 -1
- package/dist/commands/router.d.ts +227 -2
- package/dist/commands/upgrade-helper.d.ts +41 -1
- package/dist/config/chat-history.d.ts +9 -0
- package/dist/config/config.d.ts +5 -0
- package/dist/config/mesh-config.d.ts +77 -1
- package/dist/daemon/dev-server.d.ts +0 -2
- package/dist/detection/ide-detector.d.ts +13 -0
- package/dist/detection/win32-ide-version.d.ts +37 -0
- package/dist/git/change-impact-config.d.ts +159 -0
- package/dist/git/git-commands.d.ts +11 -1
- package/dist/git/git-diff.d.ts +6 -0
- package/dist/git/git-executor.d.ts +11 -0
- package/dist/git/git-status.d.ts +23 -0
- package/dist/git/git-types.d.ts +2 -50
- package/dist/git/index.d.ts +2 -0
- package/dist/index.d.ts +57 -13
- package/dist/index.js +47417 -19860
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +45865 -18449
- package/dist/index.mjs.map +1 -1
- package/dist/installer.d.ts +1 -4
- package/dist/ipc/local-ipc-server.d.ts +91 -0
- package/dist/launch.d.ts +1 -1
- package/dist/logging/async-batch-writer.d.ts +10 -0
- package/dist/logging/log-redactor.d.ts +24 -0
- package/dist/logging/log-tail-reader.d.ts +81 -0
- package/dist/logging/logger.d.ts +1 -1
- package/dist/mesh/contracts.d.ts +164 -0
- package/dist/mesh/coordinator-prompt.d.ts +89 -0
- package/dist/mesh/coordinator-registry.d.ts +59 -0
- package/dist/mesh/mesh-active-work.d.ts +174 -0
- package/dist/mesh/mesh-coordinator-config.d.ts +21 -0
- package/dist/mesh/mesh-delivery-policy.d.ts +131 -0
- package/dist/mesh/mesh-event-classify.d.ts +5 -0
- package/dist/mesh/mesh-event-forwarding.d.ts +18 -0
- package/dist/mesh/mesh-event-trace.d.ts +21 -0
- package/dist/mesh/mesh-events-coordinator.d.ts +4 -0
- package/dist/mesh/mesh-events-pending.d.ts +55 -0
- package/dist/mesh/mesh-events-stale.d.ts +52 -0
- package/dist/mesh/mesh-events-utils.d.ts +64 -0
- package/dist/mesh/mesh-events.d.ts +6 -49
- package/dist/mesh/mesh-fast-forward.d.ts +81 -0
- package/dist/mesh/mesh-host-ownership.d.ts +9 -0
- package/dist/mesh/mesh-init.d.ts +86 -0
- package/dist/mesh/mesh-ledger-reconciliation.d.ts +23 -4
- package/dist/mesh/mesh-ledger.d.ts +77 -1
- package/dist/mesh/mesh-missions.d.ts +164 -0
- package/dist/mesh/mesh-node-identity.d.ts +289 -0
- package/dist/mesh/mesh-queue-assignment.d.ts +86 -0
- package/dist/mesh/mesh-reconcile-loop.d.ts +8 -0
- package/dist/mesh/mesh-refine-batch.d.ts +68 -0
- package/dist/mesh/mesh-refine-gates.d.ts +428 -0
- package/dist/mesh/mesh-refine-status.d.ts +62 -0
- package/dist/mesh/mesh-review-inbox.d.ts +90 -0
- package/dist/mesh/mesh-routing.d.ts +70 -0
- package/dist/mesh/mesh-runtime-store.d.ts +439 -0
- package/dist/mesh/mesh-scheduling-runtime.d.ts +78 -0
- package/dist/mesh/mesh-task-stats.d.ts +49 -0
- package/dist/mesh/mesh-unresolved-forward-outbox.d.ts +30 -0
- package/dist/mesh/mesh-warmup-deadline.d.ts +68 -0
- package/dist/mesh/mesh-work-queue.d.ts +247 -5
- package/dist/mesh/preview-freshness.d.ts +18 -0
- package/dist/mesh/refine-config.d.ts +216 -0
- package/dist/mesh/worktree-bootstrap-config.d.ts +135 -0
- package/dist/providers/acp-provider-instance.d.ts +5 -0
- package/dist/providers/approval-utils.d.ts +20 -0
- package/dist/providers/chat-message-normalization.d.ts +13 -1
- package/dist/providers/cli-provider-instance.d.ts +111 -3
- package/dist/providers/contracts.d.ts +139 -6
- package/dist/providers/external-sources.d.ts +71 -0
- package/dist/providers/manual-attendance.d.ts +63 -0
- package/dist/providers/native-history/antigravity-cli-transcript.d.ts +100 -0
- package/dist/providers/native-history/claude-cli-transcript.d.ts +70 -0
- package/dist/providers/native-history/codex-cli-transcript.d.ts +73 -0
- package/dist/providers/native-history/constants.d.ts +12 -0
- package/dist/providers/native-history/dispatcher.d.ts +27 -0
- package/dist/providers/native-history/hermes-cli-transcript.d.ts +30 -0
- package/dist/providers/native-history/index.d.ts +13 -0
- package/dist/providers/provider-instance-manager.d.ts +14 -0
- package/dist/providers/provider-instance.d.ts +23 -1
- package/dist/providers/provider-loader.d.ts +26 -4
- package/dist/providers/provider-trust.d.ts +31 -0
- package/dist/providers/read-chat-contract.d.ts +29 -0
- package/dist/providers/sdk/v1/builders/acp/detect-status.d.ts +68 -0
- package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +86 -0
- package/dist/providers/sdk/v1/builders/cli/parse-approval-squash.d.ts +59 -0
- package/dist/providers/sdk/v1/builders/cli/parse-approval.d.ts +65 -0
- package/dist/providers/sdk/v1/builders/cli/parse-session.d.ts +91 -0
- package/dist/providers/sdk/v1/builders/cli/visible-region.d.ts +42 -0
- package/dist/providers/sdk/v1/fixture-tooling/format.d.ts +126 -0
- package/dist/providers/sdk/v1/fixture-tooling/index.d.ts +8 -0
- package/dist/providers/sdk/v1/fixture-tooling/replay.d.ts +38 -0
- package/dist/providers/sdk/v1/index.d.ts +30 -0
- package/dist/providers/sdk/v1/sandbox/README-design.d.ts +193 -0
- package/dist/providers/sdk/v1/sandbox/require-whitelist.d.ts +74 -0
- package/dist/providers/sdk/v1/sandbox/script-runner.d.ts +98 -0
- package/dist/providers/sdk/v1/types/cli/index.d.ts +277 -0
- package/dist/providers/sdk/v1/types/common/index.d.ts +203 -0
- package/dist/providers/sdk/v1/validators/index.d.ts +5 -0
- package/dist/providers/sdk/v1/validators/manifest.d.ts +40 -0
- package/dist/providers/sdk/v1/validators/taint.d.ts +52 -0
- package/dist/providers/spec/adapter.d.ts +91 -0
- package/dist/providers/spec/cli-adapter.d.ts +223 -0
- package/dist/providers/spec/evaluator.d.ts +45 -0
- package/dist/providers/spec/fsm-driver.d.ts +397 -0
- package/dist/providers/spec/fsm-evaluator.d.ts +76 -0
- package/dist/providers/spec/fsm-loader.d.ts +14 -0
- package/dist/providers/spec/fsm-types.d.ts +174 -0
- package/dist/providers/spec/native-history-executor.d.ts +41 -0
- package/dist/providers/spec/pre-launch-trust.d.ts +16 -0
- package/dist/providers/spec/route.d.ts +4 -0
- package/dist/providers/spec/types.d.ts +222 -0
- package/dist/providers/status-monitor.d.ts +7 -7
- package/dist/providers/transcript-v2.d.ts +176 -0
- package/dist/providers/types/interactive-prompt.d.ts +84 -0
- package/dist/providers/working-dir.d.ts +17 -0
- package/dist/repo-mesh-types.d.ts +376 -13
- package/dist/runtime-defaults.d.ts +2 -0
- package/dist/sessions/registry.d.ts +6 -0
- package/dist/shared-types-extra.d.ts +2 -4
- package/dist/shared-types.d.ts +59 -2
- package/dist/status/normalize.d.ts +1 -1
- package/dist/status/normalize.js +1 -0
- package/dist/status/normalize.js.map +1 -1
- package/dist/status/normalize.mjs +1 -0
- package/dist/status/normalize.mjs.map +1 -1
- package/dist/status/reporter.d.ts +2 -0
- package/dist/status/snapshot.d.ts +26 -0
- package/dist/system/hash.d.ts +8 -0
- package/dist/system/load-better-sqlite3.d.ts +21 -0
- package/dist/types.d.ts +5 -0
- package/package.json +7 -3
- package/src/agent-stream/poller.ts +2 -3
- package/src/agent-stream/provider-adapter.ts +1 -1
- package/src/boot/daemon-lifecycle.ts +63 -12
- package/src/boot/process-hardening.ts +89 -0
- package/src/build-info.ts +73 -0
- package/src/chat/source-machine.ts +534 -0
- package/src/chat/source-resolver.ts +0 -0
- package/src/chat/subscription-updates.ts +20 -1
- package/src/cli-adapter-types.d.ts +1 -0
- package/src/cli-adapter-types.ts +56 -2
- package/src/cli-adapters/cli-script-runner.ts +421 -0
- package/src/cli-adapters/cli-state-engine.ts +1221 -0
- package/src/cli-adapters/provider-cli-adapter.d.ts +1 -1
- package/src/cli-adapters/provider-cli-adapter.ts +836 -1141
- package/src/cli-adapters/provider-cli-parse.d.ts +1 -0
- package/src/cli-adapters/provider-cli-parse.ts +13 -0
- package/src/cli-adapters/provider-cli-runtime.ts +56 -7
- package/src/cli-adapters/provider-cli-shared.d.ts +4 -0
- package/src/cli-adapters/provider-cli-shared.ts +110 -11
- package/src/cli-adapters/pty-transport.ts +2 -1
- package/src/cli-adapters/pty-write-chunking.ts +106 -0
- package/src/cli-adapters/raw-terminal-io.ts +252 -0
- package/src/cli-adapters/resolve-executable.ts +90 -0
- package/src/cli-adapters/session-host-transport.ts +2 -1
- package/src/cli-adapters/terminal-backends/ghostty-vt-backend.ts +53 -29
- package/src/cli-adapters/terminal-backends/types.ts +10 -3
- package/src/cli-adapters/terminal-screen.ts +16 -81
- package/src/commands/WINDOWS-UPGRADE-LOCK-FAILURE.md +198 -0
- package/src/commands/chat-commands-debug-bundle.ts +398 -0
- package/src/commands/chat-commands-read.ts +2327 -0
- package/src/commands/chat-commands-scope.ts +54 -0
- package/src/commands/chat-commands-shared.ts +114 -0
- package/src/commands/chat-commands-write.ts +880 -0
- package/src/commands/chat-commands.ts +19 -1841
- package/src/commands/cli-manager.ts +513 -26
- package/src/commands/handler.ts +841 -2
- package/src/commands/high-family/index.ts +28 -0
- package/src/commands/high-family/mesh-coordinator-launch.ts +657 -0
- package/src/commands/high-family/mesh-events.ts +47 -0
- package/src/commands/high-family/mesh-status.ts +642 -0
- package/src/commands/high-family/types.ts +76 -0
- package/src/commands/low-family/coordinator-prompt.ts +72 -0
- package/src/commands/low-family/daemon-lifecycle.ts +107 -0
- package/src/commands/low-family/diagnostics.ts +57 -0
- package/src/commands/low-family/index.ts +37 -0
- package/src/commands/low-family/mesh-ledger.ts +62 -0
- package/src/commands/low-family/mesh-node-logs.ts +87 -0
- package/src/commands/low-family/notification.ts +116 -0
- package/src/commands/low-family/refine-config.ts +106 -0
- package/src/commands/low-family/session-host.ts +274 -0
- package/src/commands/low-family/spec-providerdev.ts +217 -0
- package/src/commands/low-family/status-meta.ts +112 -0
- package/src/commands/low-family/types.ts +39 -0
- package/src/commands/med-family/cli-agent.ts +230 -0
- package/src/commands/med-family/fast-forward.ts +230 -0
- package/src/commands/med-family/ide.ts +163 -0
- package/src/commands/med-family/index.ts +37 -0
- package/src/commands/med-family/mesh-crud.ts +825 -0
- package/src/commands/med-family/mesh-host-pairing.ts +234 -0
- package/src/commands/med-family/mesh-queue.ts +131 -0
- package/src/commands/med-family/mesh-restart.ts +92 -0
- package/src/commands/med-family/types.ts +120 -0
- package/src/commands/mesh-coordinator.ts +334 -124
- package/src/commands/router.ts +2395 -3206
- package/src/commands/stream-commands.ts +8 -0
- package/src/commands/upgrade-helper.ts +310 -45
- package/src/config/chat-history.ts +483 -24
- package/src/config/config.ts +12 -0
- package/src/config/mesh-config.ts +353 -22
- package/src/config/recent-activity.ts +8 -2
- package/src/daemon/dev-auto-implement.ts +3 -2
- package/src/daemon/dev-cli-debug.ts +10 -1
- package/src/daemon/dev-server.ts +0 -541
- package/src/detection/cli-detector.ts +28 -9
- package/src/detection/ide-detector.ts +55 -16
- package/src/detection/win32-ide-version.ts +106 -0
- package/src/git/change-impact-config.ts +354 -0
- package/src/git/git-commands.ts +55 -14
- package/src/git/git-diff.ts +81 -11
- package/src/git/git-executor.ts +12 -0
- package/src/git/git-status.ts +574 -43
- package/src/git/git-types.ts +14 -62
- package/src/git/git-worktree.ts +8 -1
- package/src/git/index.ts +16 -0
- package/src/index.ts +171 -12
- package/src/installer.d.ts +1 -1
- package/src/installer.ts +8 -6
- package/src/ipc/local-ipc-server.ts +278 -0
- package/src/launch.d.ts +1 -1
- package/src/launch.ts +37 -28
- package/src/logging/async-batch-writer.ts +55 -0
- package/src/logging/command-log.ts +7 -5
- package/src/logging/log-redactor.ts +100 -0
- package/src/logging/log-tail-reader.ts +341 -0
- package/src/logging/logger.ts +14 -7
- package/src/mesh/contracts.ts +338 -0
- package/src/mesh/coordinator-prompt.ts +381 -33
- package/src/mesh/coordinator-registry.ts +121 -0
- package/src/mesh/mesh-active-work.ts +645 -0
- package/src/mesh/mesh-coordinator-config.ts +97 -0
- package/src/mesh/mesh-delivery-policy.ts +315 -0
- package/src/mesh/mesh-event-classify.ts +51 -0
- package/src/mesh/mesh-event-forwarding.ts +1575 -0
- package/src/mesh/mesh-event-trace.ts +67 -0
- package/src/mesh/mesh-events-coordinator.ts +30 -0
- package/src/mesh/mesh-events-pending.ts +590 -0
- package/src/mesh/mesh-events-stale.ts +363 -0
- package/src/mesh/mesh-events-utils.ts +425 -0
- package/src/mesh/mesh-events.ts +30 -1035
- package/src/mesh/mesh-fast-forward.ts +839 -0
- package/src/mesh/mesh-host-ownership.ts +73 -0
- package/src/mesh/mesh-init.ts +260 -0
- package/src/mesh/mesh-ledger-reconciliation.ts +12 -7
- package/src/mesh/mesh-ledger.ts +596 -102
- package/src/mesh/mesh-missions.ts +322 -0
- package/src/mesh/mesh-node-identity.ts +1887 -0
- package/src/mesh/mesh-queue-assignment.ts +1491 -0
- package/src/mesh/mesh-reconcile-loop.ts +1332 -0
- package/src/mesh/mesh-refine-batch.ts +197 -0
- package/src/mesh/mesh-refine-gates.ts +1652 -0
- package/src/mesh/mesh-refine-status.ts +231 -0
- package/src/mesh/mesh-review-inbox.ts +307 -0
- package/src/mesh/mesh-routing.ts +291 -0
- package/src/mesh/mesh-runtime-store.ts +1708 -0
- package/src/mesh/mesh-scheduling-runtime.ts +199 -0
- package/src/mesh/mesh-task-stats.ts +161 -0
- package/src/mesh/mesh-unresolved-forward-outbox.ts +185 -0
- package/src/mesh/mesh-warmup-deadline.ts +152 -0
- package/src/mesh/mesh-work-queue.ts +997 -141
- package/src/mesh/preview-freshness.ts +118 -0
- package/src/mesh/refine-config.ts +423 -0
- package/src/mesh/worktree-bootstrap-config.ts +331 -0
- package/src/providers/acp-provider-instance.ts +43 -10
- package/src/providers/approval-utils.d.ts +5 -0
- package/src/providers/approval-utils.ts +57 -5
- package/src/providers/chat-message-normalization.ts +46 -6
- package/src/providers/cli-provider-instance.ts +1416 -101
- package/src/providers/contracts.d.ts +55 -0
- package/src/providers/contracts.ts +150 -6
- package/src/providers/extension-provider-instance.ts +12 -7
- package/src/providers/external-sources.ts +218 -0
- package/src/providers/ide-provider-instance.ts +35 -12
- package/src/providers/manual-attendance.ts +85 -0
- package/src/providers/native-history/antigravity-cli-transcript.ts +643 -0
- package/src/providers/native-history/claude-cli-transcript.ts +396 -0
- package/src/providers/native-history/codex-cli-transcript.ts +479 -0
- package/src/providers/native-history/constants.ts +19 -0
- package/src/providers/native-history/dispatcher.ts +339 -0
- package/src/providers/native-history/hermes-cli-transcript.ts +230 -0
- package/src/providers/native-history/index.ts +30 -0
- package/src/providers/provider-instance-manager.ts +30 -0
- package/src/providers/provider-instance.ts +19 -1
- package/src/providers/provider-loader.ts +668 -50
- package/src/providers/provider-schema.ts +87 -14
- package/src/providers/provider-trust.ts +114 -0
- package/src/providers/read-chat-contract.ts +76 -16
- package/src/providers/sdk/README.md +49 -0
- package/src/providers/sdk/v1/builders/acp/detect-status.ts +144 -0
- package/src/providers/sdk/v1/builders/cli/detect-status.ts +313 -0
- package/src/providers/sdk/v1/builders/cli/parse-approval-squash.ts +158 -0
- package/src/providers/sdk/v1/builders/cli/parse-approval.ts +250 -0
- package/src/providers/sdk/v1/builders/cli/parse-session.ts +276 -0
- package/src/providers/sdk/v1/builders/cli/visible-region.ts +143 -0
- package/src/providers/sdk/v1/fixture-tooling/format.ts +130 -0
- package/src/providers/sdk/v1/fixture-tooling/index.ts +22 -0
- package/src/providers/sdk/v1/fixture-tooling/replay.ts +352 -0
- package/src/providers/sdk/v1/index.ts +152 -0
- package/src/providers/sdk/v1/sandbox/README-design.ts +195 -0
- package/src/providers/sdk/v1/sandbox/require-whitelist.ts +472 -0
- package/src/providers/sdk/v1/sandbox/script-runner.ts +150 -0
- package/src/providers/sdk/v1/schemas/cli/provider.schema.json +504 -0
- package/src/providers/sdk/v1/schemas/primitives/acp-session-protocol-v1.json +131 -0
- package/src/providers/sdk/v1/schemas/primitives/native-history-codex-rollout-v1.json +66 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-approval-squash-v1.json +91 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-assistant-block-v1.json +91 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-cue-ordering-v1.json +47 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-dispatch-order-v1.json +32 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-footer-chrome-v1.json +42 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-index-finder-v1.json +27 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-modal-v1.json +125 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-prompt-marker-v1.json +45 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-session-id-extraction-v1.json +46 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-settled-prompt-v1.json +71 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-spinner-v1.json +83 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-transcript-pty-v1.json +83 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-visible-region-v1.json +57 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-welcome-screen-v1.json +35 -0
- package/src/providers/sdk/v1/types/cli/index.ts +365 -0
- package/src/providers/sdk/v1/types/common/index.ts +229 -0
- package/src/providers/sdk/v1/validators/index.ts +19 -0
- package/src/providers/sdk/v1/validators/manifest.ts +110 -0
- package/src/providers/sdk/v1/validators/taint.ts +309 -0
- package/src/providers/spec/adapter.ts +235 -0
- package/src/providers/spec/cli-adapter.ts +1051 -0
- package/src/providers/spec/evaluator.ts +407 -0
- package/src/providers/spec/fsm-driver.ts +1292 -0
- package/src/providers/spec/fsm-evaluator.ts +272 -0
- package/src/providers/spec/fsm-loader.ts +120 -0
- package/src/providers/spec/fsm-types.ts +273 -0
- package/src/providers/spec/native-history-executor.ts +1128 -0
- package/src/providers/spec/pre-launch-trust.ts +104 -0
- package/src/providers/spec/route.ts +51 -0
- package/src/providers/spec/types.ts +262 -0
- package/src/providers/status-monitor.d.ts +7 -7
- package/src/providers/status-monitor.ts +37 -22
- package/src/providers/transcript-v2.ts +567 -0
- package/src/providers/types/interactive-prompt.ts +536 -0
- package/src/providers/version-archive.ts +64 -24
- package/src/providers/working-dir.ts +23 -0
- package/src/repo-mesh-types.ts +542 -14
- package/src/runtime-defaults.ts +39 -0
- package/src/sessions/registry.ts +6 -0
- package/src/shared-types-extra.ts +2 -4
- package/src/shared-types.d.ts +8 -0
- package/src/shared-types.ts +64 -1
- package/src/status/builders.ts +26 -6
- package/src/status/normalize.ts +2 -0
- package/src/status/reporter.ts +19 -1
- package/src/status/snapshot.ts +95 -26
- package/src/system/hash.ts +23 -0
- package/src/system/host-memory.ts +29 -12
- package/src/system/load-better-sqlite3.ts +68 -0
- package/src/types.ts +5 -0
- package/dist/cli-adapters/terminal-backends/xterm-backend.d.ts +0 -17
- package/dist/mesh/mesh-sync.d.ts +0 -53
- package/src/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +0 -16
- package/src/cli-adapters/terminal-backends/xterm-backend.ts +0 -97
- package/src/mesh/mesh-sync.ts +0 -111
|
@@ -0,0 +1,1332 @@
|
|
|
1
|
+
// ---------------------------------------------------------------------------
|
|
2
|
+
// mesh-reconcile-loop — periodic queue → live coordinator reconciliation
|
|
3
|
+
// ---------------------------------------------------------------------------
|
|
4
|
+
// Single-model replacement for the old event-based "spontaneous forward" paths
|
|
5
|
+
// (remote P2P mesh_forward_event dispatch + live-CLI PTY fire-and-forget inject).
|
|
6
|
+
// Those pushed events at the moment a worker transitioned state, and silently
|
|
7
|
+
// dropped on the network (P2P) or when the coordinator was generating.
|
|
8
|
+
//
|
|
9
|
+
// The reliable backbone has always been the pending-events queue (SQLite +
|
|
10
|
+
// JSONL): every mesh coordinator event is persisted there before anything else
|
|
11
|
+
// (see injectMeshSystemMessage). What was missing was an *active* drainer that
|
|
12
|
+
// runs on a schedule rather than only when the coordinator (an LLM) happens to
|
|
13
|
+
// call a mesh tool.
|
|
14
|
+
//
|
|
15
|
+
// This loop is that drainer. On a fixed interval it:
|
|
16
|
+
// 1. Finds live CLI coordinator sessions on THIS daemon (meshCoordinatorFor
|
|
17
|
+
// stamp). For each mesh, drains the local queue scoped to this daemon and
|
|
18
|
+
// injects pending events into the coordinator. When a coordinator is idle it
|
|
19
|
+
// receives every queued event. When ONLY generating coordinators exist (the
|
|
20
|
+
// common case while the coordinator is blocked awaiting a worker result), the
|
|
21
|
+
// loop force-drains ONLY the force-inject events (completion / approval / stop /
|
|
22
|
+
// refine·bootstrap terminal) and force-writes them into the generating PTY —
|
|
23
|
+
// the same busy-bypass send-guard escape the live-CLI inject used to use.
|
|
24
|
+
// Non-force progress events stay queued for the next idle tick (injecting them
|
|
25
|
+
// mid-generation would be noise). This is what makes a coordinator parked in
|
|
26
|
+
// `generating` while awaiting a worker's completion actually receive it.
|
|
27
|
+
// 2. In cloud mode (dispatchMeshCommand present), pulls each remote worker
|
|
28
|
+
// node daemon's queue over P2P (get_pending_mesh_events) and re-injects via
|
|
29
|
+
// handleMeshForwardEvent — the same pull the MCP drainCoordinatorPendingEvents
|
|
30
|
+
// already does, now driven by the daemon timer instead of an LLM tool call.
|
|
31
|
+
//
|
|
32
|
+
// IMPORTANT — limits of this loop:
|
|
33
|
+
// - It only delivers to *live CLI coordinator instances* on this daemon. A
|
|
34
|
+
// pure stdio MCP coordinator (an LLM with no live CLI session to inject
|
|
35
|
+
// into) has no inject target here; that case stays pull-driven — the LLM
|
|
36
|
+
// drains the queue when it calls mesh_status / mesh_read_chat. We do NOT try
|
|
37
|
+
// to "wake" an LLM from the daemon; that is structurally impossible over a
|
|
38
|
+
// stdio request/response transport. See docs/refactoring/2026-06-15-mesh-event-to-queue-polling.md §4.7.
|
|
39
|
+
// - Queue persistence (queuePendingMeshCoordinatorEvent) and the SQLite
|
|
40
|
+
// drained=1 idempotency are the trust backbone and are untouched by this loop.
|
|
41
|
+
// ---------------------------------------------------------------------------
|
|
42
|
+
|
|
43
|
+
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
44
|
+
import type { LocalMeshEntry } from '../repo-mesh-types.js';
|
|
45
|
+
import { loadConfig } from '../config/config.js';
|
|
46
|
+
import { listMeshes } from '../config/mesh-config.js';
|
|
47
|
+
import { LOG } from '../logging/logger.js';
|
|
48
|
+
import { drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, buildPendingEventFingerprint, queuePendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
49
|
+
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
50
|
+
import { appendLedgerEntry } from './mesh-ledger.js';
|
|
51
|
+
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
52
|
+
import { handleMeshForwardEvent, shouldForceInjectMeshEvent, MESH_FORCE_INJECT_EVENTS, triggerMeshQueue, resolveForwardEventMeshId } from './mesh-events-coordinator.js';
|
|
53
|
+
import {
|
|
54
|
+
peekUnresolvedDelegateForwards,
|
|
55
|
+
ackUnresolvedDelegateForward,
|
|
56
|
+
expireStaleUnresolvedDelegateForwards,
|
|
57
|
+
} from './mesh-unresolved-forward-outbox.js';
|
|
58
|
+
import { readNonEmptyString, readMeshCompletionSummary } from './mesh-events-utils.js';
|
|
59
|
+
import { traceMeshEventStage, traceMeshEventDrop } from './mesh-event-trace.js';
|
|
60
|
+
import { expandDaemonIdForms, daemonIdsEquivalent } from '@adhdev/mesh-shared';
|
|
61
|
+
import { getActiveDirectDispatches, getQueue, reclaimStrandedAssignedTask, updateTaskStatus } from './mesh-work-queue.js';
|
|
62
|
+
import { readLedgerEntries } from './mesh-ledger.js';
|
|
63
|
+
import { pruneStaleDirectDispatches } from './mesh-active-work.js';
|
|
64
|
+
import { findTerminalLedgerEvidenceForTask, reconcileDirectDispatchCompletionFromTranscript } from './mesh-events-stale.js';
|
|
65
|
+
import { extractFinalAssistantSummaryEvidence } from '../providers/chat-message-normalization.js';
|
|
66
|
+
import type { ChatMessage } from '../types.js';
|
|
67
|
+
|
|
68
|
+
// Default reconcile cadence. approval/completion notifications to a live CLI
|
|
69
|
+
// coordinator land within at most one interval. Overridable via env for tuning.
|
|
70
|
+
const DEFAULT_RECONCILE_INTERVAL_MS = 4_000;
|
|
71
|
+
|
|
72
|
+
// PHASE 5 (auto-prune) conservative age gate. A direct dispatch whose node/session is
|
|
73
|
+
// orphaned (no longer in the live mesh) is only auto-pruned once it is at least this old,
|
|
74
|
+
// measured from its dispatch time. This protects against a node/session that is only
|
|
75
|
+
// *transiently* invisible (a momentary probe failure, a daemon restart) being pruned the
|
|
76
|
+
// instant it disappears. The MANUAL prune (mesh_prune_stale_direct) has no age gate — an
|
|
77
|
+
// operator pruning explicitly wants the orphan gone now. Overridable via env for tuning.
|
|
78
|
+
const DEFAULT_AUTO_PRUNE_MIN_AGE_MS = 24 * 60 * 60_000; // 24h
|
|
79
|
+
|
|
80
|
+
function resolveAutoPruneMinAgeMs(): number {
|
|
81
|
+
const raw = readNonEmptyString(process.env.MESH_AUTO_PRUNE_MIN_AGE_MS);
|
|
82
|
+
if (raw) {
|
|
83
|
+
const parsed = Number.parseInt(raw, 10);
|
|
84
|
+
// Clamp to [1h, 30d] so a mis-set env can't make the gate pathologically aggressive
|
|
85
|
+
// (prune the moment something blinks) or effectively disable it forever.
|
|
86
|
+
if (Number.isFinite(parsed) && parsed >= 60 * 60_000 && parsed <= 30 * 24 * 60 * 60_000) return parsed;
|
|
87
|
+
}
|
|
88
|
+
return DEFAULT_AUTO_PRUNE_MIN_AGE_MS;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function resolveReconcileIntervalMs(): number {
|
|
92
|
+
const raw = readNonEmptyString(process.env.MESH_RECONCILE_INTERVAL_MS);
|
|
93
|
+
if (raw) {
|
|
94
|
+
const parsed = Number.parseInt(raw, 10);
|
|
95
|
+
if (Number.isFinite(parsed) && parsed >= 1_000 && parsed <= 60_000) return parsed;
|
|
96
|
+
}
|
|
97
|
+
return DEFAULT_RECONCILE_INTERVAL_MS;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
interface LiveCoordinator {
|
|
101
|
+
meshId: string;
|
|
102
|
+
instance: ReturnType<DaemonComponents['instanceManager']['getInstance']>;
|
|
103
|
+
// Runtime session id of this coordinator instance (getState().instanceId). PHASE 2
|
|
104
|
+
// strict-matches an event's targetCoordinatorSessionId against this so a completion
|
|
105
|
+
// routes back to the exact originating coordinator session, not a sibling on the same
|
|
106
|
+
// daemon (the multi-coordinator misroute).
|
|
107
|
+
sessionId: string;
|
|
108
|
+
idle: boolean;
|
|
109
|
+
// True when the coordinator session is parked on a harness modal awaiting a
|
|
110
|
+
// human answer — claude-cli AskUserQuestion (waiting_choice) or a tool-consent
|
|
111
|
+
// prompt (waiting_approval). A force-inject into such a session would write raw
|
|
112
|
+
// keystrokes the modal key handler consumes, silently selecting a choice the
|
|
113
|
+
// user never made (data corruption). PHASE 2 excludes these from force-inject
|
|
114
|
+
// and leaves the event queued for a later (modal-resolved) tick.
|
|
115
|
+
modalParked: boolean;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// The set of coordinator-daemon ids THIS daemon answers to when draining the
|
|
119
|
+
// pending-events queue. A unicast completion event is stamped with the worker's
|
|
120
|
+
// meshCoordinatorDaemonId, which can be either:
|
|
121
|
+
// - the daemon's canonical status id (`standalone_<machineId>` / `daemon_<machineId>`),
|
|
122
|
+
// stamped by the MCP layer via ctx.localDaemonId (= getStatus().status.instanceId), or
|
|
123
|
+
// - the bare machineId, stamped by the local queue-assignment path (loadConfig().machineId).
|
|
124
|
+
// - the config-form node daemonId (`daemon_<machineId>`), which the MCP layer's
|
|
125
|
+
// resolveCoordinatorDaemonId prefers and stamps onto direct-dispatch workers.
|
|
126
|
+
// Draining with only one of these silently misses events stamped with the other —
|
|
127
|
+
// the exact reason a generating coordinator never self-received local completions,
|
|
128
|
+
// and the base-node completion-surface bug (base completions land full-form
|
|
129
|
+
// `daemon_<machineId>` while a coordinator that only knows itself as bare
|
|
130
|
+
// `<machineId>` never matches them). We expand to EVERY equivalent form so the
|
|
131
|
+
// scope match (host gate, self-node detection, and the drain IN-filter downstream)
|
|
132
|
+
// succeeds regardless of which path stamped the event.
|
|
133
|
+
function resolveCoordinatorDaemonIds(components: DaemonComponents): string[] {
|
|
134
|
+
const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
|
|
135
|
+
const machineId = readNonEmptyString(loadConfig().machineId);
|
|
136
|
+
return expandDaemonIdForms([statusInstanceId, machineId]);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
// Whether THIS daemon is the coordinator/host for a mesh — i.e. the daemon that
|
|
140
|
+
// owns coordinator ownership and must collect every worker node's completion
|
|
141
|
+
// events into its local queue. This is true regardless of whether a *live CLI*
|
|
142
|
+
// coordinator session currently exists: the coordinator is frequently a pure
|
|
143
|
+
// stdio MCP LLM (no live CLI session to inject into), and that LLM only sees the
|
|
144
|
+
// queue when it next calls a mesh tool. For it to see remote worker completions
|
|
145
|
+
// at all, the daemon must have already pulled them into the local queue on the
|
|
146
|
+
// timer — which is exactly what this predicate gates.
|
|
147
|
+
//
|
|
148
|
+
// Rule: this daemon hosts the mesh when meshHost.role is 'host' (the default for
|
|
149
|
+
// standalone-compat meshes with no host metadata) AND, when a hostDaemonId is
|
|
150
|
+
// pinned, it resolves to one of this daemon's ids. Member-only daemons return
|
|
151
|
+
// false — their own queue is pulled BY the host, not the other way around.
|
|
152
|
+
//
|
|
153
|
+
// `daemonIds` here is the EXPANDED self-identity set (runtime drain ids ∪ this
|
|
154
|
+
// daemon's mesh-config node id forms) — see resolveCoordinatorSelfIds. The
|
|
155
|
+
// pinned hostDaemonId is itself a config-form id and frequently does NOT equal a
|
|
156
|
+
// runtime id (bare machineId / status id), so gating on the runtime ids alone
|
|
157
|
+
// would wrongly classify the real host as a non-host and skip the remote pull
|
|
158
|
+
// entirely.
|
|
159
|
+
function daemonHostsMesh(mesh: LocalMeshEntry, daemonIds: string[]): boolean {
|
|
160
|
+
const host = mesh.meshHost;
|
|
161
|
+
// No metadata → default host (standalone compatibility, see createDefaultMeshHostMetadata).
|
|
162
|
+
if (!host) return true;
|
|
163
|
+
if (host.role && host.role !== 'host') return false;
|
|
164
|
+
const hostDaemonId = readNonEmptyString(host.hostDaemonId);
|
|
165
|
+
// Host role but no pinned hostDaemonId → treat as host (single-daemon / legacy).
|
|
166
|
+
if (!hostDaemonId) return true;
|
|
167
|
+
return daemonIdListIncludes(daemonIds, hostDaemonId);
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
function daemonIdListIncludes(ids: readonly string[], id: string | undefined): boolean {
|
|
171
|
+
if (!id) return false;
|
|
172
|
+
return ids.some(candidate => candidate === id || daemonIdsEquivalent(candidate, id));
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Resolve EVERY id-form this daemon answers to FOR A GIVEN MESH: the runtime drain
|
|
176
|
+
// ids (status id + bare machineId) unioned with this daemon's mesh-config identity
|
|
177
|
+
// forms — the self node's daemonId/machineId (the node whose daemonId/machineId
|
|
178
|
+
// matches a runtime id) and the pinned meshHost.hostDaemonId WHEN it is provably
|
|
179
|
+
// ours. This is the single source of truth for "is this id me?" across both the
|
|
180
|
+
// host gate and the remote pull filter; the worker's meshCoordinatorDaemonId stamp
|
|
181
|
+
// is guaranteed to be one of these forms (it comes from resolveCoordinatorDaemonId,
|
|
182
|
+
// which prefers the coordinator node's config-form daemonId over the runtime status id).
|
|
183
|
+
function resolveCoordinatorSelfIds(mesh: LocalMeshEntry, drainDaemonIds: string[]): string[] {
|
|
184
|
+
const ids = new Set<string>(drainDaemonIds);
|
|
185
|
+
// Expand with the config-form id(s) of the self node — the mesh node whose
|
|
186
|
+
// daemonId/machineId matches a runtime id. Its config-form daemonId is exactly
|
|
187
|
+
// what resolveCoordinatorNode()→resolveCoordinatorDaemonId() stamps onto a worker.
|
|
188
|
+
for (const node of mesh.nodes) {
|
|
189
|
+
const nodeDaemonId = readNonEmptyString(node.daemonId);
|
|
190
|
+
const nodeMachineId = readNonEmptyString(node.machineId);
|
|
191
|
+
const isSelf = (nodeDaemonId && daemonIdListIncludes(drainDaemonIds, nodeDaemonId))
|
|
192
|
+
|| (nodeMachineId && daemonIdListIncludes(drainDaemonIds, nodeMachineId));
|
|
193
|
+
if (!isSelf) continue;
|
|
194
|
+
if (nodeDaemonId) ids.add(nodeDaemonId);
|
|
195
|
+
if (nodeMachineId) ids.add(nodeMachineId);
|
|
196
|
+
}
|
|
197
|
+
// The pinned host id is included ONLY when it is provably one of THIS daemon's ids
|
|
198
|
+
// (it already matches a runtime id or a resolved self-node id). A hostDaemonId that
|
|
199
|
+
// names a DIFFERENT daemon must NOT be claimed — that would make a member-only
|
|
200
|
+
// daemon believe it is the host and pull queues it does not own. Having a node on
|
|
201
|
+
// this daemon does not make this daemon the host; daemonHostsMesh still honours a
|
|
202
|
+
// foreign hostDaemonId and rejects ownership.
|
|
203
|
+
const hostDaemonId = readNonEmptyString(mesh.meshHost?.hostDaemonId);
|
|
204
|
+
if (hostDaemonId && daemonIdListIncludes([...ids], hostDaemonId)) ids.add(hostDaemonId);
|
|
205
|
+
return [...ids];
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
// Observability: last-seen modal-park state per coordinator session, so we LOG.info
|
|
209
|
+
// only on a TRANSITION (clear → parked, parked → cleared) instead of every 4s tick.
|
|
210
|
+
// Per-process; a restart re-logs the first observation, which is desirable — it
|
|
211
|
+
// re-confirms a coordinator that is still parked after the restart (the exact
|
|
212
|
+
// "restart does not clear it" symptom the operator needs visibility into).
|
|
213
|
+
const coordinatorModalParkState = new Map<string, boolean>();
|
|
214
|
+
|
|
215
|
+
// Find live CLI coordinator instances on THIS daemon, keyed by mesh.
|
|
216
|
+
function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
|
|
217
|
+
const out: LiveCoordinator[] = [];
|
|
218
|
+
for (const inst of components.instanceManager.getByCategory('cli')) {
|
|
219
|
+
const state = inst.getState();
|
|
220
|
+
const settings = state.settings && typeof state.settings === 'object'
|
|
221
|
+
? state.settings as Record<string, unknown>
|
|
222
|
+
: {};
|
|
223
|
+
const meshId = readNonEmptyString(settings.meshCoordinatorFor);
|
|
224
|
+
if (!meshId) continue;
|
|
225
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
226
|
+
// getState() overlays the modal-park statuses: an active AskUserQuestion
|
|
227
|
+
// prompt surfaces as waiting_choice, a tool-consent prompt as waiting_approval.
|
|
228
|
+
// Lowercase literal compare — the SessionStatus enum is forked across modules
|
|
229
|
+
// and waiting_choice is absent from some of them (see cli-provider-instance).
|
|
230
|
+
const modalParked = status === 'waiting_choice' || status === 'waiting_approval';
|
|
231
|
+
const sessionId = readNonEmptyString(state.instanceId);
|
|
232
|
+
// Modal-park transition observability: a coordinator entering modal-park is what
|
|
233
|
+
// begins holding completion events under `modal_parked`; one leaving it is what
|
|
234
|
+
// drains them. Both transitions were previously SILENT (the operator had no log
|
|
235
|
+
// to diagnose a stuck/held completion), so emit a single line per edge.
|
|
236
|
+
const stateKey = `${meshId}::${sessionId || '?'}`;
|
|
237
|
+
const prevParked = coordinatorModalParkState.get(stateKey);
|
|
238
|
+
if (prevParked !== modalParked) {
|
|
239
|
+
coordinatorModalParkState.set(stateKey, modalParked);
|
|
240
|
+
if (modalParked) {
|
|
241
|
+
LOG.info('MeshReconcile', `Coordinator ${sessionId || '?'} (mesh ${meshId}) entered modal-park (status=${status}) — terminal events for it will be held until the modal is answered`);
|
|
242
|
+
} else if (prevParked === true) {
|
|
243
|
+
LOG.info('MeshReconcile', `Coordinator ${sessionId || '?'} (mesh ${meshId}) left modal-park (status=${status}) — held events will drain on this/next tick`);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
out.push({ meshId, instance: inst, sessionId, idle: status === 'idle', modalParked });
|
|
247
|
+
}
|
|
248
|
+
return out;
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
// Inject a drained pending event into a live coordinator session. Force-inject
|
|
252
|
+
// events carry force:true so they bypass the busy send-guard and land in the PTY
|
|
253
|
+
// even while the coordinator is generating (see shouldForceInjectMeshEvent).
|
|
254
|
+
function injectPendingIntoCoordinator(
|
|
255
|
+
coordinator: LiveCoordinator['instance'],
|
|
256
|
+
pending: PendingMeshCoordinatorEvent,
|
|
257
|
+
): void {
|
|
258
|
+
if (!coordinator || !pending.coordinatorMessage) return;
|
|
259
|
+
const force = shouldForceInjectMeshEvent(pending.event);
|
|
260
|
+
// EVTTRACE: event surfaced to the coordinator (injected into its live CLI session).
|
|
261
|
+
// This is the terminal happy-path stage. Observation only.
|
|
262
|
+
traceMeshEventStage('surfaced', {
|
|
263
|
+
taskId: pending.metadataEvent?.taskId,
|
|
264
|
+
sessionId: pending.metadataEvent?.targetSessionId ?? pending.targetCoordinatorSessionId,
|
|
265
|
+
nodeId: pending.nodeId,
|
|
266
|
+
meshId: pending.meshId,
|
|
267
|
+
event: pending.event,
|
|
268
|
+
}, force ? 'force-inject' : 'inject');
|
|
269
|
+
coordinator.onEvent('send_message', {
|
|
270
|
+
input: { text: pending.coordinatorMessage, textFallback: pending.coordinatorMessage },
|
|
271
|
+
...(force ? { force: true } : {}),
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// Held-event ledger dedup: fingerprints of held terminal events already written as an
|
|
276
|
+
// `event_held` ledger audit record in THIS process. Prevents the 4s reconcile tick from
|
|
277
|
+
// re-logging the same held event every interval while a coordinator stays modal-parked.
|
|
278
|
+
// Per-process only (not persisted) — if the daemon restarts while an event is still held
|
|
279
|
+
// it is re-logged once, which is desirable: it re-confirms the event is still undelivered.
|
|
280
|
+
const heldEventLedgerRecorded = new Set<string>();
|
|
281
|
+
|
|
282
|
+
// C1 (data safety): when a terminal completion/approval/bootstrap event cannot be
|
|
283
|
+
// delivered because the only coordinators are modal-parked, the event is held at
|
|
284
|
+
// drained=0 in the pending queue (SQLite + JSONL) for a later tick. That queue is
|
|
285
|
+
// disk-persisted but carries no operator-visible audit trail and can be silently
|
|
286
|
+
// dropped by the pending-file trim (100 KB / 50-entry cap). To guarantee a held
|
|
287
|
+
// completion's worker summary is never silently lost, mirror each held terminal event
|
|
288
|
+
// into the coordinator's mesh ledger as an `event_held` entry — auditable and
|
|
289
|
+
// recoverable (the finalSummary survives even if the pending copy is later trimmed or
|
|
290
|
+
// the coordinator session is force-resolved before re-drain). Idempotent per process
|
|
291
|
+
// via heldEventLedgerRecorded so a long modal park does not spam the ledger.
|
|
292
|
+
function recordHeldTerminalEventsToLedger(
|
|
293
|
+
meshId: string,
|
|
294
|
+
drainDaemonIds: string[],
|
|
295
|
+
reason: string,
|
|
296
|
+
heldForCoordinatorCount: number,
|
|
297
|
+
): void {
|
|
298
|
+
let pending: readonly PendingMeshCoordinatorEvent[];
|
|
299
|
+
try {
|
|
300
|
+
pending = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
|
|
301
|
+
} catch {
|
|
302
|
+
return; // best-effort audit — never let a peek failure break the tick
|
|
303
|
+
}
|
|
304
|
+
for (const event of pending) {
|
|
305
|
+
// Only audit terminal/force-inject events (completion / approval / stop / refine·
|
|
306
|
+
// bootstrap). Silent lifecycle events (agent:ready / generating_started) carry no
|
|
307
|
+
// worker output to preserve and re-drain harmlessly, so they need no audit trail.
|
|
308
|
+
if (!shouldForceInjectMeshEvent(event.event)) continue;
|
|
309
|
+
const fingerprint = buildPendingEventFingerprint(event);
|
|
310
|
+
const key = `${meshId}::${fingerprint || `${event.event}::${event.nodeId || ''}::${event.queuedAt}`}`;
|
|
311
|
+
if (heldEventLedgerRecorded.has(key)) continue;
|
|
312
|
+
heldEventLedgerRecorded.add(key);
|
|
313
|
+
const finalSummary = readMeshCompletionSummary(event.metadataEvent);
|
|
314
|
+
try {
|
|
315
|
+
appendLedgerEntry(meshId, {
|
|
316
|
+
kind: 'event_held',
|
|
317
|
+
...(event.nodeId ? { nodeId: event.nodeId } : {}),
|
|
318
|
+
payload: {
|
|
319
|
+
event: event.event,
|
|
320
|
+
reason,
|
|
321
|
+
recoverable: true,
|
|
322
|
+
heldForCoordinators: heldForCoordinatorCount,
|
|
323
|
+
nodeLabel: event.nodeLabel,
|
|
324
|
+
...(event.workspace ? { workspace: event.workspace } : {}),
|
|
325
|
+
targetCoordinatorDaemonId: event.targetCoordinatorDaemonId ?? null,
|
|
326
|
+
queuedAt: event.queuedAt,
|
|
327
|
+
...(fingerprint ? { fingerprint } : {}),
|
|
328
|
+
...(finalSummary ? { finalSummary } : {}),
|
|
329
|
+
},
|
|
330
|
+
});
|
|
331
|
+
LOG.info('MeshReconcile', `Ledger-recorded held ${event.event} for mesh ${meshId} (reason ${reason}) — recoverable from ledger`);
|
|
332
|
+
} catch (e: any) {
|
|
333
|
+
// Failed to persist — drop the dedup marker so the next tick retries.
|
|
334
|
+
heldEventLedgerRecorded.delete(key);
|
|
335
|
+
LOG.warn('MeshReconcile', `Failed to ledger-record held ${event.event} for mesh ${meshId}: ${e?.message || e}`);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// One reconcile tick. Two independent phases:
|
|
341
|
+
//
|
|
342
|
+
// PHASE 1 — Remote queue pull (the fix for remote worktree completions never
|
|
343
|
+
// reaching an MCP/LLM coordinator). For EVERY mesh this daemon hosts/
|
|
344
|
+
// coordinates, pull each remote worker node's pending-events queue over P2P
|
|
345
|
+
// into THIS daemon's local queue. This runs *regardless of whether a live CLI
|
|
346
|
+
// coordinator exists* — the coordinator is usually a pure stdio MCP LLM with
|
|
347
|
+
// no live CLI session, and it can only observe a remote worker's completion
|
|
348
|
+
// once that event has been pulled into the local queue (which it then drains
|
|
349
|
+
// on its next mesh tool call). Previously this pull was gated behind a live
|
|
350
|
+
// CLI coordinator and so never ran for MCP/LLM coordinators — remote
|
|
351
|
+
// completions sat on the remote node's queue until the LLM happened to call
|
|
352
|
+
// mesh_read_chat, which triggered the MCP-side pull. The daemon now does it
|
|
353
|
+
// autonomously on the timer. Standalone (no dispatchMeshCommand) skips this
|
|
354
|
+
// phase entirely — there are no remote nodes to pull from.
|
|
355
|
+
//
|
|
356
|
+
// PHASE 2 — Live CLI inject. For each mesh that has a live CLI coordinator on
|
|
357
|
+
// THIS daemon, drain the local queue and inject pending events into the PTY.
|
|
358
|
+
// Unchanged from before.
|
|
359
|
+
// Bug B: how long a row may sit 'assigned' with an unconfirmed dispatch before the
|
|
360
|
+
// watchdog reclaims it. Must be comfortably larger than the per-dispatch confirm
|
|
361
|
+
// timeout (DISPATCH_CONFIRM_TIMEOUT_MS in mesh-events-coordinator) so a slow-but-live
|
|
362
|
+
// dispatch still inside its normal confirm window is never reclaimed early — this is
|
|
363
|
+
// the durable backstop for the case the in-process confirm timer can't cover (a timer
|
|
364
|
+
// lost to a daemon restart between claim and confirm).
|
|
365
|
+
const ASSIGNED_STRANDED_DEADLINE_MS = 5 * 60_000;
|
|
366
|
+
|
|
367
|
+
// PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
|
|
368
|
+
// flips a row to 'assigned' BEFORE the fire-and-forget dispatch runs. If that dispatch
|
|
369
|
+
// neither rejects (→ no .catch requeue) nor is confirmed delivered — a relay that hangs
|
|
370
|
+
// without acking, or a confirm timer lost across a restart — the row stays 'assigned'
|
|
371
|
+
// forever: it contributes 0 pending, so PHASE 3 (gated on pendingQueueTaskCount>0) never
|
|
372
|
+
// re-examines it, and nothing but a manual requeue clears it. This is that missing net.
|
|
373
|
+
//
|
|
374
|
+
// Regression guard: a row whose delivery IS confirmed (delivered/acked/completed) is a
|
|
375
|
+
// genuinely in-flight (or completion-lost) task — left to PHASE 4's completion reconcile,
|
|
376
|
+
// never reclaimed here. And the deadline is generous so a slow-but-live dispatch still in
|
|
377
|
+
// its normal confirm window is never reclaimed early. Reclaimed rows return to 'pending'
|
|
378
|
+
// with ownership cleared, so the PHASE 3 trigger below re-dispatches them this same tick.
|
|
379
|
+
function recoverStrandedAssignedDispatches(meshId: string, store: MeshRuntimeStore): void {
|
|
380
|
+
const assigned = getQueue(meshId, { status: ['assigned'] });
|
|
381
|
+
if (!assigned.length) return;
|
|
382
|
+
const nowMs = Date.now();
|
|
383
|
+
for (const row of assigned) {
|
|
384
|
+
const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
|
|
385
|
+
if (!Number.isFinite(dispatchedAtMs)) continue; // no dispatch ts → can't age it
|
|
386
|
+
if (nowMs - dispatchedAtMs < ASSIGNED_STRANDED_DEADLINE_MS) continue; // still in confirm window
|
|
387
|
+
const terminal = findTerminalLedgerEvidenceForTask({
|
|
388
|
+
meshId,
|
|
389
|
+
taskId: row.id,
|
|
390
|
+
});
|
|
391
|
+
if (terminal) {
|
|
392
|
+
const status = terminal.kind === 'task_completed' ? 'completed' : 'failed';
|
|
393
|
+
updateTaskStatus(meshId, row.id, status);
|
|
394
|
+
LOG.warn('MeshReconcile', `Skipped stranded reclaim redispatch for terminal task ${row.id} on mesh ${meshId}; ${terminal.kind} ledger evidence already exists`);
|
|
395
|
+
traceMeshEventDrop('assigned_stranded_terminal_ledger', {
|
|
396
|
+
taskId: row.id,
|
|
397
|
+
sessionId: row.assignedSessionId,
|
|
398
|
+
nodeId: row.assignedNodeId,
|
|
399
|
+
meshId,
|
|
400
|
+
event: 'agent:generating_completed',
|
|
401
|
+
}, terminal.kind);
|
|
402
|
+
continue;
|
|
403
|
+
}
|
|
404
|
+
if (store.taskHasConfirmedDelivery(meshId, row.id)) continue; // dispatched → PHASE 4's job
|
|
405
|
+
const reclaimed = reclaimStrandedAssignedTask(meshId, row.id, {
|
|
406
|
+
reason: 'assigned_stranded_dispatch_unconfirmed',
|
|
407
|
+
ageMs: nowMs - dispatchedAtMs,
|
|
408
|
+
});
|
|
409
|
+
if (reclaimed) {
|
|
410
|
+
LOG.warn('MeshReconcile', `Reclaimed stranded assigned task ${row.id} on mesh ${meshId} `
|
|
411
|
+
+ `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, dispatched `
|
|
412
|
+
+ `${Math.round((nowMs - dispatchedAtMs) / 1000)}s ago, never confirmed delivered → ${reclaimed.status})`);
|
|
413
|
+
// EVTTRACE: the dispatch for this task was stranded (assigned, never confirmed
|
|
414
|
+
// delivered) and reclaimed (CANON-B) — its expected completion event never
|
|
415
|
+
// arrived. Observation only; the reclaim decision above is unchanged.
|
|
416
|
+
traceMeshEventDrop('assigned_stranded_reclaim', {
|
|
417
|
+
taskId: row.id,
|
|
418
|
+
sessionId: row.assignedSessionId,
|
|
419
|
+
nodeId: row.assignedNodeId,
|
|
420
|
+
meshId,
|
|
421
|
+
event: 'agent:generating_completed',
|
|
422
|
+
}, `unconfirmed ${Math.round((nowMs - dispatchedAtMs) / 1000)}s → ${reclaimed.status}`);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
export async function runMeshReconcileTick(components: DaemonComponents): Promise<void> {
|
|
428
|
+
const localDaemonId = readNonEmptyString(loadConfig().machineId) || undefined;
|
|
429
|
+
// The id-set used to scope the local queue drain (status id + machineId). See
|
|
430
|
+
// resolveCoordinatorDaemonIds — the status id is what the MCP layer stamps and
|
|
431
|
+
// is mandatory here for a generating CLI coordinator to self-receive completions.
|
|
432
|
+
const drainDaemonIds = resolveCoordinatorDaemonIds(components);
|
|
433
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
434
|
+
const store = (() => {
|
|
435
|
+
try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
|
|
436
|
+
})();
|
|
437
|
+
|
|
438
|
+
// ── PHASE 0: retry the worker-side unresolved-delegate forward outbox ──────
|
|
439
|
+
// Cloud-only (needs dispatchMeshCommand). A worker that is NOT a member of the
|
|
440
|
+
// coordinator's mesh cannot be reached by the coordinator's PHASE 1 pull (it is
|
|
441
|
+
// in no mesh.node), so its completion must be PUSHED to the coordinator. This
|
|
442
|
+
// drains the durable outbox enqueued by forwardUnresolvedDelegateEvent and retries
|
|
443
|
+
// any push that has not yet been acked. See mesh-unresolved-forward-outbox.ts.
|
|
444
|
+
if (dispatchMeshCommand) {
|
|
445
|
+
try {
|
|
446
|
+
await retryUnresolvedDelegateForwards(components);
|
|
447
|
+
} catch (e: any) {
|
|
448
|
+
LOG.warn('MeshReconcile', `Unresolved-delegate forward retry failed: ${e?.message || e}`);
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
// ── PHASE 1: pull remote node queues for every mesh this daemon hosts ──────
|
|
453
|
+
// Cloud-only (dispatchMeshCommand present). Runs whether or not a live CLI
|
|
454
|
+
// coordinator exists — this is what lets an MCP/LLM coordinator ever see a
|
|
455
|
+
// remote worker's completion.
|
|
456
|
+
if (dispatchMeshCommand) {
|
|
457
|
+
for (const mesh of listMeshes()) {
|
|
458
|
+
// Expand to every id-form this daemon answers to for this mesh (runtime
|
|
459
|
+
// drain ids ∪ config-form node/host ids) and use it for BOTH the host gate
|
|
460
|
+
// and the remote pull filter, so a worker stamp in any form is recovered.
|
|
461
|
+
const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
|
|
462
|
+
if (!daemonHostsMesh(mesh, selfIds)) continue;
|
|
463
|
+
try {
|
|
464
|
+
await pullRemoteNodeQueues(components, mesh, localDaemonId, selfIds);
|
|
465
|
+
} catch (e: any) {
|
|
466
|
+
LOG.warn('MeshReconcile', `Remote node pull failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// ── PHASE 2.5: assigned-stranded dispatch watchdog (Bug B) ─────────────────
|
|
472
|
+
// Runs before PHASE 3 so any row it returns to 'pending' is re-dispatched by the
|
|
473
|
+
// PHASE 3 trigger in this same tick. See recoverStrandedAssignedDispatches.
|
|
474
|
+
if (store) {
|
|
475
|
+
for (const mesh of listMeshes()) {
|
|
476
|
+
const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
|
|
477
|
+
if (!daemonHostsMesh(mesh, selfIds)) continue;
|
|
478
|
+
try {
|
|
479
|
+
recoverStrandedAssignedDispatches(mesh.id, store);
|
|
480
|
+
} catch (e: any) {
|
|
481
|
+
LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
// ── PHASE 3: recover pending queue claims for newly-idle sessions ──────────
|
|
487
|
+
// The event-driven claim paths (agent:ready / agent:generating_completed in
|
|
488
|
+
// mesh-events-coordinator) re-claim the queue the moment a session goes idle,
|
|
489
|
+
// but that depends on a single event being emitted AND (for a remote node)
|
|
490
|
+
// successfully forwarded to this coordinator. If that event is missed/dropped,
|
|
491
|
+
// a pending task targeting a now-idle session would sit unclaimed forever —
|
|
492
|
+
// there was no periodic safety net. This phase is that net: for every mesh this
|
|
493
|
+
// daemon hosts that has at least one pending task, run one triggerMeshQueue so a
|
|
494
|
+
// session that became idle without a delivered ready-event still gets its work.
|
|
495
|
+
//
|
|
496
|
+
// O(1) guard: skip the (relatively expensive) full idle-session + remote-idle
|
|
497
|
+
// scan entirely when the queue has no pending tasks — a COUNT(*) over the
|
|
498
|
+
// indexed status column, so an idle mesh costs one cheap query per tick.
|
|
499
|
+
// claimNextQueueTask is atomic, so racing the event-driven path can only have
|
|
500
|
+
// one winner; double-claiming is impossible.
|
|
501
|
+
for (const mesh of listMeshes()) {
|
|
502
|
+
const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
|
|
503
|
+
if (!daemonHostsMesh(mesh, selfIds)) continue;
|
|
504
|
+
if (store) {
|
|
505
|
+
try {
|
|
506
|
+
if (store.pendingQueueTaskCount(mesh.id) === 0) continue;
|
|
507
|
+
} catch { /* fall through and let triggerMeshQueue decide */ }
|
|
508
|
+
}
|
|
509
|
+
try {
|
|
510
|
+
await triggerMeshQueue(components, mesh.id);
|
|
511
|
+
} catch (e: any) {
|
|
512
|
+
LOG.warn('MeshReconcile', `Pending-claim recovery trigger failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
// ── PHASE 4: synthesize lost completions for unterminated direct dispatches ─
|
|
517
|
+
// Symmetric to PHASE 3 (which recovers a *lost claim* for a newly-idle session)
|
|
518
|
+
// but for the opposite gap: a worker that ALREADY completed, went idle, and
|
|
519
|
+
// whose terminal completion event was never persisted (dropped before reaching
|
|
520
|
+
// the queue/outbox, or its forward was lost). PHASE 1/2/3 can only deliver an
|
|
521
|
+
// event that exists in a queue — they cannot recover a completion that was
|
|
522
|
+
// never recorded, so the coordinator keeps believing the worker is generating.
|
|
523
|
+
//
|
|
524
|
+
// reconcileDirectDispatchCompletionFromTranscript already synthesizes the
|
|
525
|
+
// missing terminal event from the worker's transcript, but until now it ran
|
|
526
|
+
// ONLY when an LLM coordinator polled mesh_status (mcp_mesh_status_transcript_
|
|
527
|
+
// reconciliation). This phase pulls that same correction onto the daemon timer
|
|
528
|
+
// so it no longer depends on the LLM polling. The reconcile is idempotent
|
|
529
|
+
// (hasTerminalLedgerAfterDispatch guards against re-synthesis), so attempting it
|
|
530
|
+
// every tick for the same dispatch is safe — once a terminal exists it no-ops.
|
|
531
|
+
for (const mesh of listMeshes()) {
|
|
532
|
+
const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
|
|
533
|
+
if (!daemonHostsMesh(mesh, selfIds)) continue;
|
|
534
|
+
try {
|
|
535
|
+
await reconcileUnterminatedDirectDispatches(components, mesh, selfIds, localDaemonId);
|
|
536
|
+
} catch (e: any) {
|
|
537
|
+
LOG.warn('MeshReconcile', `Completion reconcile failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// ── PHASE 5: auto-prune orphaned direct dispatch records ───────────────────
|
|
542
|
+
// staleDirectWork (orphaned direct-dispatch rows whose node/session is no longer in the
|
|
543
|
+
// live mesh) otherwise accumulates indefinitely: a removed worktree node or a cleanly
|
|
544
|
+
// terminated session leaves its direct-dispatch row behind, stuck in a non-terminal status
|
|
545
|
+
// (e.g. generating) for days. This is NOT a false-idle bug — it is the separate problem of
|
|
546
|
+
// orphaned records that the only existing cleanup path (manual MCP mesh_prune_stale_direct)
|
|
547
|
+
// never reaches unless an operator runs it by hand.
|
|
548
|
+
//
|
|
549
|
+
// This phase runs the SAME prune core the manual tool calls (pruneStaleDirectDispatches),
|
|
550
|
+
// in execute mode, on the daemon timer. The only difference from the manual path is a
|
|
551
|
+
// conservative age gate (DEFAULT_AUTO_PRUNE_MIN_AGE_MS): a freshly-orphaned record is held
|
|
552
|
+
// back until it is provably stale, so a transient probe miss never auto-prunes live work.
|
|
553
|
+
// Every other safety rule is inherited unchanged from the core — active/pending/generating
|
|
554
|
+
// work and fresh unacknowledged dispatch failures are never pruned, ledger-only audit entries
|
|
555
|
+
// are preserved, and the prune itself is recorded with a direct_dispatch_pruned ledger entry.
|
|
556
|
+
// Idempotent: a pruned row is gone from getActiveDirectDispatches, so the next tick finds
|
|
557
|
+
// nothing to re-prune. Isolated in its own try/catch per mesh so it can never kill the tick.
|
|
558
|
+
{
|
|
559
|
+
const minAgeMs = resolveAutoPruneMinAgeMs();
|
|
560
|
+
for (const mesh of listMeshes()) {
|
|
561
|
+
const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
|
|
562
|
+
if (!daemonHostsMesh(mesh, selfIds)) continue;
|
|
563
|
+
try {
|
|
564
|
+
await autoPruneStaleDirectDispatches(components, mesh, selfIds, localDaemonId, minAgeMs);
|
|
565
|
+
} catch (e: any) {
|
|
566
|
+
LOG.warn('MeshReconcile', `Auto-prune stale direct failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
// ── PHASE 2: inject into live CLI coordinators on this daemon ──────────────
|
|
572
|
+
const coordinators = findLiveCoordinators(components);
|
|
573
|
+
if (coordinators.length === 0) {
|
|
574
|
+
// No live CLI coordinator on this daemon — nothing to inject into.
|
|
575
|
+
// (MCP-only LLM coordinators drain the local queue via their own tool
|
|
576
|
+
// calls; PHASE 1 above has already populated it from remote nodes.)
|
|
577
|
+
return;
|
|
578
|
+
}
|
|
579
|
+
|
|
580
|
+
// Group coordinators by mesh; multiple coordinator instances for one mesh is
|
|
581
|
+
// unusual but supported (each gets the same drained events).
|
|
582
|
+
const byMesh = new Map<string, LiveCoordinator[]>();
|
|
583
|
+
for (const c of coordinators) {
|
|
584
|
+
const list = byMesh.get(c.meshId);
|
|
585
|
+
if (list) list.push(c);
|
|
586
|
+
else byMesh.set(c.meshId, [c]);
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
for (const [meshId, meshCoordinators] of byMesh) {
|
|
590
|
+
// Drain the local queue scoped to this coordinator daemon and inject.
|
|
591
|
+
// - If an idle coordinator exists, FULL-drain and deliver every event to it
|
|
592
|
+
// (it can receive non-force progress events without deadlocking).
|
|
593
|
+
// - If only GENERATING coordinators exist, force-drain ONLY the force-inject
|
|
594
|
+
// events (completion/approval/stop/refine·bootstrap terminal) and force-inject
|
|
595
|
+
// them so a coordinator parked in `generating` while awaiting that very event
|
|
596
|
+
// is not deadlocked. Non-force progress events stay queued for the next idle
|
|
597
|
+
// tick — injecting them would be noise mid-generation. Both drains mark the
|
|
598
|
+
// consumed rows drained=1 atomically, so the pull path can't re-deliver.
|
|
599
|
+
const idleCoordinators = meshCoordinators.filter(c => c.idle);
|
|
600
|
+
// A coordinator parked on a harness modal (waiting_choice / waiting_approval)
|
|
601
|
+
// is non-idle, so it would otherwise be treated as a force-inject target. It
|
|
602
|
+
// must NOT be: a force-inject writes raw keystrokes into the PTY, which the
|
|
603
|
+
// modal's key handler consumes and silently resolves to a choice the user
|
|
604
|
+
// never made. Force-inject is only safe into a coordinator parked in plain
|
|
605
|
+
// `generating` (the deadlock the force path exists to break). So generating
|
|
606
|
+
// targets are the non-idle, non-modal-parked coordinators.
|
|
607
|
+
const generatingCoordinators = meshCoordinators.filter(c => !c.idle && !c.modalParked);
|
|
608
|
+
const modalParkedCoordinators = meshCoordinators.filter(c => !c.idle && c.modalParked);
|
|
609
|
+
const targetCoordinators = idleCoordinators.length > 0 ? idleCoordinators : generatingCoordinators;
|
|
610
|
+
const forceOnly = idleCoordinators.length === 0;
|
|
611
|
+
|
|
612
|
+
// ── modal-blocked short-circuit (MUST precede the drain) ──────────────────
|
|
613
|
+
// When the ONLY coordinators for this mesh are modal-parked (no idle, no plain
|
|
614
|
+
// generating target), there is nowhere safe to deliver. We skip-and-requeue:
|
|
615
|
+
// by NOT draining we leave the events at drained=0 in the queue, so a later tick
|
|
616
|
+
// (once the modal is resolved and the coordinator returns to idle/generating)
|
|
617
|
+
// delivers them. This short-circuit MUST run BEFORE drainPendingMeshCoordinatorEvents
|
|
618
|
+
// — the drain marks rows drained=1 atomically, which would lose the events for a
|
|
619
|
+
// coordinator that is only transiently blocked. (Note: generating is still
|
|
620
|
+
// force-injected via generatingCoordinators — we never block the deadlock-break.)
|
|
621
|
+
if (targetCoordinators.length === 0) {
|
|
622
|
+
if (modalParkedCoordinators.length > 0) {
|
|
623
|
+
// ── orphan escape (MUST precede the blanket modal-park hold) ──────────
|
|
624
|
+
// A modal-parked coordinator with no idle/generating sibling otherwise
|
|
625
|
+
// wedges EVERY pending event under `modal_parked` until that modal resolves
|
|
626
|
+
// — including a STRICT-routed completion whose originating coordinator
|
|
627
|
+
// session is GONE (an orphan: the worktree/session that produced it was
|
|
628
|
+
// removed, or that coordinator session died). Such an event will never be
|
|
629
|
+
// deliverable to its target session no matter what the modal-parked sibling
|
|
630
|
+
// does, so holding it under modal_parked is a permanent-held leak (the very
|
|
631
|
+
// "data restart re-reproduces it" symptom — the gate is reconstructed live
|
|
632
|
+
// from the still-parked modal, so a restart does not clear it). Route those
|
|
633
|
+
// orphan events through the strict-route hold/expire path so the bounded
|
|
634
|
+
// STRICT_SESSION_MATCH_TTL eventually expires them (recoverable, ledgered)
|
|
635
|
+
// instead of leaving them held forever. A strict event whose target session
|
|
636
|
+
// IS live but merely modal-parked is left to the blanket hold below (it is
|
|
637
|
+
// genuinely transiently blocked, not orphaned).
|
|
638
|
+
const liveSessionIds = new Set(
|
|
639
|
+
meshCoordinators.map(c => readNonEmptyString(c.sessionId)).filter(Boolean),
|
|
640
|
+
);
|
|
641
|
+
let orphanEscaped = 0;
|
|
642
|
+
const hasPendingForOrphanPeek = !store
|
|
643
|
+
|| (() => { try { return store.pendingEventCount(meshId) > 0; } catch { return true; } })();
|
|
644
|
+
if (hasPendingForOrphanPeek) {
|
|
645
|
+
// Identify which pending event NAMES correspond to orphan-targeted events
|
|
646
|
+
// (a strict targetCoordinatorSessionId that matches no live coordinator).
|
|
647
|
+
let peeked: readonly PendingMeshCoordinatorEvent[] = [];
|
|
648
|
+
try {
|
|
649
|
+
peeked = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
|
|
650
|
+
} catch { peeked = []; }
|
|
651
|
+
const isOrphan = (e: PendingMeshCoordinatorEvent): boolean => {
|
|
652
|
+
const want = readNonEmptyString(e.targetCoordinatorSessionId);
|
|
653
|
+
return !!want && !liveSessionIds.has(want);
|
|
654
|
+
};
|
|
655
|
+
const orphanEventNames = new Set(peeked.filter(isOrphan).map(e => e.event));
|
|
656
|
+
if (orphanEventNames.size > 0) {
|
|
657
|
+
// The drain filter is event-NAME scoped (not per-row), so draining by the
|
|
658
|
+
// orphan event names also pulls any non-orphan event sharing that name. Drain
|
|
659
|
+
// them all, then re-route: orphan-targeted events go through the strict-route
|
|
660
|
+
// hold/expire path (bounded TTL → eventually ledger-expired, recoverable);
|
|
661
|
+
// non-orphan events of the same name are re-queued unchanged (queuedAt
|
|
662
|
+
// preserved) so they remain genuinely held for their still-live, modal-parked
|
|
663
|
+
// target. This is the same per-event strict routing PHASE 2 does below — just
|
|
664
|
+
// reached here because the blanket modal-park short-circuit would otherwise
|
|
665
|
+
// wedge the orphans forever.
|
|
666
|
+
let drained: PendingMeshCoordinatorEvent[] = [];
|
|
667
|
+
try {
|
|
668
|
+
drained = drainPendingMeshCoordinatorEvents(
|
|
669
|
+
meshId,
|
|
670
|
+
drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
|
|
671
|
+
{ onlyEvents: orphanEventNames },
|
|
672
|
+
);
|
|
673
|
+
} catch (e: any) {
|
|
674
|
+
LOG.warn('MeshReconcile', `Orphan-escape drain failed for mesh ${meshId}: ${e?.message || e}`);
|
|
675
|
+
drained = [];
|
|
676
|
+
}
|
|
677
|
+
for (const pending of drained) {
|
|
678
|
+
if (isOrphan(pending)) {
|
|
679
|
+
holdOrExpireStrictUnmatchedEvent(pending, readNonEmptyString(pending.targetCoordinatorSessionId), meshId);
|
|
680
|
+
orphanEscaped++;
|
|
681
|
+
} else {
|
|
682
|
+
// Still-live (modal-parked) target — re-queue unchanged so it is held
|
|
683
|
+
// for the next modal-resolved tick, exactly like the blanket hold would.
|
|
684
|
+
try { queuePendingMeshCoordinatorEvent(pending); } catch { /* best-effort re-queue */ }
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
LOG.info('MeshReconcile', `Reconcile skip → modal-parked: holding pending event(s) for mesh ${meshId} (${modalParkedCoordinators.length} coordinator(s) awaiting a modal answer; events left queued${orphanEscaped > 0 ? `; ${orphanEscaped} orphan-targeted event(s) routed to strict-route TTL` : ''})`);
|
|
690
|
+
// C1: mirror held terminal events into the ledger so a held completion's
|
|
691
|
+
// worker summary is auditable/recoverable even if the modal is never
|
|
692
|
+
// resolved, the coordinator restarts, or the pending file is later trimmed.
|
|
693
|
+
// The events stay queued (drained=0) for re-drain on a later tick; this only
|
|
694
|
+
// adds the durable audit copy. Idempotent per process — only newly-held
|
|
695
|
+
// events are logged. O(1)-gated: skip the peek when the queue is empty.
|
|
696
|
+
let hasPending = true;
|
|
697
|
+
if (store) {
|
|
698
|
+
try { hasPending = store.pendingEventCount(meshId) > 0; } catch { /* peek below */ }
|
|
699
|
+
}
|
|
700
|
+
if (hasPending) {
|
|
701
|
+
recordHeldTerminalEventsToLedger(
|
|
702
|
+
meshId,
|
|
703
|
+
drainDaemonIds.length > 0 ? drainDaemonIds : (localDaemonId ? [localDaemonId] : []),
|
|
704
|
+
'modal_parked',
|
|
705
|
+
modalParkedCoordinators.length,
|
|
706
|
+
);
|
|
707
|
+
}
|
|
708
|
+
}
|
|
709
|
+
continue;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
// O(1) guard: skip the drain entirely when the queue is empty.
|
|
713
|
+
if (store) {
|
|
714
|
+
try {
|
|
715
|
+
if (store.pendingEventCount(meshId) === 0) continue;
|
|
716
|
+
} catch { /* fall through to drain */ }
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
let pendingEvents: PendingMeshCoordinatorEvent[] = [];
|
|
720
|
+
try {
|
|
721
|
+
pendingEvents = drainPendingMeshCoordinatorEvents(
|
|
722
|
+
meshId,
|
|
723
|
+
drainDaemonIds.length > 0 ? drainDaemonIds : localDaemonId,
|
|
724
|
+
forceOnly ? { onlyEvents: MESH_FORCE_INJECT_EVENTS } : undefined,
|
|
725
|
+
);
|
|
726
|
+
} catch (e: any) {
|
|
727
|
+
LOG.warn('MeshReconcile', `Drain failed for mesh ${meshId}: ${e?.message || e}`);
|
|
728
|
+
continue;
|
|
729
|
+
}
|
|
730
|
+
if (pendingEvents.length === 0) continue;
|
|
731
|
+
|
|
732
|
+
const mode = forceOnly ? 'force-drain → generating' : 'inject → idle';
|
|
733
|
+
LOG.info('MeshReconcile', `Reconcile ${mode}: ${pendingEvents.length} pending event(s) → ${targetCoordinators.length} coordinator(s) for mesh ${meshId}`);
|
|
734
|
+
for (const pending of pendingEvents) {
|
|
735
|
+
// Strict session routing (multi-coordinator): when the event names an
|
|
736
|
+
// originating coordinator session, deliver ONLY to the live coordinator whose
|
|
737
|
+
// session id matches — a sibling coordinator on the same daemon must NOT receive
|
|
738
|
+
// another coordinator's completion. When the event carries no session id (legacy /
|
|
739
|
+
// version-skewed / single-coordinator), fall back to the daemon-level set
|
|
740
|
+
// (unchanged behaviour — regression-0 for the common case).
|
|
741
|
+
const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
|
|
742
|
+
if (wantSession) {
|
|
743
|
+
const matched = targetCoordinators.filter(c => c.sessionId === wantSession);
|
|
744
|
+
if (matched.length === 0) {
|
|
745
|
+
// The originating coordinator session is not deliverable on this daemon
|
|
746
|
+
// right now (gone, or modal-parked and excluded from targets). Strict mode
|
|
747
|
+
// does NOT broadcast to siblings — hold the event for a later tick, and
|
|
748
|
+
// ledger-expire it past a TTL so it can never wedge forever.
|
|
749
|
+
holdOrExpireStrictUnmatchedEvent(pending, wantSession, meshId);
|
|
750
|
+
continue;
|
|
751
|
+
}
|
|
752
|
+
for (const c of matched) injectPendingIntoCoordinator(c.instance, pending);
|
|
753
|
+
continue;
|
|
754
|
+
}
|
|
755
|
+
for (const c of targetCoordinators) {
|
|
756
|
+
injectPendingIntoCoordinator(c.instance, pending);
|
|
757
|
+
}
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
// Strict-routing TTL: how long a drained completion whose originating coordinator session
|
|
763
|
+
// is not currently deliverable is held (re-queued for re-drain) before it is ledger-
|
|
764
|
+
// expired. Bounded so a coordinator session that never returns cannot wedge the event
|
|
765
|
+
// forever; broad enough to ride out a transient modal-park / brief restart.
|
|
766
|
+
const STRICT_SESSION_MATCH_TTL_MS = 60_000;
|
|
767
|
+
|
|
768
|
+
// Re-queue (hold) a strict-routed event whose coordinator session is not live, or — once it
|
|
769
|
+
// has aged past STRICT_SESSION_MATCH_TTL_MS — ledger-expire it (recoverable) and drop it.
|
|
770
|
+
// We deliberately do NOT broadcast an aged-out event to sibling coordinators: that is the
|
|
771
|
+
// very misroute strict routing exists to prevent. The drain already marked the row drained=1,
|
|
772
|
+
// so re-queuing re-persists a fresh undrained copy (dedup keys on drained=0 only); queuedAt is
|
|
773
|
+
// preserved so the TTL measures the event's true age across re-queues.
|
|
774
|
+
function holdOrExpireStrictUnmatchedEvent(
|
|
775
|
+
pending: PendingMeshCoordinatorEvent,
|
|
776
|
+
wantSession: string,
|
|
777
|
+
meshId: string,
|
|
778
|
+
): void {
|
|
779
|
+
const queuedAt = typeof pending.queuedAt === 'number' ? pending.queuedAt : Date.now();
|
|
780
|
+
if (Date.now() - queuedAt <= STRICT_SESSION_MATCH_TTL_MS) {
|
|
781
|
+
try {
|
|
782
|
+
queuePendingMeshCoordinatorEvent(pending); // preserves queuedAt → true age retained
|
|
783
|
+
LOG.info('MeshReconcile', `Strict route hold: coordinator session ${wantSession} not live on mesh ${meshId} — re-queued (${pending.event})`);
|
|
784
|
+
// EVTTRACE: event held (re-queued) — its originating coordinator session is not
|
|
785
|
+
// currently deliverable. Held, not dropped; surfaces later or expires past TTL.
|
|
786
|
+
traceMeshEventDrop('strict_route_hold', {
|
|
787
|
+
taskId: pending.metadataEvent?.taskId,
|
|
788
|
+
sessionId: pending.metadataEvent?.targetSessionId ?? wantSession,
|
|
789
|
+
nodeId: pending.nodeId,
|
|
790
|
+
meshId,
|
|
791
|
+
event: pending.event,
|
|
792
|
+
}, `coordinatorSession=${wantSession} not live`);
|
|
793
|
+
} catch (e: any) {
|
|
794
|
+
LOG.warn('MeshReconcile', `Strict route re-queue failed for ${pending.event} on mesh ${meshId}: ${e?.message || e}`);
|
|
795
|
+
}
|
|
796
|
+
return;
|
|
797
|
+
}
|
|
798
|
+
const finalSummary = readMeshCompletionSummary(pending.metadataEvent || {});
|
|
799
|
+
try {
|
|
800
|
+
appendLedgerEntry(meshId, {
|
|
801
|
+
kind: 'event_held',
|
|
802
|
+
...(pending.nodeId ? { nodeId: pending.nodeId } : {}),
|
|
803
|
+
payload: {
|
|
804
|
+
event: pending.event,
|
|
805
|
+
reason: 'strict_route_expired',
|
|
806
|
+
recoverable: true,
|
|
807
|
+
targetCoordinatorSessionId: wantSession,
|
|
808
|
+
targetCoordinatorDaemonId: pending.targetCoordinatorDaemonId ?? null,
|
|
809
|
+
nodeLabel: pending.nodeLabel,
|
|
810
|
+
...(pending.workspace ? { workspace: pending.workspace } : {}),
|
|
811
|
+
queuedAt,
|
|
812
|
+
...(finalSummary ? { finalSummary } : {}),
|
|
813
|
+
},
|
|
814
|
+
});
|
|
815
|
+
LOG.warn('MeshReconcile', `Strict route expire: coordinator session ${wantSession} never returned for mesh ${meshId} — recorded to ledger (recoverable), dropped (${pending.event})`);
|
|
816
|
+
// EVTTRACE: event expired past the strict-route TTL — dropped (recoverable, ledgered).
|
|
817
|
+
traceMeshEventDrop('strict_route_expired', {
|
|
818
|
+
taskId: pending.metadataEvent?.taskId,
|
|
819
|
+
sessionId: pending.metadataEvent?.targetSessionId ?? wantSession,
|
|
820
|
+
nodeId: pending.nodeId,
|
|
821
|
+
meshId,
|
|
822
|
+
event: pending.event,
|
|
823
|
+
}, `coordinatorSession=${wantSession} never returned`);
|
|
824
|
+
} catch (e: any) {
|
|
825
|
+
LOG.warn('MeshReconcile', `Failed to ledger-expire strict-unmatched ${pending.event} for mesh ${meshId}: ${e?.message || e}`);
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
|
|
829
|
+
// Cloud-only: retry the worker-side unresolved-delegate forward outbox. For each
|
|
830
|
+
// durably-queued entry, push it to its coordinator daemon over P2P (mesh_forward_event)
|
|
831
|
+
// and ack (mark drained) ONLY on a successful, non-rejected response. A failed or
|
|
832
|
+
// rejected push leaves the entry queued for the next tick — at-least-once delivery.
|
|
833
|
+
// Stale entries (coordinator unreachable past the max age) are expired first so the
|
|
834
|
+
// outbox can't grow without bound. The coordinator dedups duplicate deliveries on its
|
|
835
|
+
// own fingerprint, so a retry that races the original immediate push is harmless.
|
|
836
|
+
// RECONCILE-MESHID-DROP: per-entry count of consecutive HARD rejections (the coordinator
|
|
837
|
+
// returned success:false, e.g. "meshId required"). A rejection means the push was delivered
|
|
838
|
+
// and deterministically refused — retrying the identical payload every 4s can never succeed,
|
|
839
|
+
// so it would loop until the 30-minute age expiry, spamming the log the whole time. After
|
|
840
|
+
// MAX_FORWARD_REJECTIONS such rejections we drop the entry (drain it) with ONE fail-loud
|
|
841
|
+
// warning. Transient transport failures (the dispatch throws — coordinator momentarily
|
|
842
|
+
// unreachable) do NOT count here; those legitimately retry until the age expiry. In-memory
|
|
843
|
+
// (keyed by the durable outbox row id) is sufficient: a daemon restart re-arms the loop, and
|
|
844
|
+
// the age expiry remains the durable backstop. Cleared whenever an entry is delivered/drained.
|
|
845
|
+
const unresolvedForwardRejectionCounts = new Map<string, number>();
|
|
846
|
+
const MAX_FORWARD_REJECTIONS = 5;
|
|
847
|
+
|
|
848
|
+
export function __resetUnresolvedForwardRejectionCountsForTests(): void {
|
|
849
|
+
unresolvedForwardRejectionCounts.clear();
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
async function retryUnresolvedDelegateForwards(components: DaemonComponents): Promise<void> {
|
|
853
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
854
|
+
if (!dispatchMeshCommand) return;
|
|
855
|
+
|
|
856
|
+
// Drop entries that have exhausted their retry budget (fail-loud inside).
|
|
857
|
+
expireStaleUnresolvedDelegateForwards();
|
|
858
|
+
|
|
859
|
+
const entries = peekUnresolvedDelegateForwards();
|
|
860
|
+
if (entries.length === 0) {
|
|
861
|
+
// Nothing queued — clear any stale per-entry rejection counters so the map can't grow.
|
|
862
|
+
if (unresolvedForwardRejectionCounts.size > 0) unresolvedForwardRejectionCounts.clear();
|
|
863
|
+
return;
|
|
864
|
+
}
|
|
865
|
+
|
|
866
|
+
// Every id-form THIS daemon answers to. A self-addressed outbox entry (coordinator
|
|
867
|
+
// == this daemon) must never be cross-dialled — see the self-route branch below.
|
|
868
|
+
const selfIds = resolveCoordinatorDaemonIds(components);
|
|
869
|
+
const isSelfCoordinatorId = (id: string): boolean =>
|
|
870
|
+
selfIds.some(self => daemonIdsEquivalent(self, id));
|
|
871
|
+
|
|
872
|
+
for (const entry of entries) {
|
|
873
|
+
// EVTTRACE correlation context for this outbox entry's retry.
|
|
874
|
+
const entryTraceCtx = {
|
|
875
|
+
taskId: (entry.payload as Record<string, unknown>).taskId,
|
|
876
|
+
sessionId: readNonEmptyString(entry.payload.targetSessionId) || readNonEmptyString(entry.payload.sessionId),
|
|
877
|
+
nodeId: readNonEmptyString(entry.payload.nodeId),
|
|
878
|
+
event: readNonEmptyString(entry.payload.event),
|
|
879
|
+
};
|
|
880
|
+
|
|
881
|
+
// Self-addressed forward: the coordinator daemon this entry targets IS this
|
|
882
|
+
// daemon (a self-coordinating / single-node mesh, or a delegate whose coordinator
|
|
883
|
+
// anchor resolved to our own id). A cross-daemon mesh_forward_event to our own id
|
|
884
|
+
// is REFUSED by the dispatch self-dial guard ("Refusing to send ... to this
|
|
885
|
+
// daemon's own id; route via the local router instead") on every retry, so the
|
|
886
|
+
// entry can never be acked and loops forever (~every tick), spamming the log and
|
|
887
|
+
// pinning the outbox row permanently undrained. Honour the guard's own advice:
|
|
888
|
+
// route the event straight through the local receiver (handleMeshForwardEvent —
|
|
889
|
+
// the same path the coordinator runs on receiving a remote push), then ack it.
|
|
890
|
+
// We drain regardless of the local result: a cross-daemon dispatch could not have
|
|
891
|
+
// resolved it either (the guard rejects before the receiver ever runs), so leaving
|
|
892
|
+
// it queued only re-spams. handleMeshForwardEvent has the BEST recovery chance —
|
|
893
|
+
// this daemon hosts the mesh, so its workspace/nodeId → meshId recovery applies.
|
|
894
|
+
if (isSelfCoordinatorId(entry.coordinatorDaemonId)) {
|
|
895
|
+
let localResult: any;
|
|
896
|
+
try {
|
|
897
|
+
traceMeshEventStage('forward_send', entryTraceCtx, `self → local router (${entry.coordinatorDaemonId})`);
|
|
898
|
+
localResult = handleMeshForwardEvent(components, entry.payload);
|
|
899
|
+
} catch (e: any) {
|
|
900
|
+
LOG.warn('MeshReconcile', `Local route of self-addressed forward to ${entry.coordinatorDaemonId} threw: ${e?.message || e} — draining anyway to break the retry loop`);
|
|
901
|
+
}
|
|
902
|
+
ackUnresolvedDelegateForward(entry.id);
|
|
903
|
+
unresolvedForwardRejectionCounts.delete(entry.id);
|
|
904
|
+
if (localResult && localResult.success === false) {
|
|
905
|
+
LOG.warn('MeshReconcile', `Self-addressed unresolved-delegate ${readNonEmptyString(entry.payload.event)} rejected by local router (${readNonEmptyString(localResult.error) || 'no reason'}) — drained to break the self-forward retry loop`);
|
|
906
|
+
traceMeshEventDrop('self_forward_local_rejected', entryTraceCtx, readNonEmptyString(localResult.error) || 'no reason');
|
|
907
|
+
} else {
|
|
908
|
+
LOG.info('MeshReconcile', `Self-addressed unresolved-delegate ${readNonEmptyString(entry.payload.event)} routed via local router (coordinator ${entry.coordinatorDaemonId} is self) — drained`);
|
|
909
|
+
}
|
|
910
|
+
continue;
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
// RECONCILE-MESHID-DROP: the stored forward payload was built when the worker
|
|
914
|
+
// "couldn't resolve" its meshId, so the coordinator rejects it "meshId required"
|
|
915
|
+
// when its own workspace/nodeId recovery misses. The worker can usually resolve it
|
|
916
|
+
// now (member node membership / live-session meshNodeFor) — stamp it on so the
|
|
917
|
+
// coordinator accepts. Covers entries persisted before this fix AND late-bound
|
|
918
|
+
// sessions. No-op when the payload already carries a meshId or none is resolvable.
|
|
919
|
+
let pushPayload = entry.payload;
|
|
920
|
+
if (!readNonEmptyString(pushPayload.meshId)) {
|
|
921
|
+
const recoveredMeshId = resolveForwardEventMeshId(components, pushPayload);
|
|
922
|
+
if (recoveredMeshId) {
|
|
923
|
+
pushPayload = { ...pushPayload, meshId: recoveredMeshId };
|
|
924
|
+
traceMeshEventStage('forward_meshid_recovered', entryTraceCtx, `meshId=${recoveredMeshId}`);
|
|
925
|
+
}
|
|
926
|
+
}
|
|
927
|
+
|
|
928
|
+
let result: any;
|
|
929
|
+
try {
|
|
930
|
+
traceMeshEventStage('forward_send', entryTraceCtx, `retry → ${entry.coordinatorDaemonId}`);
|
|
931
|
+
result = await dispatchMeshCommand(entry.coordinatorDaemonId, 'mesh_forward_event', pushPayload);
|
|
932
|
+
} catch (e: any) {
|
|
933
|
+
// Coordinator unreachable (transport threw) — keep the entry queued and try again
|
|
934
|
+
// next tick. This is NOT a hard rejection, so it does not count toward the cap;
|
|
935
|
+
// the age expiry bounds a permanently-offline coordinator.
|
|
936
|
+
LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} failed: ${e?.message || e} — left queued`);
|
|
937
|
+
traceMeshEventDrop('retry_forward_failed', entryTraceCtx, e?.message || String(e));
|
|
938
|
+
continue;
|
|
939
|
+
}
|
|
940
|
+
if (result && result.success === false) {
|
|
941
|
+
// Hard rejection: the push was delivered and deterministically refused. Retrying
|
|
942
|
+
// the identical payload can never succeed, so bound it — after MAX_FORWARD_REJECTIONS
|
|
943
|
+
// drop (drain) the entry with one fail-loud warning instead of re-spamming every tick.
|
|
944
|
+
const rejections = (unresolvedForwardRejectionCounts.get(entry.id) || 0) + 1;
|
|
945
|
+
unresolvedForwardRejectionCounts.set(entry.id, rejections);
|
|
946
|
+
const reason = readNonEmptyString(result.error) || 'no reason';
|
|
947
|
+
if (rejections >= MAX_FORWARD_REJECTIONS) {
|
|
948
|
+
ackUnresolvedDelegateForward(entry.id);
|
|
949
|
+
unresolvedForwardRejectionCounts.delete(entry.id);
|
|
950
|
+
LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected ${rejections}x (${reason}) — dropping unresolved-delegate ${readNonEmptyString(entry.payload.event)} (sess=${readNonEmptyString(entry.payload.targetSessionId) || readNonEmptyString(entry.payload.sessionId) || '-'}) to stop the retry loop`);
|
|
951
|
+
traceMeshEventDrop('retry_forward_exhausted', entryTraceCtx, `${reason} (${rejections} rejections)`);
|
|
952
|
+
} else {
|
|
953
|
+
LOG.warn('MeshReconcile', `Retry forward to coordinator ${entry.coordinatorDaemonId} rejected (${reason}) — left queued (attempt ${rejections}/${MAX_FORWARD_REJECTIONS})`);
|
|
954
|
+
traceMeshEventDrop('retry_forward_rejected', entryTraceCtx, reason);
|
|
955
|
+
}
|
|
956
|
+
continue;
|
|
957
|
+
}
|
|
958
|
+
// Acked — mark the durable copy delivered.
|
|
959
|
+
ackUnresolvedDelegateForward(entry.id);
|
|
960
|
+
unresolvedForwardRejectionCounts.delete(entry.id);
|
|
961
|
+
LOG.info('MeshReconcile', `Retried+delivered unresolved-delegate ${readNonEmptyString(entry.payload.event)} to coordinator ${entry.coordinatorDaemonId}`);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
|
|
965
|
+
// Cloud-only: poll each remote worker node daemon for pending coordinator events
|
|
966
|
+
// and re-inject them locally via handleMeshForwardEvent (which re-queues +
|
|
967
|
+
// surfaces to the live coordinator on the next tick / immediately if idle).
|
|
968
|
+
//
|
|
969
|
+
// Scoping: the remote handler (get_pending_mesh_events) drains its queue filtered
|
|
970
|
+
// by coordinatorDaemonId — returning events targeted at that id OR unscoped, and
|
|
971
|
+
// leaving events targeted at a *different* coordinator. A remote worker stamps the
|
|
972
|
+
// coordinator id in one of SEVERAL forms (the canonical status id `standalone_`/
|
|
973
|
+
// `daemon_<machineId>` stamped by the MCP layer, the bare machineId stamped by the
|
|
974
|
+
// local queue path, OR — most commonly for remote launches — the coordinator mesh
|
|
975
|
+
// node's config-form `daemonId`, which resolveCoordinatorDaemonId prefers and which
|
|
976
|
+
// is NOT canonicalised). `candidateDaemonIds` is the already-expanded self-identity
|
|
977
|
+
// set (resolveCoordinatorSelfIds: runtime drain ids ∪ this daemon's mesh-config node/
|
|
978
|
+
// host id forms), so we pull ONCE PER candidate id and a completion stamped with any
|
|
979
|
+
// of them is recovered. The remote drain is atomic (drained=1), so issuing multiple
|
|
980
|
+
// pulls cannot double-deliver — the first pull that matches consumes the event; the
|
|
981
|
+
// rest see nothing. When no ids resolve we fall back to a single unscoped pull.
|
|
982
|
+
async function pullRemoteNodeQueues(
|
|
983
|
+
components: DaemonComponents,
|
|
984
|
+
mesh: LocalMeshEntry,
|
|
985
|
+
localDaemonId: string | undefined,
|
|
986
|
+
candidateDaemonIds: string[],
|
|
987
|
+
): Promise<void> {
|
|
988
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
989
|
+
if (!dispatchMeshCommand) return;
|
|
990
|
+
const meshId = mesh.id;
|
|
991
|
+
|
|
992
|
+
// One args object per candidate coordinator-id form, or a single unscoped pull
|
|
993
|
+
// when none resolve.
|
|
994
|
+
const pulls: Array<Record<string, unknown>> = candidateDaemonIds.length > 0
|
|
995
|
+
? candidateDaemonIds.map(id => ({ meshId, coordinatorDaemonId: id }))
|
|
996
|
+
: [{ meshId }];
|
|
997
|
+
|
|
998
|
+
for (const node of mesh.nodes) {
|
|
999
|
+
const nodeDaemonId = readNonEmptyString(node.daemonId);
|
|
1000
|
+
// Skip nodes without a daemon, and nodes on THIS daemon (their events are
|
|
1001
|
+
// already in the local queue drained in PHASE 2). "This daemon" is matched
|
|
1002
|
+
// against the full self-identity set (candidateDaemonIds), not just the bare
|
|
1003
|
+
// localDaemonId — a self node can be registered under the config-form daemonId
|
|
1004
|
+
// (`daemon_<machineId>`) which would NOT equal bare localDaemonId, and pulling
|
|
1005
|
+
// from ourselves over P2P is both wasteful and a self-dispatch hazard.
|
|
1006
|
+
if (!nodeDaemonId) continue;
|
|
1007
|
+
if (daemonIdsEquivalent(nodeDaemonId, localDaemonId)) continue;
|
|
1008
|
+
if (daemonIdListIncludes(candidateDaemonIds, nodeDaemonId)) continue;
|
|
1009
|
+
|
|
1010
|
+
for (const pendingEventArgs of pulls) {
|
|
1011
|
+
let events: unknown;
|
|
1012
|
+
try {
|
|
1013
|
+
events = await dispatchMeshCommand(nodeDaemonId, 'get_pending_mesh_events', pendingEventArgs);
|
|
1014
|
+
} catch {
|
|
1015
|
+
// Remote pull is best-effort; the node may be offline. Retry next tick.
|
|
1016
|
+
break; // node unreachable — don't bother with the other id form this tick.
|
|
1017
|
+
}
|
|
1018
|
+
const list = extractPendingEvents(events).filter(e => readNonEmptyString(e?.meshId) === meshId);
|
|
1019
|
+
for (const event of list) {
|
|
1020
|
+
const payload = buildForwardPayloadFromPending(event);
|
|
1021
|
+
if (!payload.event || !payload.meshId) continue;
|
|
1022
|
+
try {
|
|
1023
|
+
handleMeshForwardEvent(components, payload);
|
|
1024
|
+
} catch { /* best-effort re-inject */ }
|
|
1025
|
+
}
|
|
1026
|
+
}
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
|
|
1030
|
+
// Pull the read_chat payload out of whatever envelope the transport returned.
|
|
1031
|
+
// A local commandHandler.handle() returns the CommandResult directly; a remote
|
|
1032
|
+
// dispatchMeshCommand returns it possibly wrapped in { payload } / { result }.
|
|
1033
|
+
function unwrapReadChatPayload(raw: unknown): Record<string, unknown> | null {
|
|
1034
|
+
let cursor: unknown = raw;
|
|
1035
|
+
for (let depth = 0; depth < 4 && cursor && typeof cursor === 'object'; depth++) {
|
|
1036
|
+
const record = cursor as Record<string, unknown>;
|
|
1037
|
+
if (Array.isArray(record.messages)) return record;
|
|
1038
|
+
if (record.payload && typeof record.payload === 'object') { cursor = record.payload; continue; }
|
|
1039
|
+
if (record.result && typeof record.result === 'object') { cursor = record.result; continue; }
|
|
1040
|
+
if (record.data && typeof record.data === 'object') { cursor = record.data; continue; }
|
|
1041
|
+
break;
|
|
1042
|
+
}
|
|
1043
|
+
return cursor && typeof cursor === 'object' ? cursor as Record<string, unknown> : null;
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
function readChatPayloadStatus(payload: Record<string, unknown> | null): string {
|
|
1047
|
+
return readNonEmptyString(payload?.status).toLowerCase();
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
// PHASE 4 helper. For every active (non-terminal) direct dispatch this daemon
|
|
1051
|
+
// hosts, confirm the worker session is idle via a read_chat and — if a final
|
|
1052
|
+
// assistant summary is present but no terminal ledger exists for that dispatch —
|
|
1053
|
+
// synthesize the missing completion through reconcileDirectDispatchCompletionFromTranscript.
|
|
1054
|
+
//
|
|
1055
|
+
// read_chat is resolved against the target node: a node on THIS daemon is read
|
|
1056
|
+
// through the local commandHandler; a remote node is read over P2P via
|
|
1057
|
+
// dispatchMeshCommand. Both yield the same { messages, status, providerSessionId }
|
|
1058
|
+
// shape. We only synthesize when the session reports idle AND a final assistant
|
|
1059
|
+
// message exists — the same evidence bar the MCP poll path uses — so an actively
|
|
1060
|
+
// generating worker is never falsely completed. The reconcile itself is idempotent.
|
|
1061
|
+
async function reconcileUnterminatedDirectDispatches(
|
|
1062
|
+
components: DaemonComponents,
|
|
1063
|
+
mesh: LocalMeshEntry,
|
|
1064
|
+
selfIds: string[],
|
|
1065
|
+
localDaemonId: string | undefined,
|
|
1066
|
+
): Promise<void> {
|
|
1067
|
+
const dispatches = getActiveDirectDispatches(mesh.id);
|
|
1068
|
+
if (dispatches.length === 0) return; // cheap exit — nothing dispatched, nothing to reconcile
|
|
1069
|
+
|
|
1070
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
1071
|
+
const nodeById = new Map(mesh.nodes.map(n => [n.id, n] as const));
|
|
1072
|
+
|
|
1073
|
+
for (const dispatch of dispatches) {
|
|
1074
|
+
const sessionId = readNonEmptyString(dispatch.sessionId);
|
|
1075
|
+
const nodeId = readNonEmptyString(dispatch.nodeId);
|
|
1076
|
+
const taskId = readNonEmptyString(dispatch.taskId);
|
|
1077
|
+
if (!sessionId || !nodeId || !taskId) continue;
|
|
1078
|
+
|
|
1079
|
+
const node = nodeById.get(nodeId);
|
|
1080
|
+
const nodeDaemonId = readNonEmptyString(node?.daemonId);
|
|
1081
|
+
// A node is local when it has no daemonId, names this daemon, or actually
|
|
1082
|
+
// has a live instance here. Anything else is reached over P2P.
|
|
1083
|
+
const isLocalNode = !nodeDaemonId
|
|
1084
|
+
|| daemonIdListIncludes(selfIds, nodeDaemonId)
|
|
1085
|
+
|| daemonIdsEquivalent(nodeDaemonId, localDaemonId)
|
|
1086
|
+
|| !!components.instanceManager.getInstance(sessionId);
|
|
1087
|
+
|
|
1088
|
+
const providerType = readNonEmptyString(dispatch.providerType);
|
|
1089
|
+
const readArgs: Record<string, unknown> = {
|
|
1090
|
+
sessionId,
|
|
1091
|
+
targetSessionId: sessionId,
|
|
1092
|
+
tailLimit: 10,
|
|
1093
|
+
...(node?.workspace ? { workspace: node.workspace } : {}),
|
|
1094
|
+
...(providerType ? { agentType: providerType, providerType } : {}),
|
|
1095
|
+
};
|
|
1096
|
+
|
|
1097
|
+
let payload: Record<string, unknown> | null = null;
|
|
1098
|
+
try {
|
|
1099
|
+
if (isLocalNode) {
|
|
1100
|
+
const result = await components.commandHandler.handle('read_chat', readArgs);
|
|
1101
|
+
if (result && (result as { success?: boolean }).success === false) continue;
|
|
1102
|
+
payload = unwrapReadChatPayload(result);
|
|
1103
|
+
} else if (dispatchMeshCommand) {
|
|
1104
|
+
const result = await dispatchMeshCommand(nodeDaemonId, 'read_chat', readArgs);
|
|
1105
|
+
payload = unwrapReadChatPayload(result);
|
|
1106
|
+
if (payload && (payload as { success?: boolean }).success === false) continue;
|
|
1107
|
+
} else {
|
|
1108
|
+
continue; // remote node but no P2P transport — can't read; retry next tick
|
|
1109
|
+
}
|
|
1110
|
+
} catch {
|
|
1111
|
+
continue; // best-effort; session may be gone or node offline — retry next tick
|
|
1112
|
+
}
|
|
1113
|
+
if (!payload) continue;
|
|
1114
|
+
|
|
1115
|
+
// Only act on a session that has actually settled to idle. A generating /
|
|
1116
|
+
// waiting_approval session is mid-turn — synthesizing a completion now would
|
|
1117
|
+
// be wrong. (idle is the only status the MCP poll path reconciles too.)
|
|
1118
|
+
if (readChatPayloadStatus(payload) !== 'idle') continue;
|
|
1119
|
+
|
|
1120
|
+
const messages = Array.isArray(payload.messages) ? payload.messages as ChatMessage[] : [];
|
|
1121
|
+
const evidence = extractFinalAssistantSummaryEvidence(messages);
|
|
1122
|
+
if (!evidence.finalSummary) continue; // no assistant result yet — nothing to attribute
|
|
1123
|
+
|
|
1124
|
+
// STALE-SUMMARY guard (modal-parked / reused-session misattribution): a direct
|
|
1125
|
+
// dispatch frequently reuses a session that already ran a PRIOR task. read_chat
|
|
1126
|
+
// returns the tail of the WHOLE session, so extractFinalAssistantSummaryEvidence
|
|
1127
|
+
// picks the latest user-facing assistant message — which, for a task that has
|
|
1128
|
+
// barely started (the session momentarily reads idle between turns), is the prior
|
|
1129
|
+
// task's final summary. The downstream reconcile proves the summary is after the
|
|
1130
|
+
// LEDGER task_dispatched entry; here we additionally have the AUTHORITATIVE per-task
|
|
1131
|
+
// dispatchedAt (the dispatch-store row, immune to ledger-ordering quirks), so when
|
|
1132
|
+
// the selected transcript message is provably BEFORE this task's own dispatch we
|
|
1133
|
+
// refuse it outright — it is a prior task's summary, not this task's output (the
|
|
1134
|
+
// 2843ms-duration stale-summary bug where task 2e3f501e copy-pasted 4eca2d9d's
|
|
1135
|
+
// summary). When the message carries no usable timestamp we do NOT block here: the
|
|
1136
|
+
// downstream reconcile already rejects a non-JSON summary it cannot prove is
|
|
1137
|
+
// post-dispatch (transcript_not_proven_after_dispatch), and a structured
|
|
1138
|
+
// final_summary_json is self-attributing — so a timeless provider is not
|
|
1139
|
+
// over-blocked while the provable-stale case is still caught.
|
|
1140
|
+
const dispatchedAtMs = Date.parse(readNonEmptyString(dispatch.dispatchedAt));
|
|
1141
|
+
const transcriptAtMs = Date.parse(evidence.transcriptMessageAt ?? '');
|
|
1142
|
+
if (Number.isFinite(dispatchedAtMs) && Number.isFinite(transcriptAtMs) && transcriptAtMs < dispatchedAtMs) {
|
|
1143
|
+
LOG.info('MeshReconcile', `Stale-summary guard: skipping transcript reconcile for task ${taskId} on node ${nodeId} (mesh ${mesh.id}) — final assistant message (${evidence.transcriptMessageAt}) predates this task's dispatch (${dispatch.dispatchedAt}); it is a prior task's summary`);
|
|
1144
|
+
traceMeshEventDrop('reconcile_stale_summary_before_dispatch', {
|
|
1145
|
+
taskId, sessionId, nodeId, meshId: mesh.id, event: 'agent:generating_completed',
|
|
1146
|
+
}, `transcriptAt=${evidence.transcriptMessageAt} < dispatchedAt=${dispatch.dispatchedAt}`);
|
|
1147
|
+
continue;
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
const providerSessionId = readNonEmptyString(payload.providerSessionId);
|
|
1151
|
+
const coordinatorDaemonId = selfIds.find(id => !!id);
|
|
1152
|
+
try {
|
|
1153
|
+
const result = reconcileDirectDispatchCompletionFromTranscript({
|
|
1154
|
+
meshId: mesh.id,
|
|
1155
|
+
nodeId,
|
|
1156
|
+
sessionId,
|
|
1157
|
+
providerType: providerType || undefined,
|
|
1158
|
+
providerSessionId: providerSessionId || undefined,
|
|
1159
|
+
taskId,
|
|
1160
|
+
finalSummary: evidence.finalSummary,
|
|
1161
|
+
...(evidence.transcriptMessageAt ? { transcriptMessageAt: evidence.transcriptMessageAt } : {}),
|
|
1162
|
+
...(coordinatorDaemonId ? { targetCoordinatorDaemonId: coordinatorDaemonId } : {}),
|
|
1163
|
+
source: 'daemon_reconcile_transcript_completion',
|
|
1164
|
+
});
|
|
1165
|
+
if (result.reconciled) {
|
|
1166
|
+
LOG.info('MeshReconcile', `Synthesized missing completion (${result.kind}) for task ${taskId} on node ${nodeId} (mesh ${mesh.id})`);
|
|
1167
|
+
}
|
|
1168
|
+
} catch (e: any) {
|
|
1169
|
+
LOG.warn('MeshReconcile', `Transcript completion reconcile threw for task ${taskId}: ${e?.message || e}`);
|
|
1170
|
+
}
|
|
1171
|
+
}
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
// PHASE 5 helper. Build the live-node view (mesh.nodes decorated with each node's live
|
|
1175
|
+
// session list) and run the shared prune core in execute mode with the conservative age gate.
|
|
1176
|
+
//
|
|
1177
|
+
// Orphan detection needs the SAME live-session evidence the manual MCP prune uses: a node still
|
|
1178
|
+
// in mesh.nodes whose session list no longer contains the dispatched sessionId is "session not
|
|
1179
|
+
// present" (prunable); a node missing from mesh.nodes entirely is "node no longer in live mesh"
|
|
1180
|
+
// (prunable). We obtain live sessions per node via get_status_metadata — local nodes through the
|
|
1181
|
+
// local commandHandler, remote nodes over P2P (dispatchMeshCommand) — exactly the transports
|
|
1182
|
+
// PHASE 4 already uses. A node we cannot probe (offline) keeps an empty session list; combined
|
|
1183
|
+
// with the age gate that only matters once the orphan is genuinely old.
|
|
1184
|
+
//
|
|
1185
|
+
// O(1) fast exit: when there are no active direct dispatches at all there is nothing to prune,
|
|
1186
|
+
// so we skip the (per-node) status probes entirely — an idle mesh costs one indexed query.
|
|
1187
|
+
async function autoPruneStaleDirectDispatches(
|
|
1188
|
+
components: DaemonComponents,
|
|
1189
|
+
mesh: LocalMeshEntry,
|
|
1190
|
+
selfIds: string[],
|
|
1191
|
+
localDaemonId: string | undefined,
|
|
1192
|
+
minAgeMs: number,
|
|
1193
|
+
): Promise<void> {
|
|
1194
|
+
const directDispatches = getActiveDirectDispatches(mesh.id);
|
|
1195
|
+
if (directDispatches.length === 0) return; // nothing dispatched → nothing to prune
|
|
1196
|
+
|
|
1197
|
+
const liveNodes = await collectLiveNodesWithSessions(components, mesh, selfIds, localDaemonId);
|
|
1198
|
+
|
|
1199
|
+
const result = pruneStaleDirectDispatches({
|
|
1200
|
+
meshId: mesh.id,
|
|
1201
|
+
queue: getQueue(mesh.id),
|
|
1202
|
+
ledgerEntries: readLedgerEntries(mesh.id, { tail: 500 }),
|
|
1203
|
+
directDispatches,
|
|
1204
|
+
nodes: liveNodes,
|
|
1205
|
+
execute: true,
|
|
1206
|
+
minAgeMs,
|
|
1207
|
+
source: 'daemon_reconcile_auto_prune',
|
|
1208
|
+
});
|
|
1209
|
+
|
|
1210
|
+
// Log only when something was actually pruned — silence on the common no-op tick.
|
|
1211
|
+
if (result.prunedCount > 0) {
|
|
1212
|
+
LOG.info('MeshReconcile', `Auto-pruned ${result.prunedCount} orphaned direct dispatch record(s) for mesh ${mesh.id}`);
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
|
|
1216
|
+
// Probe each node for its live session list (get_status_metadata) and return mesh.nodes
|
|
1217
|
+
// decorated with a `sessions` array — the shape buildMeshActiveWork / sessionStatusFromNodes
|
|
1218
|
+
// consume to decide whether a dispatched session is still present. Best-effort: an unreachable
|
|
1219
|
+
// node yields an empty session list rather than throwing.
|
|
1220
|
+
async function collectLiveNodesWithSessions(
|
|
1221
|
+
components: DaemonComponents,
|
|
1222
|
+
mesh: LocalMeshEntry,
|
|
1223
|
+
selfIds: string[],
|
|
1224
|
+
localDaemonId: string | undefined,
|
|
1225
|
+
): Promise<any[]> {
|
|
1226
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
1227
|
+
return Promise.all(mesh.nodes.map(async (node) => {
|
|
1228
|
+
const nodeDaemonId = readNonEmptyString(node.daemonId);
|
|
1229
|
+
const isLocalNode = !nodeDaemonId
|
|
1230
|
+
|| daemonIdListIncludes(selfIds, nodeDaemonId)
|
|
1231
|
+
|| daemonIdsEquivalent(nodeDaemonId, localDaemonId);
|
|
1232
|
+
let statusResult: unknown;
|
|
1233
|
+
try {
|
|
1234
|
+
if (isLocalNode) {
|
|
1235
|
+
statusResult = await components.commandHandler.handle('get_status_metadata', {});
|
|
1236
|
+
} else if (dispatchMeshCommand) {
|
|
1237
|
+
statusResult = await dispatchMeshCommand(nodeDaemonId, 'get_status_metadata', {});
|
|
1238
|
+
} else {
|
|
1239
|
+
return node; // remote node, no P2P transport — leave undecorated
|
|
1240
|
+
}
|
|
1241
|
+
} catch {
|
|
1242
|
+
return node; // unreachable — leave undecorated (empty session list)
|
|
1243
|
+
}
|
|
1244
|
+
const sessions = extractStatusMetadataSessions(statusResult);
|
|
1245
|
+
return sessions.length > 0 ? { ...node, sessions } : node;
|
|
1246
|
+
}));
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
// Pull the live session list out of a get_status_metadata result, tolerating the same
|
|
1250
|
+
// envelope shapes unwrapReadChatPayload handles (direct CommandResult or { payload }/{ result }).
|
|
1251
|
+
function extractStatusMetadataSessions(raw: unknown): any[] {
|
|
1252
|
+
let cursor: unknown = raw;
|
|
1253
|
+
for (let depth = 0; depth < 4 && cursor && typeof cursor === 'object'; depth++) {
|
|
1254
|
+
const record = cursor as Record<string, unknown>;
|
|
1255
|
+
const status = record.status && typeof record.status === 'object' ? record.status as Record<string, unknown> : undefined;
|
|
1256
|
+
if (status && Array.isArray(status.sessions)) return status.sessions;
|
|
1257
|
+
if (Array.isArray(record.sessions)) return record.sessions;
|
|
1258
|
+
if (record.payload && typeof record.payload === 'object') { cursor = record.payload; continue; }
|
|
1259
|
+
if (record.result && typeof record.result === 'object') { cursor = record.result; continue; }
|
|
1260
|
+
if (record.data && typeof record.data === 'object') { cursor = record.data; continue; }
|
|
1261
|
+
break;
|
|
1262
|
+
}
|
|
1263
|
+
return [];
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
function extractPendingEvents(raw: unknown): any[] {
|
|
1267
|
+
if (Array.isArray(raw)) return raw;
|
|
1268
|
+
if (raw && typeof raw === 'object') {
|
|
1269
|
+
const events = (raw as Record<string, unknown>).events;
|
|
1270
|
+
if (Array.isArray(events)) return events;
|
|
1271
|
+
}
|
|
1272
|
+
return [];
|
|
1273
|
+
}
|
|
1274
|
+
|
|
1275
|
+
// Flatten a queued PendingMeshCoordinatorEvent into the flat payload shape
|
|
1276
|
+
// handleMeshForwardEvent expects (mirrors the MCP buildMeshForwardPayloadFromPendingEvent).
|
|
1277
|
+
function buildForwardPayloadFromPending(event: any): Record<string, unknown> {
|
|
1278
|
+
const metadata = event?.metadataEvent && typeof event.metadataEvent === 'object'
|
|
1279
|
+
? event.metadataEvent as Record<string, unknown>
|
|
1280
|
+
: {};
|
|
1281
|
+
return {
|
|
1282
|
+
event: readNonEmptyString(event?.event),
|
|
1283
|
+
meshId: readNonEmptyString(event?.meshId),
|
|
1284
|
+
nodeId: readNonEmptyString(event?.nodeId) || readNonEmptyString(metadata.meshNodeId),
|
|
1285
|
+
workspace: readNonEmptyString(event?.workspace) || readNonEmptyString(metadata.workspace),
|
|
1286
|
+
// Preserve the originating coordinator session id across the relay. It is normally
|
|
1287
|
+
// carried inside metadataEvent.meshCoordinatorSessionId (spread below), but pass the
|
|
1288
|
+
// top-level field through explicitly too so the handleMeshForwardEvent whitelist
|
|
1289
|
+
// recovers it regardless of which carrier the producing daemon used.
|
|
1290
|
+
...(readNonEmptyString(event?.targetCoordinatorSessionId)
|
|
1291
|
+
? { targetCoordinatorSessionId: readNonEmptyString(event.targetCoordinatorSessionId) }
|
|
1292
|
+
: {}),
|
|
1293
|
+
...metadata,
|
|
1294
|
+
// NOTIF-MISS (FIX 3): surface the dispatch task id at the TOP LEVEL so the relay's
|
|
1295
|
+
// received-stage trace (and buildRelayMetadataEvent) recovers it regardless of which
|
|
1296
|
+
// carrier the producing daemon used. The metadata spread above may carry the id only as
|
|
1297
|
+
// `meshActiveTaskId` (a worker provider event), leaving top-level `taskId` unset and the
|
|
1298
|
+
// received stage rendering `task=-`. Resolve both carriers into an explicit `taskId` so
|
|
1299
|
+
// dedup stays task-scoped end-to-end. Only set when a non-empty id exists (no clobber to
|
|
1300
|
+
// undefined when neither is present).
|
|
1301
|
+
...((): Record<string, unknown> => {
|
|
1302
|
+
const tid = readNonEmptyString(metadata.taskId) || readNonEmptyString(metadata.meshActiveTaskId);
|
|
1303
|
+
return tid ? { taskId: tid } : {};
|
|
1304
|
+
})(),
|
|
1305
|
+
};
|
|
1306
|
+
}
|
|
1307
|
+
|
|
1308
|
+
interface ReconcileLoopHandle {
|
|
1309
|
+
stop(): void;
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
// Start the periodic reconcile loop. Returns a handle with stop() for shutdown.
|
|
1313
|
+
export function setupMeshReconcileLoop(components: DaemonComponents): ReconcileLoopHandle {
|
|
1314
|
+
const intervalMs = resolveReconcileIntervalMs();
|
|
1315
|
+
let running = false;
|
|
1316
|
+
const timer = setInterval(() => {
|
|
1317
|
+
if (running) return; // never overlap ticks
|
|
1318
|
+
running = true;
|
|
1319
|
+
void runMeshReconcileTick(components)
|
|
1320
|
+
.catch((e: any) => LOG.warn('MeshReconcile', `Reconcile tick error: ${e?.message || e}`))
|
|
1321
|
+
.finally(() => { running = false; });
|
|
1322
|
+
}, intervalMs);
|
|
1323
|
+
// Don't keep the process alive solely for this timer.
|
|
1324
|
+
if (typeof timer.unref === 'function') timer.unref();
|
|
1325
|
+
LOG.info('MeshReconcile', `Mesh reconcile loop started (interval ${intervalMs}ms)`);
|
|
1326
|
+
return {
|
|
1327
|
+
stop() {
|
|
1328
|
+
clearInterval(timer);
|
|
1329
|
+
LOG.info('MeshReconcile', 'Mesh reconcile loop stopped');
|
|
1330
|
+
},
|
|
1331
|
+
};
|
|
1332
|
+
}
|