@adhdev/daemon-core 0.9.82-rc.34 → 0.9.82-rc.341
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/daemon-lifecycle.d.ts +8 -0
- package/dist/boot/process-hardening.d.ts +50 -0
- package/dist/build-info.d.ts +37 -0
- package/dist/chat/source-machine.d.ts +166 -0
- package/dist/chat/source-resolver.d.ts +104 -0
- package/dist/chat/subscription-updates.d.ts +1 -0
- package/dist/cli-adapter-types.d.ts +46 -2
- package/dist/cli-adapters/cli-script-runner.d.ts +117 -0
- package/dist/cli-adapters/cli-state-engine.d.ts +198 -0
- package/dist/cli-adapters/provider-cli-adapter.d.ts +116 -64
- package/dist/cli-adapters/provider-cli-parse.d.ts +4 -0
- package/dist/cli-adapters/provider-cli-runtime.d.ts +34 -0
- package/dist/cli-adapters/provider-cli-shared.d.ts +53 -0
- package/dist/cli-adapters/raw-terminal-io.d.ts +37 -0
- package/dist/cli-adapters/resolve-executable.d.ts +14 -0
- package/dist/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +6 -3
- package/dist/cli-adapters/terminal-backends/types.d.ts +10 -2
- package/dist/cli-adapters/terminal-screen.d.ts +4 -11
- package/dist/commands/cli-manager.d.ts +2 -1
- package/dist/commands/handler.d.ts +110 -0
- package/dist/commands/mesh-coordinator.d.ts +85 -1
- package/dist/commands/router.d.ts +308 -1
- package/dist/commands/upgrade-helper.d.ts +12 -0
- package/dist/config/chat-history.d.ts +9 -0
- package/dist/config/config.d.ts +5 -0
- package/dist/config/mesh-config.d.ts +77 -1
- package/dist/detection/ide-detector.d.ts +13 -0
- package/dist/detection/win32-ide-version.d.ts +37 -0
- package/dist/git/change-impact-config.d.ts +159 -0
- package/dist/git/git-commands.d.ts +11 -1
- package/dist/git/git-diff.d.ts +6 -0
- package/dist/git/git-executor.d.ts +11 -0
- package/dist/git/git-status.d.ts +23 -0
- package/dist/git/git-types.d.ts +2 -50
- package/dist/git/index.d.ts +2 -0
- package/dist/index.d.ts +52 -13
- package/dist/index.js +42012 -17098
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +41683 -16890
- package/dist/index.mjs.map +1 -1
- package/dist/installer.d.ts +1 -4
- package/dist/ipc/local-ipc-server.d.ts +91 -0
- package/dist/launch.d.ts +1 -1
- package/dist/logging/async-batch-writer.d.ts +10 -0
- package/dist/logging/log-redactor.d.ts +24 -0
- package/dist/logging/log-tail-reader.d.ts +46 -0
- package/dist/mesh/contracts.d.ts +164 -0
- package/dist/mesh/coordinator-prompt.d.ts +36 -0
- package/dist/mesh/coordinator-registry.d.ts +59 -0
- package/dist/mesh/mesh-active-work.d.ts +174 -0
- package/dist/mesh/mesh-delivery-policy.d.ts +131 -0
- package/dist/mesh/mesh-events-coordinator.d.ts +169 -0
- package/dist/mesh/mesh-events-pending.d.ts +45 -0
- package/dist/mesh/mesh-events-stale.d.ts +40 -0
- package/dist/mesh/mesh-events-utils.d.ts +66 -0
- package/dist/mesh/mesh-events.d.ts +6 -49
- package/dist/mesh/mesh-fast-forward.d.ts +81 -0
- package/dist/mesh/mesh-host-ownership.d.ts +9 -0
- package/dist/mesh/mesh-init.d.ts +86 -0
- package/dist/mesh/mesh-ledger-reconciliation.d.ts +23 -3
- package/dist/mesh/mesh-ledger.d.ts +77 -1
- package/dist/mesh/mesh-missions.d.ts +116 -0
- package/dist/mesh/mesh-reconcile-loop.d.ts +7 -0
- package/dist/mesh/mesh-refine-batch.d.ts +68 -0
- package/dist/mesh/mesh-refine-status.d.ts +62 -0
- package/dist/mesh/mesh-review-inbox.d.ts +90 -0
- package/dist/mesh/mesh-routing.d.ts +70 -0
- package/dist/mesh/mesh-runtime-store.d.ts +418 -0
- package/dist/mesh/mesh-task-stats.d.ts +49 -0
- package/dist/mesh/mesh-unresolved-forward-outbox.d.ts +30 -0
- package/dist/mesh/mesh-work-queue.d.ts +208 -5
- package/dist/mesh/preview-freshness.d.ts +18 -0
- package/dist/mesh/refine-config.d.ts +216 -0
- package/dist/mesh/worktree-bootstrap-config.d.ts +135 -0
- package/dist/providers/acp-provider-instance.d.ts +2 -0
- package/dist/providers/approval-utils.d.ts +13 -0
- package/dist/providers/chat-message-normalization.d.ts +12 -0
- package/dist/providers/cli-provider-instance.d.ts +76 -3
- package/dist/providers/contracts.d.ts +139 -6
- package/dist/providers/external-sources.d.ts +71 -0
- package/dist/providers/native-history/antigravity-cli-transcript.d.ts +100 -0
- package/dist/providers/native-history/claude-cli-transcript.d.ts +70 -0
- package/dist/providers/native-history/codex-cli-transcript.d.ts +73 -0
- package/dist/providers/native-history/dispatcher.d.ts +27 -0
- package/dist/providers/native-history/hermes-cli-transcript.d.ts +30 -0
- package/dist/providers/native-history/index.d.ts +13 -0
- package/dist/providers/provider-instance-manager.d.ts +13 -0
- package/dist/providers/provider-instance.d.ts +13 -1
- package/dist/providers/provider-loader.d.ts +26 -4
- package/dist/providers/provider-trust.d.ts +31 -0
- package/dist/providers/read-chat-contract.d.ts +29 -0
- package/dist/providers/sdk/v1/builders/acp/detect-status.d.ts +68 -0
- package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +85 -0
- package/dist/providers/sdk/v1/builders/cli/parse-approval-squash.d.ts +59 -0
- package/dist/providers/sdk/v1/builders/cli/parse-approval.d.ts +65 -0
- package/dist/providers/sdk/v1/builders/cli/parse-session.d.ts +91 -0
- package/dist/providers/sdk/v1/builders/cli/visible-region.d.ts +42 -0
- package/dist/providers/sdk/v1/fixture-tooling/format.d.ts +126 -0
- package/dist/providers/sdk/v1/fixture-tooling/index.d.ts +8 -0
- package/dist/providers/sdk/v1/fixture-tooling/replay.d.ts +38 -0
- package/dist/providers/sdk/v1/index.d.ts +30 -0
- package/dist/providers/sdk/v1/sandbox/README-design.d.ts +193 -0
- package/dist/providers/sdk/v1/sandbox/require-whitelist.d.ts +74 -0
- package/dist/providers/sdk/v1/sandbox/script-runner.d.ts +98 -0
- package/dist/providers/sdk/v1/types/cli/index.d.ts +277 -0
- package/dist/providers/sdk/v1/types/common/index.d.ts +203 -0
- package/dist/providers/sdk/v1/validators/index.d.ts +5 -0
- package/dist/providers/sdk/v1/validators/manifest.d.ts +40 -0
- package/dist/providers/sdk/v1/validators/taint.d.ts +52 -0
- package/dist/providers/spec/adapter.d.ts +69 -0
- package/dist/providers/spec/cli-adapter.d.ts +217 -0
- package/dist/providers/spec/evaluator.d.ts +23 -0
- package/dist/providers/spec/fsm-driver.d.ts +306 -0
- package/dist/providers/spec/fsm-evaluator.d.ts +72 -0
- package/dist/providers/spec/fsm-loader.d.ts +14 -0
- package/dist/providers/spec/fsm-types.d.ts +144 -0
- package/dist/providers/spec/native-history-executor.d.ts +41 -0
- package/dist/providers/spec/pre-launch-trust.d.ts +16 -0
- package/dist/providers/spec/route.d.ts +4 -0
- package/dist/providers/spec/types.d.ts +180 -0
- package/dist/providers/status-monitor.d.ts +7 -7
- package/dist/providers/transcript-v2.d.ts +176 -0
- package/dist/providers/types/interactive-prompt.d.ts +84 -0
- package/dist/providers/working-dir.d.ts +17 -0
- package/dist/repo-mesh-types.d.ts +295 -13
- package/dist/sessions/registry.d.ts +6 -0
- package/dist/shared-types-extra.d.ts +2 -4
- package/dist/shared-types.d.ts +59 -2
- package/dist/status/normalize.d.ts +1 -1
- package/dist/status/normalize.js +1 -0
- package/dist/status/normalize.js.map +1 -1
- package/dist/status/normalize.mjs +1 -0
- package/dist/status/normalize.mjs.map +1 -1
- package/dist/status/reporter.d.ts +2 -0
- package/dist/status/snapshot.d.ts +26 -0
- package/dist/system/load-better-sqlite3.d.ts +21 -0
- package/dist/types.d.ts +5 -0
- package/package.json +7 -3
- package/src/agent-stream/poller.ts +2 -3
- package/src/agent-stream/provider-adapter.ts +1 -1
- package/src/boot/daemon-lifecycle.ts +55 -12
- package/src/boot/process-hardening.ts +89 -0
- package/src/build-info.ts +73 -0
- package/src/chat/source-machine.ts +534 -0
- package/src/chat/source-resolver.ts +0 -0
- package/src/chat/subscription-updates.ts +20 -1
- package/src/cli-adapter-types.d.ts +1 -0
- package/src/cli-adapter-types.ts +47 -2
- package/src/cli-adapters/cli-script-runner.ts +421 -0
- package/src/cli-adapters/cli-state-engine.ts +1215 -0
- package/src/cli-adapters/provider-cli-adapter.d.ts +1 -1
- package/src/cli-adapters/provider-cli-adapter.ts +774 -1141
- package/src/cli-adapters/provider-cli-parse.d.ts +1 -0
- package/src/cli-adapters/provider-cli-parse.ts +13 -0
- package/src/cli-adapters/provider-cli-runtime.ts +56 -7
- package/src/cli-adapters/provider-cli-shared.d.ts +4 -0
- package/src/cli-adapters/provider-cli-shared.ts +100 -11
- package/src/cli-adapters/pty-transport.ts +2 -1
- package/src/cli-adapters/raw-terminal-io.ts +252 -0
- package/src/cli-adapters/resolve-executable.ts +90 -0
- package/src/cli-adapters/session-host-transport.ts +2 -1
- package/src/cli-adapters/terminal-backends/ghostty-vt-backend.ts +53 -29
- package/src/cli-adapters/terminal-backends/types.ts +10 -3
- package/src/cli-adapters/terminal-screen.ts +16 -81
- package/src/commands/chat-commands.ts +1823 -62
- package/src/commands/cli-manager.ts +286 -14
- package/src/commands/handler.ts +809 -2
- package/src/commands/mesh-coordinator.ts +332 -122
- package/src/commands/router.ts +7433 -1317
- package/src/commands/stream-commands.ts +8 -0
- package/src/commands/upgrade-helper.ts +81 -30
- package/src/config/chat-history.ts +483 -24
- package/src/config/config.ts +12 -0
- package/src/config/mesh-config.ts +388 -6
- package/src/config/recent-activity.ts +8 -2
- package/src/daemon/dev-auto-implement.ts +3 -2
- package/src/daemon/dev-cli-debug.ts +10 -1
- package/src/detection/cli-detector.ts +28 -9
- package/src/detection/ide-detector.ts +55 -16
- package/src/detection/win32-ide-version.ts +106 -0
- package/src/git/change-impact-config.ts +354 -0
- package/src/git/git-commands.ts +55 -14
- package/src/git/git-diff.ts +53 -0
- package/src/git/git-executor.ts +12 -0
- package/src/git/git-status.ts +574 -43
- package/src/git/git-types.ts +14 -62
- package/src/git/git-worktree.ts +8 -1
- package/src/git/index.ts +16 -0
- package/src/index.ts +146 -12
- package/src/installer.d.ts +1 -1
- package/src/installer.ts +8 -6
- package/src/ipc/local-ipc-server.ts +278 -0
- package/src/launch.d.ts +1 -1
- package/src/launch.ts +37 -28
- package/src/logging/async-batch-writer.ts +55 -0
- package/src/logging/log-redactor.ts +100 -0
- package/src/logging/log-tail-reader.ts +220 -0
- package/src/logging/logger.ts +2 -1
- package/src/mesh/contracts.ts +329 -0
- package/src/mesh/coordinator-prompt.ts +234 -31
- package/src/mesh/coordinator-registry.ts +121 -0
- package/src/mesh/mesh-active-work.ts +623 -0
- package/src/mesh/mesh-delivery-policy.ts +315 -0
- package/src/mesh/mesh-events-coordinator.ts +2144 -0
- package/src/mesh/mesh-events-pending.ts +508 -0
- package/src/mesh/mesh-events-stale.ts +290 -0
- package/src/mesh/mesh-events-utils.ts +300 -0
- package/src/mesh/mesh-events.ts +30 -1035
- package/src/mesh/mesh-fast-forward.ts +839 -0
- package/src/mesh/mesh-host-ownership.ts +73 -0
- package/src/mesh/mesh-init.ts +260 -0
- package/src/mesh/mesh-ledger-reconciliation.ts +12 -5
- package/src/mesh/mesh-ledger.ts +589 -102
- package/src/mesh/mesh-missions.ts +248 -0
- package/src/mesh/mesh-reconcile-loop.ts +822 -0
- package/src/mesh/mesh-refine-batch.ts +197 -0
- package/src/mesh/mesh-refine-status.ts +231 -0
- package/src/mesh/mesh-review-inbox.ts +307 -0
- package/src/mesh/mesh-routing.ts +291 -0
- package/src/mesh/mesh-runtime-store.ts +1626 -0
- package/src/mesh/mesh-task-stats.ts +154 -0
- package/src/mesh/mesh-unresolved-forward-outbox.ts +185 -0
- package/src/mesh/mesh-work-queue.ts +888 -141
- package/src/mesh/preview-freshness.ts +118 -0
- package/src/mesh/refine-config.ts +423 -0
- package/src/mesh/worktree-bootstrap-config.ts +331 -0
- package/src/providers/acp-provider-instance.ts +25 -9
- package/src/providers/approval-utils.d.ts +4 -0
- package/src/providers/approval-utils.ts +47 -5
- package/src/providers/chat-message-normalization.ts +45 -5
- package/src/providers/cli-provider-instance.ts +1038 -99
- package/src/providers/contracts.d.ts +55 -0
- package/src/providers/contracts.ts +150 -6
- package/src/providers/extension-provider-instance.ts +12 -7
- package/src/providers/external-sources.ts +218 -0
- package/src/providers/ide-provider-instance.ts +35 -12
- package/src/providers/native-history/antigravity-cli-transcript.ts +643 -0
- package/src/providers/native-history/claude-cli-transcript.ts +396 -0
- package/src/providers/native-history/codex-cli-transcript.ts +479 -0
- package/src/providers/native-history/dispatcher.ts +340 -0
- package/src/providers/native-history/hermes-cli-transcript.ts +230 -0
- package/src/providers/native-history/index.ts +30 -0
- package/src/providers/provider-instance-manager.ts +30 -0
- package/src/providers/provider-instance.ts +10 -1
- package/src/providers/provider-loader.ts +670 -50
- package/src/providers/provider-schema.ts +87 -14
- package/src/providers/provider-trust.ts +114 -0
- package/src/providers/read-chat-contract.ts +76 -16
- package/src/providers/sdk/README.md +49 -0
- package/src/providers/sdk/v1/builders/acp/detect-status.ts +144 -0
- package/src/providers/sdk/v1/builders/cli/detect-status.ts +262 -0
- package/src/providers/sdk/v1/builders/cli/parse-approval-squash.ts +158 -0
- package/src/providers/sdk/v1/builders/cli/parse-approval.ts +250 -0
- package/src/providers/sdk/v1/builders/cli/parse-session.ts +276 -0
- package/src/providers/sdk/v1/builders/cli/visible-region.ts +143 -0
- package/src/providers/sdk/v1/fixture-tooling/format.ts +130 -0
- package/src/providers/sdk/v1/fixture-tooling/index.ts +22 -0
- package/src/providers/sdk/v1/fixture-tooling/replay.ts +352 -0
- package/src/providers/sdk/v1/index.ts +152 -0
- package/src/providers/sdk/v1/sandbox/README-design.ts +195 -0
- package/src/providers/sdk/v1/sandbox/require-whitelist.ts +472 -0
- package/src/providers/sdk/v1/sandbox/script-runner.ts +150 -0
- package/src/providers/sdk/v1/schemas/cli/provider.schema.json +504 -0
- package/src/providers/sdk/v1/schemas/primitives/acp-session-protocol-v1.json +131 -0
- package/src/providers/sdk/v1/schemas/primitives/native-history-codex-rollout-v1.json +66 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-approval-squash-v1.json +91 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-assistant-block-v1.json +91 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-cue-ordering-v1.json +47 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-dispatch-order-v1.json +32 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-footer-chrome-v1.json +42 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-index-finder-v1.json +27 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-modal-v1.json +125 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-prompt-marker-v1.json +45 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-session-id-extraction-v1.json +46 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-settled-prompt-v1.json +71 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-spinner-v1.json +83 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-transcript-pty-v1.json +83 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-visible-region-v1.json +57 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-welcome-screen-v1.json +35 -0
- package/src/providers/sdk/v1/types/cli/index.ts +365 -0
- package/src/providers/sdk/v1/types/common/index.ts +229 -0
- package/src/providers/sdk/v1/validators/index.ts +19 -0
- package/src/providers/sdk/v1/validators/manifest.ts +110 -0
- package/src/providers/sdk/v1/validators/taint.ts +309 -0
- package/src/providers/spec/adapter.ts +168 -0
- package/src/providers/spec/cli-adapter.ts +986 -0
- package/src/providers/spec/evaluator.ts +345 -0
- package/src/providers/spec/fsm-driver.ts +1015 -0
- package/src/providers/spec/fsm-evaluator.ts +255 -0
- package/src/providers/spec/fsm-loader.ts +120 -0
- package/src/providers/spec/fsm-types.ts +238 -0
- package/src/providers/spec/native-history-executor.ts +1136 -0
- package/src/providers/spec/pre-launch-trust.ts +104 -0
- package/src/providers/spec/route.ts +51 -0
- package/src/providers/spec/types.ts +226 -0
- package/src/providers/status-monitor.d.ts +7 -7
- package/src/providers/status-monitor.ts +37 -22
- package/src/providers/transcript-v2.ts +567 -0
- package/src/providers/types/interactive-prompt.ts +536 -0
- package/src/providers/version-archive.ts +64 -24
- package/src/providers/working-dir.ts +23 -0
- package/src/repo-mesh-types.ts +378 -14
- package/src/sessions/registry.ts +6 -0
- package/src/shared-types-extra.ts +2 -4
- package/src/shared-types.d.ts +8 -0
- package/src/shared-types.ts +64 -1
- package/src/status/builders.ts +26 -6
- package/src/status/normalize.ts +2 -0
- package/src/status/reporter.ts +19 -1
- package/src/status/snapshot.ts +95 -26
- package/src/system/host-memory.ts +29 -12
- package/src/system/load-better-sqlite3.ts +68 -0
- package/src/types.ts +5 -0
- package/dist/cli-adapters/terminal-backends/xterm-backend.d.ts +0 -17
- package/dist/mesh/mesh-sync.d.ts +0 -53
- package/src/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +0 -16
- package/src/cli-adapters/terminal-backends/xterm-backend.ts +0 -97
- package/src/mesh/mesh-sync.ts +0 -111
|
@@ -0,0 +1,2144 @@
|
|
|
1
|
+
import { existsSync } from 'fs';
|
|
2
|
+
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
3
|
+
import { loadConfig } from '../config/config.js';
|
|
4
|
+
import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
|
|
5
|
+
import { detectCLI } from '../detection/cli-detector.js';
|
|
6
|
+
import { LOG } from '../logging/logger.js';
|
|
7
|
+
import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
|
|
8
|
+
import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
|
|
9
|
+
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents } from './mesh-work-queue.js';
|
|
10
|
+
import { fastForwardMeshNode } from './mesh-fast-forward.js';
|
|
11
|
+
import { createSessionDelivery, markSessionDeliveriesTerminal, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
|
|
12
|
+
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
13
|
+
import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
14
|
+
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
15
|
+
import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent, isUnroutableDelegateRejection } from './mesh-routing.js';
|
|
16
|
+
import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUnresolvedDelegateForward } from './mesh-unresolved-forward-outbox.js';
|
|
17
|
+
import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
18
|
+
import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
19
|
+
import { normalizeMeshNodeId, meshNodeIdMatches, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
20
|
+
import {
|
|
21
|
+
findRecentTerminalLedgerEvidence,
|
|
22
|
+
hasDispatchAfterTerminal,
|
|
23
|
+
hasUnterminalDirectDispatchLedgerEntry,
|
|
24
|
+
buildNoProgressCompletionReconciliation,
|
|
25
|
+
} from './mesh-events-stale.js';
|
|
26
|
+
import {
|
|
27
|
+
buildMeshSystemMessage,
|
|
28
|
+
readNonEmptyString,
|
|
29
|
+
readRecord,
|
|
30
|
+
resolveEventSessionId,
|
|
31
|
+
readRefineJobId,
|
|
32
|
+
readWorkerResultMetadata,
|
|
33
|
+
resolveMeshSurfacedSessionPreview,
|
|
34
|
+
} from './mesh-events-utils.js';
|
|
35
|
+
|
|
36
|
+
// The set of coordinator-daemon ids this daemon answers to when draining the
|
|
37
|
+
// pending-events queue (canonical status id + bare machineId). Mirrors
|
|
38
|
+
// resolveCoordinatorDaemonIds in mesh-reconcile-loop — a unicast event may be
|
|
39
|
+
// stamped with either id depending on which dispatch path created the worker.
|
|
40
|
+
function resolveCoordinatorDrainDaemonIds(components: DaemonComponents): string[] {
|
|
41
|
+
const ids = new Set<string>();
|
|
42
|
+
const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
|
|
43
|
+
if (statusInstanceId) ids.add(statusInstanceId);
|
|
44
|
+
const machineId = readNonEmptyString(loadConfig().machineId);
|
|
45
|
+
if (machineId) ids.add(machineId);
|
|
46
|
+
return [...ids];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// ---------------------------------------------------------------------------
|
|
50
|
+
// Remote Node Idle Session Tracking
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
const REMOTE_IDLE_SESSION_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
|
53
|
+
|
|
54
|
+
// ---------------------------------------------------------------------------
|
|
55
|
+
// Workspace-to-mesh lookup cache
|
|
56
|
+
// ---------------------------------------------------------------------------
|
|
57
|
+
const meshByWorkspaceCache = new Map<string, { mesh: any; cachedAt: number }>();
|
|
58
|
+
const MESH_WORKSPACE_CACHE_TTL_MS = 5_000;
|
|
59
|
+
const IDLE_AUTO_FAST_FORWARD_THROTTLE_MS = 30 * 60 * 1000;
|
|
60
|
+
const idleAutoFastForwardLastAttempt = new Map<string, number>();
|
|
61
|
+
|
|
62
|
+
function getCachedMeshByWorkspace(workspace: string): any {
|
|
63
|
+
const now = Date.now();
|
|
64
|
+
const cached = meshByWorkspaceCache.get(workspace);
|
|
65
|
+
if (cached && now - cached.cachedAt < MESH_WORKSPACE_CACHE_TTL_MS) return cached.mesh;
|
|
66
|
+
const mesh = getMeshByRepo(workspace);
|
|
67
|
+
meshByWorkspaceCache.set(workspace, { mesh, cachedAt: now });
|
|
68
|
+
return mesh;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function __resetIdleAutoFastForwardForTests(): void {
|
|
72
|
+
idleAutoFastForwardLastAttempt.clear();
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function __resetMeshWorkspaceCacheForTests(): void {
|
|
76
|
+
meshByWorkspaceCache.clear();
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function sweepExpiredRemoteIdleSessions(): void {
|
|
80
|
+
try {
|
|
81
|
+
MeshRuntimeStore.getInstance().pruneExpiredRemoteIdleSessions();
|
|
82
|
+
} catch { /* best-effort */ }
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function getMeshWithCache(components: DaemonComponents, meshId: string): any | undefined {
|
|
86
|
+
const localMesh = getMesh(meshId);
|
|
87
|
+
if (localMesh) return localMesh;
|
|
88
|
+
return components.router?.getCachedInlineMesh(meshId);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS = 30 * 60 * 1000;
|
|
92
|
+
|
|
93
|
+
function isIntentionalCleanupStopMetadata(event: Record<string, unknown>): boolean {
|
|
94
|
+
return event.intentional === true
|
|
95
|
+
|| event.intentionalStop === true
|
|
96
|
+
|| event.operatorCleanup === true
|
|
97
|
+
|| event.reason === 'operator_cleanup'
|
|
98
|
+
|| event.stopReason === 'operator_cleanup'
|
|
99
|
+
|| event.cleanupReason === 'operator_cleanup'
|
|
100
|
+
|| event.source === 'mesh_cleanup_sessions'
|
|
101
|
+
|| event.source === 'mesh_remove_node';
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
function hasRecentIntentionalCleanupStop(meshId: string, sessionId?: string, nodeId?: string): boolean {
|
|
105
|
+
if (!sessionId && !nodeId) return false;
|
|
106
|
+
const cutoff = Date.now() - INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS;
|
|
107
|
+
const entries = readLedgerEntries(meshId, { tail: 200 });
|
|
108
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
109
|
+
const entry = entries[i];
|
|
110
|
+
const timestamp = new Date(entry.timestamp).getTime();
|
|
111
|
+
if (!Number.isNaN(timestamp) && timestamp < cutoff) break;
|
|
112
|
+
if (!isIntentionalCleanupStopEntry(entry)) continue;
|
|
113
|
+
if (sessionId && entry.sessionId === sessionId) return true;
|
|
114
|
+
// Normalized node-id match (P4): the cleanup-stop entry's node id may be stored as
|
|
115
|
+
// `nodeId` or `node_id` and the `nodeId` arg can be in either form — a raw `===`
|
|
116
|
+
// would miss a genuine intentional-cleanup entry and fail to suppress the stop event.
|
|
117
|
+
if (!sessionId && nodeId && meshNodeIdMatches(entry as unknown as MeshNodeIdentified, nodeId)) return true;
|
|
118
|
+
}
|
|
119
|
+
return false;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function shouldSuppressIntentionalCleanupStop(args: {
|
|
123
|
+
event: string;
|
|
124
|
+
meshId: string;
|
|
125
|
+
metadataEvent: Record<string, unknown>;
|
|
126
|
+
sessionId?: string;
|
|
127
|
+
nodeId?: string;
|
|
128
|
+
}): boolean {
|
|
129
|
+
if (args.event !== 'agent:stopped' && args.event !== 'monitor:no_progress') return false;
|
|
130
|
+
if (isIntentionalCleanupStopMetadata(args.metadataEvent)) return true;
|
|
131
|
+
return hasRecentIntentionalCleanupStop(args.meshId, args.sessionId, args.nodeId);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const RECENT_COMPLETION_FINGERPRINT_TTL_MS = 10 * 60 * 1000;
|
|
135
|
+
|
|
136
|
+
function hasFingerprintSeen(fingerprint: string): boolean {
|
|
137
|
+
try {
|
|
138
|
+
return MeshRuntimeStore.getInstance().hasCompletionFingerprint(fingerprint);
|
|
139
|
+
} catch {
|
|
140
|
+
return false;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function recordFingerprintSeen(fingerprint: string): void {
|
|
145
|
+
try {
|
|
146
|
+
const db = MeshRuntimeStore.getInstance();
|
|
147
|
+
db.recordCompletionFingerprint(fingerprint, RECENT_COMPLETION_FINGERPRINT_TTL_MS);
|
|
148
|
+
db.sweepExpiredFingerprints();
|
|
149
|
+
} catch { /* best-effort; duplicate events are preferable to a crash */ }
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function readEventTimestamp(value: unknown): number | null {
|
|
153
|
+
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
|
154
|
+
if (typeof value === 'string' && value.trim()) {
|
|
155
|
+
const numeric = Number(value);
|
|
156
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
157
|
+
const parsed = Date.parse(value);
|
|
158
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
159
|
+
}
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function buildMeshCompletionFingerprint(args: {
|
|
164
|
+
meshId: string;
|
|
165
|
+
event: string;
|
|
166
|
+
sessionId: string;
|
|
167
|
+
providerType?: string;
|
|
168
|
+
providerSessionId?: string;
|
|
169
|
+
timestamp?: number | null;
|
|
170
|
+
finalSummary?: string;
|
|
171
|
+
coordinatorDaemonId?: string;
|
|
172
|
+
}): string {
|
|
173
|
+
const timestampPart = Number.isFinite(args.timestamp)
|
|
174
|
+
? String(args.timestamp)
|
|
175
|
+
: readNonEmptyString(args.finalSummary).slice(0, 200);
|
|
176
|
+
return [
|
|
177
|
+
args.meshId,
|
|
178
|
+
args.event,
|
|
179
|
+
args.sessionId,
|
|
180
|
+
args.providerType || '',
|
|
181
|
+
args.providerSessionId || '',
|
|
182
|
+
timestampPart,
|
|
183
|
+
args.coordinatorDaemonId || '',
|
|
184
|
+
].join('::');
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
function isDuplicateMeshCompletionEvent(args: {
|
|
188
|
+
meshId: string;
|
|
189
|
+
event: string;
|
|
190
|
+
sessionId: string;
|
|
191
|
+
providerType?: string;
|
|
192
|
+
providerSessionId?: string;
|
|
193
|
+
timestamp?: number | null;
|
|
194
|
+
finalSummary?: string;
|
|
195
|
+
coordinatorDaemonId?: string;
|
|
196
|
+
taskId?: string;
|
|
197
|
+
nodeId?: string;
|
|
198
|
+
}): boolean {
|
|
199
|
+
const fingerprint = buildMeshCompletionFingerprint(args);
|
|
200
|
+
if (!fingerprint) return false;
|
|
201
|
+
if (hasFingerprintSeen(fingerprint)) {
|
|
202
|
+
if (args.taskId) {
|
|
203
|
+
recordCompletionConflict({
|
|
204
|
+
meshId: args.meshId,
|
|
205
|
+
fingerprint,
|
|
206
|
+
conflictingTaskId: args.taskId,
|
|
207
|
+
conflictingSessionId: args.sessionId,
|
|
208
|
+
event: args.event,
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
return true;
|
|
212
|
+
}
|
|
213
|
+
recordFingerprintSeen(fingerprint);
|
|
214
|
+
return false;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function isDuplicateMeshApprovalEvent(args: {
|
|
218
|
+
meshId: string;
|
|
219
|
+
sessionId: string;
|
|
220
|
+
providerType?: string;
|
|
221
|
+
timestamp?: number | null;
|
|
222
|
+
modalMessage?: string;
|
|
223
|
+
modalButtons?: unknown;
|
|
224
|
+
}): boolean {
|
|
225
|
+
const modalButtons = Array.isArray(args.modalButtons)
|
|
226
|
+
? args.modalButtons.map(button => String(button).trim()).filter(Boolean)
|
|
227
|
+
: [];
|
|
228
|
+
const approvalIdentity = Number.isFinite(args.timestamp)
|
|
229
|
+
? String(args.timestamp)
|
|
230
|
+
: JSON.stringify({ message: args.modalMessage || '', buttons: modalButtons });
|
|
231
|
+
if (!approvalIdentity || approvalIdentity === '{"message":"","buttons":[]}') return false;
|
|
232
|
+
const fingerprint = [
|
|
233
|
+
args.meshId,
|
|
234
|
+
'agent:waiting_approval',
|
|
235
|
+
args.sessionId,
|
|
236
|
+
args.providerType || '',
|
|
237
|
+
approvalIdentity,
|
|
238
|
+
].join('::');
|
|
239
|
+
if (hasFingerprintSeen(fingerprint)) return true;
|
|
240
|
+
recordFingerprintSeen(fingerprint);
|
|
241
|
+
return false;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function isDuplicateRefineTerminalEvent(meshId: string, eventName: string, metadataEvent: Record<string, unknown>): boolean {
|
|
245
|
+
const jobId = readRefineJobId({ metadataEvent });
|
|
246
|
+
const fingerprint = jobId && new Set(['refine:completed', 'refine:failed']).has(eventName) ? `${meshId}::${eventName}::${jobId}` : '';
|
|
247
|
+
if (!fingerprint) return false;
|
|
248
|
+
if (hasFingerprintSeen(fingerprint)) return true;
|
|
249
|
+
recordFingerprintSeen(fingerprint);
|
|
250
|
+
return false;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// ---------------------------------------------------------------------------
|
|
254
|
+
// Queue assignment
|
|
255
|
+
// ---------------------------------------------------------------------------
|
|
256
|
+
|
|
257
|
+
export function tryAssignQueueTask(
|
|
258
|
+
components: DaemonComponents,
|
|
259
|
+
meshId: string,
|
|
260
|
+
nodeId: string,
|
|
261
|
+
sessionId: string,
|
|
262
|
+
providerType: string
|
|
263
|
+
): boolean {
|
|
264
|
+
const mesh = getMeshWithCache(components, meshId);
|
|
265
|
+
const node = mesh?.nodes.find((n: any) => readMeshNodeId(n) === nodeId);
|
|
266
|
+
const capabilityTags = buildMeshNodeCapabilityTags(node, providerType);
|
|
267
|
+
// Per-(node, provider) maxParallel cap (RepoMeshNodePolicy.providerRoles) layers
|
|
268
|
+
// on top of the global/taskMode caps — stricter wins. Resolved here where the
|
|
269
|
+
// claiming session's providerType + node policy are both known, then enforced
|
|
270
|
+
// inside the atomic claim transaction so concurrent claims can't overshoot it.
|
|
271
|
+
const providerMaxParallel = resolveProviderMaxParallel(node?.policy, providerType);
|
|
272
|
+
const task = claimNextTask(meshId, nodeId, sessionId, capabilityTags, {
|
|
273
|
+
providerType,
|
|
274
|
+
...(providerMaxParallel !== undefined ? { providerMaxParallel } : {}),
|
|
275
|
+
});
|
|
276
|
+
if (!task) {
|
|
277
|
+
return false;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
|
|
281
|
+
|
|
282
|
+
if (node?.daemonId && components.dispatchMeshCommand) {
|
|
283
|
+
const isLocalNode = components.cliManager.adapters.has(sessionId);
|
|
284
|
+
if (!isLocalNode) {
|
|
285
|
+
const delivery = createSessionDelivery({
|
|
286
|
+
meshId,
|
|
287
|
+
nodeId,
|
|
288
|
+
sessionId,
|
|
289
|
+
providerType,
|
|
290
|
+
taskId: task.id,
|
|
291
|
+
kind: 'task',
|
|
292
|
+
message: task.message,
|
|
293
|
+
status: 'delivering',
|
|
294
|
+
});
|
|
295
|
+
const localDaemonIdForDispatch = readNonEmptyString(loadConfig().machineId) || undefined;
|
|
296
|
+
components.dispatchMeshCommand(node.daemonId, 'agent_command', {
|
|
297
|
+
targetSessionId: sessionId,
|
|
298
|
+
cliType: providerType,
|
|
299
|
+
action: 'send_chat',
|
|
300
|
+
message: task.message,
|
|
301
|
+
meshContext: {
|
|
302
|
+
meshId,
|
|
303
|
+
nodeId,
|
|
304
|
+
taskId: task.id,
|
|
305
|
+
...(localDaemonIdForDispatch ? { coordinatorDaemonId: localDaemonIdForDispatch } : {}),
|
|
306
|
+
},
|
|
307
|
+
}).then(() => {
|
|
308
|
+
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
309
|
+
}).catch((e: any) => {
|
|
310
|
+
LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
|
|
311
|
+
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
312
|
+
updateTaskStatus(meshId, task.id, 'pending');
|
|
313
|
+
try {
|
|
314
|
+
appendLedgerEntry(meshId, {
|
|
315
|
+
kind: 'dispatch_failed' as any,
|
|
316
|
+
nodeId,
|
|
317
|
+
sessionId,
|
|
318
|
+
payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
|
|
319
|
+
});
|
|
320
|
+
} catch { /* ledger write is best-effort */ }
|
|
321
|
+
});
|
|
322
|
+
return true;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
// Stamp mesh context onto the session so completion events route correctly
|
|
327
|
+
// via setupMeshEventForwarding. Without this, manually-opened idle sessions
|
|
328
|
+
// (mesh_launch_session without auto-launch) lack meshNodeFor/meshNodeId and
|
|
329
|
+
// agent:generating_completed is silently dropped as isMeshDelegate=false.
|
|
330
|
+
try {
|
|
331
|
+
const inst = components.instanceManager.getInstance(sessionId);
|
|
332
|
+
if (inst && typeof inst.updateSettings === 'function') {
|
|
333
|
+
// Adopting a (possibly manually-opened) local session as a worker: apply the
|
|
334
|
+
// delegated-worker auto-approve policy here too, so a session that was launched
|
|
335
|
+
// without autoApprove still auto-approves once the coordinator dispatches a task
|
|
336
|
+
// to it (the "approval notification fires only for certain delegated sessions"
|
|
337
|
+
// case). updateSettings preserves runtime mesh keys; passing autoApprove keeps it.
|
|
338
|
+
//
|
|
339
|
+
// This local-dispatch branch also runs on the coordinator daemon for a co-located
|
|
340
|
+
// session, so the coordinator daemon id IS this daemon's id. Stamp it alongside
|
|
341
|
+
// the node identity so the session is fully relay-safe (meshCoordinatorDaemonId is
|
|
342
|
+
// the anchor the forwarder keys on), matching what mesh_launch_session stamps.
|
|
343
|
+
const localDaemonId = readNonEmptyString(loadConfig().machineId);
|
|
344
|
+
inst.updateSettings({
|
|
345
|
+
meshNodeFor: meshId,
|
|
346
|
+
meshNodeId: nodeId,
|
|
347
|
+
launchedByCoordinator: true,
|
|
348
|
+
autoApprove: resolveDelegatedWorkerAutoApprove(mesh?.policy, node?.policy),
|
|
349
|
+
...(localDaemonId ? { meshCoordinatorDaemonId: localDaemonId } : {}),
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
} catch { /* best-effort — dispatch still proceeds */ }
|
|
353
|
+
|
|
354
|
+
const delivery = createSessionDelivery({
|
|
355
|
+
meshId,
|
|
356
|
+
nodeId,
|
|
357
|
+
sessionId,
|
|
358
|
+
providerType,
|
|
359
|
+
taskId: task.id,
|
|
360
|
+
kind: 'task',
|
|
361
|
+
message: task.message,
|
|
362
|
+
status: 'delivering',
|
|
363
|
+
});
|
|
364
|
+
components.cliManager.handleCliCommand('agent_command', {
|
|
365
|
+
targetSessionId: sessionId,
|
|
366
|
+
cliType: providerType,
|
|
367
|
+
action: 'send_chat',
|
|
368
|
+
message: task.message,
|
|
369
|
+
}).then(() => {
|
|
370
|
+
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
371
|
+
}).catch((e: any) => {
|
|
372
|
+
LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
|
|
373
|
+
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
374
|
+
updateTaskStatus(meshId, task.id, 'failed');
|
|
375
|
+
});
|
|
376
|
+
|
|
377
|
+
return true;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
const autoLaunchInProgress = new Set<string>();
|
|
381
|
+
const autoLaunchCooldownUntil = new Map<string, number>();
|
|
382
|
+
const AUTO_LAUNCH_COOLDOWN_MS = 5_000;
|
|
383
|
+
// A remote auto-launch (launch_cli forward) is fire-and-async: the worker session
|
|
384
|
+
// spawns, reaches idle, emits agent:ready, that ready is queued on the worker, pulled
|
|
385
|
+
// by this coordinator (reconcile PHASE 1), and only THEN claims the task. That round
|
|
386
|
+
// trip routinely exceeds the 5s per-(mesh,node) cooldown, so cooldown alone lets the
|
|
387
|
+
// reconcile loop fire a SECOND launch for the same still-pending task before the first
|
|
388
|
+
// session's claim lands — every tick spawns yet another orphan session (observed live:
|
|
389
|
+
// 26 sessions for one task). This is a per-TASK await-claim window: once a task has a
|
|
390
|
+
// successfully-launched session whose claim we are still waiting on, do not launch it
|
|
391
|
+
// again until the window lapses. It is generous (a slow remote spawn can take tens of
|
|
392
|
+
// seconds) but bounded so a launch that silently never reaches idle is eventually retried.
|
|
393
|
+
const AUTO_LAUNCH_AWAIT_CLAIM_MS = 90_000;
|
|
394
|
+
|
|
395
|
+
// De-dup for repeated `skipped` ledger noise: the reconcile loop re-runs the queue
|
|
396
|
+
// trigger every 4s, so a task that can't be claimed (e.g. a remote node with no
|
|
397
|
+
// transport, or a node under cooldown) would otherwise append an identical
|
|
398
|
+
// session_auto_launch{phase:'skipped'} entry on every tick — flooding the ledger.
|
|
399
|
+
// We suppress a `skipped` ledger append when the immediately-prior recorded event
|
|
400
|
+
// for that task was the SAME (phase, reason). Any non-skip phase (started/failed/
|
|
401
|
+
// completed) or a changed reason resets the de-dup so real transitions still record.
|
|
402
|
+
const lastAutoLaunchLedgerKey = new Map<string, string>();
|
|
403
|
+
const AUTO_LAUNCH_LEDGER_DEDUP_MAX = 2000;
|
|
404
|
+
|
|
405
|
+
function sweepExpiredCooldowns(): void {
|
|
406
|
+
const now = Date.now();
|
|
407
|
+
for (const [key, until] of autoLaunchCooldownUntil) {
|
|
408
|
+
if (now >= until) autoLaunchCooldownUntil.delete(key);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function normalizeProviderPriority(policy: unknown): string[] {
|
|
413
|
+
const raw = policy && typeof policy === 'object' && !Array.isArray(policy)
|
|
414
|
+
? (policy as Record<string, unknown>).providerPriority
|
|
415
|
+
: undefined;
|
|
416
|
+
if (!Array.isArray(raw)) return [];
|
|
417
|
+
const seen = new Set<string>();
|
|
418
|
+
return raw
|
|
419
|
+
.map(type => typeof type === 'string' ? type.trim() : '')
|
|
420
|
+
.filter(Boolean)
|
|
421
|
+
.filter(type => {
|
|
422
|
+
if (seen.has(type)) return false;
|
|
423
|
+
seen.add(type);
|
|
424
|
+
return true;
|
|
425
|
+
});
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
function isTerminalSessionStatus(status: string): boolean {
|
|
429
|
+
return ['stopped', 'failed', 'terminated', 'exited', 'closed'].includes(status);
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function isIdleSessionState(state: any): boolean {
|
|
433
|
+
const status = readNonEmptyString(state?.status).toLowerCase();
|
|
434
|
+
if (isTerminalSessionStatus(status)) return false;
|
|
435
|
+
return status === 'idle' || state?.activeChat?.status === 'waiting_input';
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function isDirtyNode(node: any): boolean {
|
|
439
|
+
return node?.health === 'dirty' || node?.git?.dirty === true;
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
function resolveAutoFastForwardPolicy(mesh: any): { enabled: boolean; maxBehind?: number; requireCleanSubmodules: boolean } {
|
|
443
|
+
const record = mesh?.policy?.autoFastForward && typeof mesh.policy.autoFastForward === 'object' && !Array.isArray(mesh.policy.autoFastForward)
|
|
444
|
+
? mesh.policy.autoFastForward as Record<string, unknown>
|
|
445
|
+
: {};
|
|
446
|
+
const maxBehind = Number(record.maxBehind);
|
|
447
|
+
return {
|
|
448
|
+
enabled: record.enabled !== false,
|
|
449
|
+
...(Number.isFinite(maxBehind) && maxBehind >= 0 ? { maxBehind: Math.floor(maxBehind) } : {}),
|
|
450
|
+
requireCleanSubmodules: record.requireCleanSubmodules !== false,
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
function sessionStateLooksActive(state: any): boolean {
|
|
455
|
+
const status = readNonEmptyString(state?.status).toLowerCase();
|
|
456
|
+
const chatStatus = readNonEmptyString(state?.activeChat?.status).toLowerCase();
|
|
457
|
+
// 'long_generating' is retained as a legacy alias for the renamed 'no_progress' busy status.
|
|
458
|
+
const active = new Set(['generating', 'streaming', 'no_progress', 'long_generating', 'working', 'starting', 'waiting_approval']);
|
|
459
|
+
return active.has(status) || active.has(chatStatus);
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
function nodeHasActiveMeshWork(components: DaemonComponents, meshId: string, nodeId: string, currentSessionId?: string): boolean {
|
|
463
|
+
if (nodeHasActiveAssignment(meshId, nodeId)) return true;
|
|
464
|
+
return components.instanceManager.getByCategory('cli').some((inst: any) => {
|
|
465
|
+
const state = inst.getState();
|
|
466
|
+
const settings = state.settings as Record<string, unknown> || {};
|
|
467
|
+
if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
|
|
468
|
+
const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
469
|
+
if (instNodeId !== nodeId) return false;
|
|
470
|
+
const sessionId = readNonEmptyString(state.instanceId);
|
|
471
|
+
if (currentSessionId && sessionId === currentSessionId && isIdleSessionState(state)) return false;
|
|
472
|
+
return sessionStateLooksActive(state);
|
|
473
|
+
});
|
|
474
|
+
}
|
|
475
|
+
|
|
476
|
+
function isLaunchableNode(node: any): boolean {
|
|
477
|
+
if (!node || node.status === 'disabled' || node.status === 'removed') return false;
|
|
478
|
+
const health = readNonEmptyString(node.health).toLowerCase();
|
|
479
|
+
if (!health) return true;
|
|
480
|
+
return health === 'online' || health === 'unknown';
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
/** Whether a mesh node's daemon/machine identity resolves to THIS coordinator daemon
|
|
484
|
+
* (i.e. the queue session can be spawned by a direct local `launch_cli`). */
|
|
485
|
+
function isLocalAutoLaunchNode(node: any): boolean {
|
|
486
|
+
const daemonId = readNonEmptyString(node?.daemonId);
|
|
487
|
+
const machineId = readNonEmptyString(node?.machineId);
|
|
488
|
+
const appConfig = loadConfig();
|
|
489
|
+
const localMachineId = readNonEmptyString(appConfig.machineId) || readNonEmptyString(appConfig.registeredMachineId);
|
|
490
|
+
const cloudDaemonId = localMachineId ? `daemon_${localMachineId}` : '';
|
|
491
|
+
const standaloneDaemonId = localMachineId ? `standalone_${localMachineId}` : '';
|
|
492
|
+
|
|
493
|
+
const daemonMatchesLocal = !daemonId || daemonId === cloudDaemonId || daemonId === standaloneDaemonId;
|
|
494
|
+
const machineMatchesLocal = !machineId || (!!localMachineId && machineId === localMachineId);
|
|
495
|
+
|
|
496
|
+
if (node?.isLocalWorktree === true) {
|
|
497
|
+
return daemonMatchesLocal && machineMatchesLocal;
|
|
498
|
+
}
|
|
499
|
+
if (daemonId || machineId) {
|
|
500
|
+
return daemonMatchesLocal && machineMatchesLocal;
|
|
501
|
+
}
|
|
502
|
+
return true;
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
/**
|
|
506
|
+
* Resolve how a pending queue task should be auto-launched onto a node.
|
|
507
|
+
*
|
|
508
|
+
* - `local`: spawn directly on this daemon via cliManager.handleCliCommand('launch_cli').
|
|
509
|
+
* - `remote`: forward `launch_cli` to the node's daemon via dispatchMeshCommand
|
|
510
|
+
* (mirrors what mesh_launch_session does). Requires dispatchMeshCommand AND a
|
|
511
|
+
* resolvable coordinator daemonId for relay-safe completion routing.
|
|
512
|
+
* - `skip`: not launchable from here — carries the reason (e.g. a remote node with
|
|
513
|
+
* no dispatch transport, or no coordinator daemonId to stamp).
|
|
514
|
+
*/
|
|
515
|
+
function resolveAutoLaunchTarget(components: DaemonComponents, node: any): {
|
|
516
|
+
mode: 'local' | 'remote' | 'skip';
|
|
517
|
+
reason?: string;
|
|
518
|
+
daemonId?: string;
|
|
519
|
+
coordinatorDaemonId?: string;
|
|
520
|
+
} {
|
|
521
|
+
if (isLocalAutoLaunchNode(node)) return { mode: 'local' };
|
|
522
|
+
|
|
523
|
+
// Remote node. Forwarding the launch is possible only with a dispatch transport
|
|
524
|
+
// (cloud mode) plus a coordinator daemonId to stamp into the worker so completion
|
|
525
|
+
// events route back here. Without either, fall back to a graceful skip.
|
|
526
|
+
const daemonId = readNonEmptyString(node?.daemonId);
|
|
527
|
+
if (!daemonId) return { mode: 'skip', reason: 'remote_auto_launch_unsupported' };
|
|
528
|
+
if (!components.dispatchMeshCommand) return { mode: 'skip', reason: 'remote_auto_launch_unsupported' };
|
|
529
|
+
const coordinatorDaemonId = readNonEmptyString(loadConfig().machineId);
|
|
530
|
+
if (!coordinatorDaemonId) return { mode: 'skip', reason: 'remote_auto_launch_no_coordinator_daemon_id' };
|
|
531
|
+
return { mode: 'remote', daemonId, coordinatorDaemonId };
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
function activeAssignedCount(meshId: string): number {
|
|
535
|
+
return getQueue(meshId, { status: ['assigned'] as any }).length;
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
/** Active assignments that hold the one-active-per-node / global-parallel invariant
|
|
539
|
+
* (everything except read-only diagnoses, which run unbounded by the write cap). */
|
|
540
|
+
export function activeWriteAssignedCount(meshId: string): number {
|
|
541
|
+
return getQueue(meshId, { status: ['assigned'] as any })
|
|
542
|
+
.filter(task => task.taskMode !== 'live_debug_readonly').length;
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
/** Active read-only (live_debug_readonly) assignments, for the read-only safety cap. */
|
|
546
|
+
export function activeReadonlyAssignedCount(meshId: string): number {
|
|
547
|
+
return getQueue(meshId, { status: ['assigned'] as any })
|
|
548
|
+
.filter(task => task.taskMode === 'live_debug_readonly').length;
|
|
549
|
+
}
|
|
550
|
+
|
|
551
|
+
function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
|
|
552
|
+
return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
/** Active (status='assigned') task count for a node — the load metric for
|
|
556
|
+
* least-loaded / round-robin ranking. Lower = preferred. */
|
|
557
|
+
function nodeActiveLoad(meshId: string, nodeId: string): number {
|
|
558
|
+
return MeshRuntimeStore.getInstance().nodeActiveAssignmentCount(meshId, nodeId);
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
/**
|
|
562
|
+
* The mesh-wide scheduling strategy. Defaults to 'first_eligible' (strict
|
|
563
|
+
* no-change) for any mesh that does not set it. Only governs the final tie-break;
|
|
564
|
+
* eligibility, capacity, and priority gates apply identically to every strategy.
|
|
565
|
+
*/
|
|
566
|
+
function resolveSchedulingStrategy(mesh: any): RepoMeshSchedulingStrategy {
|
|
567
|
+
return normalizeMeshSchedulingStrategy(mesh?.policy?.schedulingStrategy);
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
/**
|
|
571
|
+
* Order eligible nodes for assignment per the mesh scheduling pipeline:
|
|
572
|
+
* PRIORITY (schedulingPriority desc) → TIE-BREAK (strategy).
|
|
573
|
+
*
|
|
574
|
+
* The caller has already applied the TAG hard-filter and is responsible for the
|
|
575
|
+
* MAX-ALLOC capacity gate (the per-node launch/claim checks). This function only
|
|
576
|
+
* decides the *preference order* among nodes that are otherwise eligible.
|
|
577
|
+
*
|
|
578
|
+
* - 'first_eligible' (default): returns the input order verbatim and does NOT touch
|
|
579
|
+
* the round-robin cursor — byte-for-byte the pre-feature behavior.
|
|
580
|
+
* - 'priority_only': schedulingPriority desc, then input order (load ignored).
|
|
581
|
+
* - 'least_loaded': schedulingPriority desc, then active load asc, then input order.
|
|
582
|
+
* - 'round_robin': same as least_loaded, but among nodes tied at (priority, load)
|
|
583
|
+
* the input order is rotated by a per-mesh cursor that advances once per pass.
|
|
584
|
+
*
|
|
585
|
+
* `nodes` carries the original config/array index so the tie-break can fall back to
|
|
586
|
+
* deterministic input order. `bumpCursor` advances the round-robin cursor exactly
|
|
587
|
+
* once per scheduling pass (only consulted for 'round_robin').
|
|
588
|
+
*/
|
|
589
|
+
interface RankableNode { nodeId: string; node: any; index: number }
|
|
590
|
+
|
|
591
|
+
/** Test-only: the pure node-ordering stage (PRIORITY → TIE-BREAK). Exposed so the
|
|
592
|
+
* scheduling pipeline can be unit-tested without standing up live CLI sessions. */
|
|
593
|
+
export function __orderEligibleNodesForTests(
|
|
594
|
+
meshId: string,
|
|
595
|
+
strategy: RepoMeshSchedulingStrategy,
|
|
596
|
+
nodes: RankableNode[],
|
|
597
|
+
opts?: { bumpCursor?: boolean },
|
|
598
|
+
): RankableNode[] {
|
|
599
|
+
return orderEligibleNodes(meshId, strategy, nodes, opts);
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
function orderEligibleNodes(
|
|
603
|
+
meshId: string,
|
|
604
|
+
strategy: RepoMeshSchedulingStrategy,
|
|
605
|
+
nodes: RankableNode[],
|
|
606
|
+
opts?: { bumpCursor?: boolean },
|
|
607
|
+
): RankableNode[] {
|
|
608
|
+
if (strategy === 'first_eligible' || nodes.length <= 1) {
|
|
609
|
+
return nodes;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
const priorityOf = (n: { node: any }) => resolveNodeSchedulingPriority(n.node?.policy);
|
|
613
|
+
|
|
614
|
+
// Round-robin rotation offset: rotate the deterministic input order by a
|
|
615
|
+
// per-mesh cursor so the tie-break winner among equal (priority, load) nodes
|
|
616
|
+
// cycles across passes. The cursor advances once per scheduling pass.
|
|
617
|
+
let rotation = 0;
|
|
618
|
+
if (strategy === 'round_robin') {
|
|
619
|
+
const cursor = opts?.bumpCursor
|
|
620
|
+
? MeshRuntimeStore.getInstance().bumpSchedulerCursor(meshId)
|
|
621
|
+
: MeshRuntimeStore.getInstance().getSchedulerCursor(meshId);
|
|
622
|
+
rotation = ((cursor % nodes.length) + nodes.length) % nodes.length;
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
// Rotation rank: position of each node after rotating input order by `rotation`.
|
|
626
|
+
// For non-round-robin strategies rotation is 0, so this is just the input index.
|
|
627
|
+
const rotationRank = (index: number) => (index - rotation + nodes.length) % nodes.length;
|
|
628
|
+
|
|
629
|
+
return [...nodes].sort((a, b) => {
|
|
630
|
+
const prioDelta = priorityOf(b) - priorityOf(a); // higher priority first
|
|
631
|
+
if (prioDelta !== 0) return prioDelta;
|
|
632
|
+
if (strategy === 'least_loaded' || strategy === 'round_robin') {
|
|
633
|
+
const loadDelta = nodeActiveLoad(meshId, a.nodeId) - nodeActiveLoad(meshId, b.nodeId);
|
|
634
|
+
if (loadDelta !== 0) return loadDelta;
|
|
635
|
+
}
|
|
636
|
+
return rotationRank(a.index) - rotationRank(b.index);
|
|
637
|
+
});
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
/** Active assignments on a (node, provider) — pre-launch guard for the per-(node,
|
|
641
|
+
* provider) maxParallel cap. The authoritative enforcement is in the claim
|
|
642
|
+
* transaction; this only avoids spawning a session that would fail the claim. */
|
|
643
|
+
function activeProviderAssignedCount(meshId: string, nodeId: string, providerType: string): number {
|
|
644
|
+
return getQueue(meshId, { status: ['assigned'] as any })
|
|
645
|
+
.filter(task => task.assignedNodeId === nodeId && task.assignedProviderType === providerType).length;
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
function sessionHasActiveAssignment(meshId: string, sessionId: string): boolean {
|
|
649
|
+
if (getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedSessionId === sessionId)) {
|
|
650
|
+
return true;
|
|
651
|
+
}
|
|
652
|
+
// Direct dispatches (mesh_send_task) are tracked in mesh_direct_dispatches, not the
|
|
653
|
+
// work queue. A session completing a still-active direct dispatch IS an active
|
|
654
|
+
// assignment — without this, findRecentTerminalLedgerEvidence dedup wrongly suppresses
|
|
655
|
+
// the canonical agent:generating_completed for direct-dispatch tasks (validation/general),
|
|
656
|
+
// so the coordinator polling get_pending_mesh_events never observes task_completed and the
|
|
657
|
+
// session goes silently idle. This check runs before markSessionTerminal marks the
|
|
658
|
+
// dispatch terminal, so the in-flight dispatch is still observable here.
|
|
659
|
+
try {
|
|
660
|
+
if (getActiveDirectDispatches(meshId).some(d => d.sessionId === sessionId)) return true;
|
|
661
|
+
if (hasUnterminalDirectDispatchLedgerEntry(meshId, sessionId)) return true;
|
|
662
|
+
} catch { /* best-effort — fall through to false */ }
|
|
663
|
+
return false;
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
function liveSessionCountForNode(components: DaemonComponents, meshId: string, nodeId: string): number {
|
|
667
|
+
return components.instanceManager.getByCategory('cli').filter((inst: any) => {
|
|
668
|
+
const state = inst.getState();
|
|
669
|
+
const settings = state.settings as Record<string, unknown> || {};
|
|
670
|
+
if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
|
|
671
|
+
const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
672
|
+
if (instNodeId !== nodeId) return false;
|
|
673
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
674
|
+
return !isTerminalSessionStatus(status);
|
|
675
|
+
}).length;
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
function recordAutoLaunchEvent(meshId: string, args: {
|
|
679
|
+
phase: 'skipped' | 'started' | 'failed' | 'completed';
|
|
680
|
+
taskId: string;
|
|
681
|
+
nodeId?: string;
|
|
682
|
+
providerType?: string;
|
|
683
|
+
sessionId?: string;
|
|
684
|
+
reason?: string;
|
|
685
|
+
error?: string;
|
|
686
|
+
}) {
|
|
687
|
+
// Suppress consecutive identical `skipped` entries for the same task (4s reconcile
|
|
688
|
+
// re-trigger noise). Non-skip phases and changed reasons always record and reset
|
|
689
|
+
// the de-dup so genuine state transitions remain visible in the ledger.
|
|
690
|
+
const dedupKey = `${meshId}:${args.taskId}`;
|
|
691
|
+
const currentSig = `${args.phase}|${args.reason || ''}`;
|
|
692
|
+
if (args.phase === 'skipped' && lastAutoLaunchLedgerKey.get(dedupKey) === currentSig) {
|
|
693
|
+
return;
|
|
694
|
+
}
|
|
695
|
+
lastAutoLaunchLedgerKey.set(dedupKey, currentSig);
|
|
696
|
+
if (lastAutoLaunchLedgerKey.size > AUTO_LAUNCH_LEDGER_DEDUP_MAX) {
|
|
697
|
+
// Bound memory: drop the oldest insertion (Map preserves insertion order).
|
|
698
|
+
const oldest = lastAutoLaunchLedgerKey.keys().next().value;
|
|
699
|
+
if (oldest !== undefined) lastAutoLaunchLedgerKey.delete(oldest);
|
|
700
|
+
}
|
|
701
|
+
try {
|
|
702
|
+
appendLedgerEntry(meshId, {
|
|
703
|
+
kind: 'session_auto_launch',
|
|
704
|
+
nodeId: args.nodeId,
|
|
705
|
+
sessionId: args.sessionId,
|
|
706
|
+
providerType: args.providerType,
|
|
707
|
+
payload: {
|
|
708
|
+
phase: args.phase,
|
|
709
|
+
taskId: args.taskId,
|
|
710
|
+
reason: args.reason,
|
|
711
|
+
error: args.error,
|
|
712
|
+
},
|
|
713
|
+
});
|
|
714
|
+
} catch (e: any) {
|
|
715
|
+
LOG.warn('MeshQueue', `Failed to record auto-launch ledger event: ${e?.message || e}`);
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
function markAutoLaunch(meshId: string, taskId: string, args: {
|
|
720
|
+
status: 'skipped' | 'started' | 'failed' | 'completed';
|
|
721
|
+
reason?: string;
|
|
722
|
+
nodeId?: string;
|
|
723
|
+
providerType?: string;
|
|
724
|
+
sessionId?: string;
|
|
725
|
+
error?: string;
|
|
726
|
+
}) {
|
|
727
|
+
recordTaskAutoLaunch(meshId, taskId, {
|
|
728
|
+
status: args.status,
|
|
729
|
+
reason: args.reason || args.error,
|
|
730
|
+
nodeId: args.nodeId,
|
|
731
|
+
providerType: args.providerType,
|
|
732
|
+
sessionId: args.sessionId,
|
|
733
|
+
});
|
|
734
|
+
recordAutoLaunchEvent(meshId, {
|
|
735
|
+
phase: args.status,
|
|
736
|
+
taskId,
|
|
737
|
+
nodeId: args.nodeId,
|
|
738
|
+
providerType: args.providerType,
|
|
739
|
+
sessionId: args.sessionId,
|
|
740
|
+
reason: args.reason,
|
|
741
|
+
error: args.error,
|
|
742
|
+
});
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
async function resolveUsableProvider(
|
|
746
|
+
components: DaemonComponents,
|
|
747
|
+
nodeId: string,
|
|
748
|
+
node: any,
|
|
749
|
+
requiredTags?: string[],
|
|
750
|
+
): Promise<{ providerType?: string; reason?: string }> {
|
|
751
|
+
const providerPriority = normalizeProviderPriority(node?.policy);
|
|
752
|
+
if (!providerPriority.length) return { reason: 'missing_provider_priority' };
|
|
753
|
+
const providerLoader = components.providerLoader;
|
|
754
|
+
if (!providerLoader) return { reason: 'provider_loader_unavailable' };
|
|
755
|
+
|
|
756
|
+
const failed: string[] = [];
|
|
757
|
+
for (const requestedType of providerPriority) {
|
|
758
|
+
const normalizedType = typeof providerLoader.resolveAlias === 'function'
|
|
759
|
+
? providerLoader.resolveAlias(requestedType)
|
|
760
|
+
: requestedType;
|
|
761
|
+
// Skip providers that can't satisfy the task's requiredTags (e.g. provider=hermes-cli
|
|
762
|
+
// means only hermes-cli qualifies, not any other type in providerPriority).
|
|
763
|
+
if (requiredTags?.length && !nodeSatisfiesRequiredTags(requiredTags, buildMeshNodeCapabilityTags(node, normalizedType))) {
|
|
764
|
+
failed.push(`${requestedType}: required_tags_mismatch`);
|
|
765
|
+
continue;
|
|
766
|
+
}
|
|
767
|
+
if (typeof providerLoader.isMachineProviderEnabled === 'function' && !providerLoader.isMachineProviderEnabled(normalizedType)) {
|
|
768
|
+
failed.push(`${requestedType}: disabled`);
|
|
769
|
+
continue;
|
|
770
|
+
}
|
|
771
|
+
let detected: any;
|
|
772
|
+
try {
|
|
773
|
+
detected = await detectCLI(normalizedType, providerLoader, { includeVersion: false });
|
|
774
|
+
} catch (e: any) {
|
|
775
|
+
failed.push(`${requestedType}: detect failed: ${e?.message || e}`);
|
|
776
|
+
continue;
|
|
777
|
+
}
|
|
778
|
+
if (typeof providerLoader.setCliDetectionResults === 'function') {
|
|
779
|
+
providerLoader.setCliDetectionResults([{
|
|
780
|
+
id: normalizedType,
|
|
781
|
+
installed: !!detected,
|
|
782
|
+
path: detected?.path,
|
|
783
|
+
}], false);
|
|
784
|
+
}
|
|
785
|
+
(components as any).onStatusChange?.();
|
|
786
|
+
if (detected) return { providerType: normalizedType };
|
|
787
|
+
failed.push(`${requestedType}: not detected`);
|
|
788
|
+
}
|
|
789
|
+
return { reason: `provider_priority_unusable: ${failed.join('; ') || nodeId}` };
|
|
790
|
+
}
|
|
791
|
+
|
|
792
|
+
// Canonical mesh node-id normalization. A node may arrive from the local config
|
|
793
|
+
// form (`id`) or the inline-cache form (`nodeId`/`node_id`) — see
|
|
794
|
+
// readInlineMeshNodeId in commands/router.ts. Comparing only `node.id` against a
|
|
795
|
+
// task.targetNodeId silently drops inline-cached worktree nodes, leaving a
|
|
796
|
+
// target-routed task permanently pending with a misleading
|
|
797
|
+
// `no_node_satisfies_required_tags` skip.
|
|
798
|
+
function readMeshNodeId(node: any): string {
|
|
799
|
+
// Delegate to the shared 3-way (id / nodeId / node_id) normalizer so this
|
|
800
|
+
// and every other mesh node-id read agree on identity. Coalesce to '' to
|
|
801
|
+
// preserve the existing string return contract for callers that do
|
|
802
|
+
// `=== task.targetNodeId` / `if (!nodeId)`.
|
|
803
|
+
return normalizeMeshNodeId(node) ?? '';
|
|
804
|
+
}
|
|
805
|
+
|
|
806
|
+
async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, meshId: string, mesh: any): Promise<boolean> {
|
|
807
|
+
const queue = getQueue(meshId);
|
|
808
|
+
const pending = queue.filter(task => task.status === 'pending');
|
|
809
|
+
if (!pending.length) return false;
|
|
810
|
+
|
|
811
|
+
const maxParallelTasks = Math.max(1, Math.floor(Number(mesh?.policy?.maxParallelTasks) || 2));
|
|
812
|
+
// Read-only diagnoses carry no isolation/merge cost, so they are exempt from the
|
|
813
|
+
// write-task parallel cap. To prevent runaway auto-launch they get their own,
|
|
814
|
+
// higher safety cap (2x the write cap).
|
|
815
|
+
const maxReadonlyParallelTasks = Math.max(2, maxParallelTasks * 2);
|
|
816
|
+
for (const task of pending) {
|
|
817
|
+
const isReadonly = task.taskMode === 'live_debug_readonly';
|
|
818
|
+
if (isReadonly) {
|
|
819
|
+
if (activeReadonlyAssignedCount(meshId) >= maxReadonlyParallelTasks) {
|
|
820
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_readonly_parallel_tasks_reached' });
|
|
821
|
+
continue;
|
|
822
|
+
}
|
|
823
|
+
} else if (activeWriteAssignedCount(meshId) >= maxParallelTasks) {
|
|
824
|
+
// Write tasks are capped; skip this one but keep scanning so a later
|
|
825
|
+
// read-only task in the queue can still launch under its own cap.
|
|
826
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_parallel_tasks_reached' });
|
|
827
|
+
continue;
|
|
828
|
+
}
|
|
829
|
+
if (task.targetSessionId) {
|
|
830
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
|
|
831
|
+
continue;
|
|
832
|
+
}
|
|
833
|
+
|
|
834
|
+
// Per-task await-claim guard. A prior auto-launch already spawned a session for
|
|
835
|
+
// this task and we are waiting for that session's idle→claim to land (remote
|
|
836
|
+
// claims arrive via the worker→coordinator agent:ready pull, which can lag well
|
|
837
|
+
// past the per-node cooldown). Re-launching now would spawn a duplicate orphan
|
|
838
|
+
// session that never gets work. The task leaves `pending` the instant the claim
|
|
839
|
+
// succeeds, so this guard only suppresses the in-flight window; if the launched
|
|
840
|
+
// session never reaches idle within the window, a later tick retries.
|
|
841
|
+
if (task.autoLaunch?.status === 'completed' && task.autoLaunch.sessionId) {
|
|
842
|
+
const launchedAtMs = Date.parse(task.autoLaunch.updatedAt);
|
|
843
|
+
if (Number.isFinite(launchedAtMs) && Date.now() - launchedAtMs < AUTO_LAUNCH_AWAIT_CLAIM_MS) {
|
|
844
|
+
// Record the skip in the ledger ONLY (dedup'd). Do NOT call markAutoLaunch
|
|
845
|
+
// here: recordTaskAutoLaunch overwrites task.autoLaunch wholesale, which would
|
|
846
|
+
// erase the very `completed` record (status + sessionId + updatedAt) this guard
|
|
847
|
+
// reads on the next tick, reopening the duplicate-launch hole it closes.
|
|
848
|
+
recordAutoLaunchEvent(meshId, { phase: 'skipped', taskId: task.id, reason: 'awaiting_launched_session_claim', nodeId: task.autoLaunch.nodeId, sessionId: task.autoLaunch.sessionId });
|
|
849
|
+
continue;
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
|
|
853
|
+
const candidateNodes = Array.isArray(mesh?.nodes)
|
|
854
|
+
? mesh.nodes.filter((node: any) => {
|
|
855
|
+
if (task.targetNodeId && readMeshNodeId(node) !== task.targetNodeId) return false;
|
|
856
|
+
// Skip nodes that can never satisfy requiredTags regardless of which provider
|
|
857
|
+
// from providerPriority is selected. A node satisfies tags if at least one
|
|
858
|
+
// provider in its priority list would produce matching capability tags.
|
|
859
|
+
if (task.requiredTags?.length) {
|
|
860
|
+
const priorities = normalizeProviderPriority(node?.policy);
|
|
861
|
+
const providerCandidates = priorities.length ? priorities : [undefined as unknown as string];
|
|
862
|
+
return providerCandidates.some(p =>
|
|
863
|
+
nodeSatisfiesRequiredTags(task.requiredTags, buildMeshNodeCapabilityTags(node, p))
|
|
864
|
+
);
|
|
865
|
+
}
|
|
866
|
+
return true;
|
|
867
|
+
})
|
|
868
|
+
: [];
|
|
869
|
+
if (!candidateNodes.length) {
|
|
870
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'no_node_satisfies_required_tags', nodeId: task.targetNodeId });
|
|
871
|
+
continue;
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
// PRIORITY → TIE-BREAK: order the eligible (TAG-filtered) candidate nodes by
|
|
875
|
+
// the mesh scheduling strategy. 'first_eligible' (default) returns them in
|
|
876
|
+
// config/array order unchanged, so distribution is strictly opt-in. The
|
|
877
|
+
// per-node MAX-ALLOC capacity gate (nodeHasActiveAssignment, provider cap,
|
|
878
|
+
// maxConcurrentSessions) is still applied inside the loop below; this only
|
|
879
|
+
// chooses which eligible node is *tried first*.
|
|
880
|
+
const strategy = resolveSchedulingStrategy(mesh);
|
|
881
|
+
const orderedCandidateNodes = strategy === 'first_eligible'
|
|
882
|
+
? candidateNodes
|
|
883
|
+
: orderEligibleNodes(
|
|
884
|
+
meshId,
|
|
885
|
+
strategy,
|
|
886
|
+
candidateNodes
|
|
887
|
+
.map((node: any, index: number) => ({ nodeId: readMeshNodeId(node), node, index }))
|
|
888
|
+
.filter((c: RankableNode) => c.nodeId),
|
|
889
|
+
{ bumpCursor: true },
|
|
890
|
+
).map((c: RankableNode) => c.node);
|
|
891
|
+
|
|
892
|
+
for (const node of orderedCandidateNodes) {
|
|
893
|
+
const nodeId = readMeshNodeId(node);
|
|
894
|
+
if (!nodeId) continue;
|
|
895
|
+
const launchKey = `${meshId}:${nodeId}`;
|
|
896
|
+
const now = Date.now();
|
|
897
|
+
const cooldownUntil = autoLaunchCooldownUntil.get(launchKey) || 0;
|
|
898
|
+
if (cooldownUntil > 0 && now >= cooldownUntil) autoLaunchCooldownUntil.delete(launchKey);
|
|
899
|
+
if (autoLaunchInProgress.has(launchKey)) {
|
|
900
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_in_progress', nodeId });
|
|
901
|
+
continue;
|
|
902
|
+
}
|
|
903
|
+
if (now < cooldownUntil) {
|
|
904
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_cooldown', nodeId });
|
|
905
|
+
continue;
|
|
906
|
+
}
|
|
907
|
+
if (isDirtyNode(node)) {
|
|
908
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'dirty_workspace', nodeId });
|
|
909
|
+
continue;
|
|
910
|
+
}
|
|
911
|
+
if (!isLaunchableNode(node)) {
|
|
912
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
|
|
913
|
+
continue;
|
|
914
|
+
}
|
|
915
|
+
const launchTarget = resolveAutoLaunchTarget(components, node);
|
|
916
|
+
if (launchTarget.mode === 'skip') {
|
|
917
|
+
// Remote node we can't reach (no transport / no coordinator daemonId).
|
|
918
|
+
// Set a cooldown so the 4s reconcile loop doesn't re-attempt this node
|
|
919
|
+
// every tick; the de-dup'd skip ledger keeps it diagnosable without flood.
|
|
920
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: launchTarget.reason || 'auto_launch_unavailable', nodeId });
|
|
921
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
922
|
+
continue;
|
|
923
|
+
}
|
|
924
|
+
// Write tasks keep the one-active-per-node invariant (worktree isolation);
|
|
925
|
+
// read-only (live_debug_readonly) diagnoses may auto-launch onto a node
|
|
926
|
+
// that already has an active assignment.
|
|
927
|
+
if (task.taskMode !== 'live_debug_readonly' && nodeHasActiveAssignment(meshId, nodeId)) {
|
|
928
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_active_assignment', nodeId });
|
|
929
|
+
continue;
|
|
930
|
+
}
|
|
931
|
+
const maxConcurrentSessions = Number(node?.policy?.maxConcurrentSessions);
|
|
932
|
+
if (Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0 && liveSessionCountForNode(components, meshId, nodeId) >= maxConcurrentSessions) {
|
|
933
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_concurrent_sessions_reached', nodeId });
|
|
934
|
+
continue;
|
|
935
|
+
}
|
|
936
|
+
|
|
937
|
+
autoLaunchInProgress.add(launchKey);
|
|
938
|
+
try {
|
|
939
|
+
const resolved = await resolveUsableProvider(components, nodeId, node, task.requiredTags);
|
|
940
|
+
if (!resolved.providerType) {
|
|
941
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: resolved.reason || 'provider_unusable', nodeId });
|
|
942
|
+
continue;
|
|
943
|
+
}
|
|
944
|
+
|
|
945
|
+
// Don't spawn a session for a (node, provider) already at its declared
|
|
946
|
+
// maxParallel cap — it would launch only to fail the claim. The claim
|
|
947
|
+
// transaction enforces the cap regardless; this just avoids a doomed launch.
|
|
948
|
+
const providerCap = resolveProviderMaxParallel(node?.policy, resolved.providerType);
|
|
949
|
+
if (
|
|
950
|
+
providerCap !== undefined
|
|
951
|
+
&& activeProviderAssignedCount(meshId, nodeId, resolved.providerType) >= providerCap
|
|
952
|
+
) {
|
|
953
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_provider_parallel_reached', nodeId, providerType: resolved.providerType });
|
|
954
|
+
continue;
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
// Shared worker-launch envelope. For a local node it spawns directly on this
|
|
958
|
+
// daemon; for a remote node the identical command is forwarded to the node's
|
|
959
|
+
// daemon (mirrors mesh_launch_session), with the coordinator daemonId stamped
|
|
960
|
+
// so the worker's completion events route back to this coordinator.
|
|
961
|
+
const launchSettings: Record<string, unknown> = {
|
|
962
|
+
// Worker launch envelope: role + mesh context so worker can route completion events.
|
|
963
|
+
role: 'worker',
|
|
964
|
+
meshNodeFor: meshId,
|
|
965
|
+
meshNodeId: nodeId,
|
|
966
|
+
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
967
|
+
// Coordinator-dispatched worker: auto-approve unless mesh/node policy
|
|
968
|
+
// opts out (default true). Lands in settingsOverride and beats the
|
|
969
|
+
// global per-provider-type autoApprove config (see shouldAutoApprove).
|
|
970
|
+
autoApprove: resolveDelegatedWorkerAutoApprove(mesh?.policy, node?.policy),
|
|
971
|
+
launchedByCoordinator: true,
|
|
972
|
+
autoLaunchedForQueueTaskId: task.id,
|
|
973
|
+
};
|
|
974
|
+
|
|
975
|
+
if (launchTarget.mode === 'remote') {
|
|
976
|
+
// Relay-safe completion routing: stamp the coordinator anchor the same way
|
|
977
|
+
// mesh_launch_session does so the worker forwards events back to this daemon.
|
|
978
|
+
const remoteSettings: Record<string, unknown> = {
|
|
979
|
+
...launchSettings,
|
|
980
|
+
meshCoordinatorDaemonId: launchTarget.coordinatorDaemonId,
|
|
981
|
+
meshCoordinatorNodeId: nodeId,
|
|
982
|
+
};
|
|
983
|
+
markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
|
|
984
|
+
let launchResult: any;
|
|
985
|
+
try {
|
|
986
|
+
launchResult = await components.dispatchMeshCommand!(launchTarget.daemonId!, 'launch_cli', {
|
|
987
|
+
cliType: resolved.providerType,
|
|
988
|
+
dir: node.workspace,
|
|
989
|
+
settings: remoteSettings,
|
|
990
|
+
});
|
|
991
|
+
} catch (e: any) {
|
|
992
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason: `remote_launch_dispatch_failed: ${e?.message || String(e)}`, nodeId, providerType: resolved.providerType });
|
|
993
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
994
|
+
return false;
|
|
995
|
+
}
|
|
996
|
+
const payload = (launchResult && typeof launchResult === 'object' && 'payload' in launchResult && launchResult.payload && typeof launchResult.payload === 'object')
|
|
997
|
+
? launchResult.payload
|
|
998
|
+
: launchResult;
|
|
999
|
+
if (!payload?.success) {
|
|
1000
|
+
const reason = readNonEmptyString(payload?.error) || 'remote_launch_cli_failed';
|
|
1001
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason, nodeId, providerType: resolved.providerType });
|
|
1002
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
1003
|
+
return false;
|
|
1004
|
+
}
|
|
1005
|
+
// Remote launch is async: the worker session will register and emit agent:ready,
|
|
1006
|
+
// which (forwarded back here) drives the claim via the normal event path / PHASE 1
|
|
1007
|
+
// reconcile. Set a cooldown so the 4s loop doesn't re-launch before that lands.
|
|
1008
|
+
const remoteSessionId = readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.id) || readNonEmptyString(payload.runtimeSessionId);
|
|
1009
|
+
markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId: remoteSessionId || undefined });
|
|
1010
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
1011
|
+
return true;
|
|
1012
|
+
}
|
|
1013
|
+
|
|
1014
|
+
markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
|
|
1015
|
+
const launchResult: any = await components.cliManager.handleCliCommand('launch_cli', {
|
|
1016
|
+
cliType: resolved.providerType,
|
|
1017
|
+
dir: node.workspace,
|
|
1018
|
+
settings: launchSettings,
|
|
1019
|
+
});
|
|
1020
|
+
if (!launchResult?.success) {
|
|
1021
|
+
const reason = launchResult?.error || 'launch_cli_failed';
|
|
1022
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason, nodeId, providerType: resolved.providerType });
|
|
1023
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
1024
|
+
return false;
|
|
1025
|
+
}
|
|
1026
|
+
const sessionId = readNonEmptyString(launchResult.sessionId) || readNonEmptyString(launchResult.id) || readNonEmptyString(launchResult.runtimeSessionId);
|
|
1027
|
+
if (!sessionId) {
|
|
1028
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason: 'launch_missing_session_id', nodeId, providerType: resolved.providerType });
|
|
1029
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
1030
|
+
return false;
|
|
1031
|
+
}
|
|
1032
|
+
markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId });
|
|
1033
|
+
tryAssignQueueTask(components, meshId, nodeId, sessionId, resolved.providerType);
|
|
1034
|
+
return true;
|
|
1035
|
+
} catch (e: any) {
|
|
1036
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', error: e?.message || String(e), nodeId });
|
|
1037
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
1038
|
+
return false;
|
|
1039
|
+
} finally {
|
|
1040
|
+
autoLaunchInProgress.delete(launchKey);
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
1043
|
+
}
|
|
1044
|
+
return false;
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
export interface MeshQueueTriggerResult {
|
|
1048
|
+
success: true;
|
|
1049
|
+
meshId: string;
|
|
1050
|
+
pendingBefore: number;
|
|
1051
|
+
assignedBefore: number;
|
|
1052
|
+
pendingAfter: number;
|
|
1053
|
+
assignedAfter: number;
|
|
1054
|
+
claimed: boolean;
|
|
1055
|
+
newlyAssignedTasks: Array<{
|
|
1056
|
+
id: string;
|
|
1057
|
+
nodeId?: string;
|
|
1058
|
+
sessionId?: string;
|
|
1059
|
+
}>;
|
|
1060
|
+
localIdleSessionsChecked: number;
|
|
1061
|
+
remoteIdleSessionsChecked: number;
|
|
1062
|
+
skippedSessions: Array<{
|
|
1063
|
+
nodeId?: string;
|
|
1064
|
+
sessionId?: string;
|
|
1065
|
+
reason: string;
|
|
1066
|
+
status?: string;
|
|
1067
|
+
}>;
|
|
1068
|
+
autoLaunchStarted: boolean;
|
|
1069
|
+
noIdleMeshSessionAvailable?: boolean;
|
|
1070
|
+
}
|
|
1071
|
+
|
|
1072
|
+
function countQueueStatus(meshId: string, status: 'pending' | 'assigned'): number {
|
|
1073
|
+
return getQueue(meshId, { status: [status] as any }).length;
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
function getQueueStatusById(meshId: string): Map<string, string> {
|
|
1077
|
+
return new Map(getQueue(meshId).map(task => [task.id, task.status]));
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
export async function triggerMeshQueue(components: DaemonComponents, meshId: string): Promise<MeshQueueTriggerResult> {
|
|
1081
|
+
const mesh = getMeshWithCache(components, meshId);
|
|
1082
|
+
const pendingBefore = countQueueStatus(meshId, 'pending');
|
|
1083
|
+
const assignedBefore = countQueueStatus(meshId, 'assigned');
|
|
1084
|
+
const beforeStatus = getQueueStatusById(meshId);
|
|
1085
|
+
const skippedSessions: MeshQueueTriggerResult['skippedSessions'] = [];
|
|
1086
|
+
let localIdleSessionsChecked = 0;
|
|
1087
|
+
let remoteIdleSessionsChecked = 0;
|
|
1088
|
+
let autoLaunchStarted = false;
|
|
1089
|
+
if (!mesh) {
|
|
1090
|
+
return {
|
|
1091
|
+
success: true,
|
|
1092
|
+
meshId,
|
|
1093
|
+
pendingBefore,
|
|
1094
|
+
assignedBefore,
|
|
1095
|
+
pendingAfter: pendingBefore,
|
|
1096
|
+
assignedAfter: assignedBefore,
|
|
1097
|
+
claimed: false,
|
|
1098
|
+
newlyAssignedTasks: [],
|
|
1099
|
+
localIdleSessionsChecked,
|
|
1100
|
+
remoteIdleSessionsChecked,
|
|
1101
|
+
skippedSessions: [{ reason: 'mesh_not_found' }],
|
|
1102
|
+
autoLaunchStarted,
|
|
1103
|
+
noIdleMeshSessionAvailable: true,
|
|
1104
|
+
};
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
// Collect every idle mesh session (local CLI instances + remote idle records)
|
|
1108
|
+
// as drain candidates. The drain ORDER depends on the scheduling strategy:
|
|
1109
|
+
// - 'first_eligible' (default): local-first, then remote, exactly as before.
|
|
1110
|
+
// - otherwise: local + remote merged into one pool and drained in scheduling
|
|
1111
|
+
// order (priority → load → tie-break). This local-first debias is required
|
|
1112
|
+
// because without it the coordinator's own local node is always visited
|
|
1113
|
+
// first and greedily absorbs all untargeted work before any remote idle
|
|
1114
|
+
// session is even considered — the comparator alone can't spread work if
|
|
1115
|
+
// local is always tried first.
|
|
1116
|
+
type IdleCandidate = { nodeId: string; sessionId: string; providerType: string; origin: 'local' | 'remote'; node: any };
|
|
1117
|
+
const strategy = resolveSchedulingStrategy(mesh);
|
|
1118
|
+
const localCandidates: IdleCandidate[] = [];
|
|
1119
|
+
|
|
1120
|
+
const cliInstances = components.instanceManager.getByCategory('cli');
|
|
1121
|
+
for (const inst of cliInstances) {
|
|
1122
|
+
const state = inst.getState();
|
|
1123
|
+
const settings = state.settings as Record<string, unknown> || {};
|
|
1124
|
+
|
|
1125
|
+
const instMeshId = readNonEmptyString(settings.meshNodeFor);
|
|
1126
|
+
if (instMeshId !== meshId) continue;
|
|
1127
|
+
|
|
1128
|
+
const nodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
1129
|
+
if (!nodeId) continue;
|
|
1130
|
+
|
|
1131
|
+
if (!isIdleSessionState(state)) {
|
|
1132
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
1133
|
+
skippedSessions.push({
|
|
1134
|
+
nodeId,
|
|
1135
|
+
sessionId: readNonEmptyString(state.instanceId),
|
|
1136
|
+
reason: isTerminalSessionStatus(status) ? 'terminal_session' : 'session_not_idle',
|
|
1137
|
+
status: status || undefined,
|
|
1138
|
+
});
|
|
1139
|
+
continue;
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
const sessionId = state.instanceId;
|
|
1143
|
+
const providerType = state.type || readNonEmptyString(settings.providerType);
|
|
1144
|
+
|
|
1145
|
+
if (providerType) {
|
|
1146
|
+
localIdleSessionsChecked += 1;
|
|
1147
|
+
localCandidates.push({ nodeId, sessionId, providerType, origin: 'local', node: mesh.nodes.find((n: any) => readMeshNodeId(n) === nodeId) });
|
|
1148
|
+
} else {
|
|
1149
|
+
skippedSessions.push({
|
|
1150
|
+
nodeId,
|
|
1151
|
+
sessionId,
|
|
1152
|
+
reason: 'provider_type_missing',
|
|
1153
|
+
});
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
let remoteSessions: Array<{ nodeId: string; sessionId: string; providerType: string }> = [];
|
|
1158
|
+
try {
|
|
1159
|
+
remoteSessions = MeshRuntimeStore.getInstance().getRemoteIdleSessions();
|
|
1160
|
+
} catch { /* best-effort */ }
|
|
1161
|
+
|
|
1162
|
+
const remoteCandidates: IdleCandidate[] = [];
|
|
1163
|
+
for (const idle of remoteSessions) {
|
|
1164
|
+
// Match with the shared 3-form normalizer (id / nodeId / node_id), not raw
|
|
1165
|
+
// `n.id`, so an inline-cached worktree node whose identity arrived under a
|
|
1166
|
+
// different form is not silently dropped — leaving a remote idle session
|
|
1167
|
+
// unable to claim its pending queue task.
|
|
1168
|
+
const node = mesh.nodes.find((n: any) => meshNodeIdMatches(n, idle.nodeId));
|
|
1169
|
+
if (node) {
|
|
1170
|
+
remoteIdleSessionsChecked += 1;
|
|
1171
|
+
remoteCandidates.push({ nodeId: idle.nodeId, sessionId: idle.sessionId, providerType: idle.providerType, origin: 'remote', node });
|
|
1172
|
+
}
|
|
1173
|
+
}
|
|
1174
|
+
|
|
1175
|
+
const assignIdleCandidate = (candidate: IdleCandidate): void => {
|
|
1176
|
+
const assigned = tryAssignQueueTask(components, meshId, candidate.nodeId, candidate.sessionId, candidate.providerType);
|
|
1177
|
+
if (assigned && candidate.origin === 'remote') {
|
|
1178
|
+
try {
|
|
1179
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(candidate.nodeId, candidate.sessionId);
|
|
1180
|
+
} catch { /* best-effort */ }
|
|
1181
|
+
}
|
|
1182
|
+
};
|
|
1183
|
+
|
|
1184
|
+
if (strategy === 'first_eligible') {
|
|
1185
|
+
// Strict no-change: drain local idle sessions first (original order), then
|
|
1186
|
+
// remote idle sessions. tryAssignQueueTask is a no-op when nothing matches.
|
|
1187
|
+
for (const candidate of localCandidates) assignIdleCandidate(candidate);
|
|
1188
|
+
for (const candidate of remoteCandidates) assignIdleCandidate(candidate);
|
|
1189
|
+
} else {
|
|
1190
|
+
// Merge local + remote into one pool and drain in scheduling order. Each
|
|
1191
|
+
// assignment mutates a node's active load, and the next pick re-reads it,
|
|
1192
|
+
// so re-ranking after every assignment keeps the spread fair as load shifts.
|
|
1193
|
+
const pool = [...localCandidates, ...remoteCandidates];
|
|
1194
|
+
const baseIndex = new Map<string, number>();
|
|
1195
|
+
pool.forEach((c, i) => { if (!baseIndex.has(c.nodeId)) baseIndex.set(c.nodeId, i); });
|
|
1196
|
+
// Bump the round-robin cursor once for this whole drain pass.
|
|
1197
|
+
const uniqueNodes = [...new Set(pool.map(c => c.nodeId))]
|
|
1198
|
+
.map((nodeId, index) => ({ nodeId, node: pool.find(c => c.nodeId === nodeId)?.node, index }));
|
|
1199
|
+
const ranked = orderEligibleNodes(meshId, strategy, uniqueNodes, { bumpCursor: true });
|
|
1200
|
+
const rankIndex = new Map<string, number>(ranked.map((r, i) => [r.nodeId, i]));
|
|
1201
|
+
const remaining = [...pool];
|
|
1202
|
+
while (remaining.length > 0) {
|
|
1203
|
+
// Re-rank each pass so a node that just took work defers its next session.
|
|
1204
|
+
remaining.sort((a, b) => {
|
|
1205
|
+
const aPrio = resolveNodeSchedulingPriority(a.node?.policy);
|
|
1206
|
+
const bPrio = resolveNodeSchedulingPriority(b.node?.policy);
|
|
1207
|
+
if (aPrio !== bPrio) return bPrio - aPrio;
|
|
1208
|
+
if (strategy === 'least_loaded' || strategy === 'round_robin') {
|
|
1209
|
+
const loadDelta = nodeActiveLoad(meshId, a.nodeId) - nodeActiveLoad(meshId, b.nodeId);
|
|
1210
|
+
if (loadDelta !== 0) return loadDelta;
|
|
1211
|
+
}
|
|
1212
|
+
return (rankIndex.get(a.nodeId) ?? 0) - (rankIndex.get(b.nodeId) ?? 0);
|
|
1213
|
+
});
|
|
1214
|
+
assignIdleCandidate(remaining.shift()!);
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
autoLaunchStarted = await maybeAutoLaunchOneQueueSession(components, meshId, mesh);
|
|
1219
|
+
const afterQueue = getQueue(meshId);
|
|
1220
|
+
const pendingAfter = afterQueue.filter(task => task.status === 'pending').length;
|
|
1221
|
+
const assignedAfter = afterQueue.filter(task => task.status === 'assigned').length;
|
|
1222
|
+
const newlyAssignedTasks = afterQueue
|
|
1223
|
+
.filter(task => task.status === 'assigned' && beforeStatus.get(task.id) !== 'assigned')
|
|
1224
|
+
.map(task => ({
|
|
1225
|
+
id: task.id,
|
|
1226
|
+
nodeId: task.assignedNodeId,
|
|
1227
|
+
sessionId: task.assignedSessionId,
|
|
1228
|
+
}));
|
|
1229
|
+
return {
|
|
1230
|
+
success: true,
|
|
1231
|
+
meshId,
|
|
1232
|
+
pendingBefore,
|
|
1233
|
+
assignedBefore,
|
|
1234
|
+
pendingAfter,
|
|
1235
|
+
assignedAfter,
|
|
1236
|
+
claimed: newlyAssignedTasks.length > 0,
|
|
1237
|
+
newlyAssignedTasks,
|
|
1238
|
+
localIdleSessionsChecked,
|
|
1239
|
+
remoteIdleSessionsChecked,
|
|
1240
|
+
skippedSessions,
|
|
1241
|
+
autoLaunchStarted,
|
|
1242
|
+
...(pendingAfter > 0 && newlyAssignedTasks.length === 0 && localIdleSessionsChecked === 0 && remoteIdleSessionsChecked === 0 && !autoLaunchStarted
|
|
1243
|
+
? { noIdleMeshSessionAvailable: true }
|
|
1244
|
+
: {}),
|
|
1245
|
+
};
|
|
1246
|
+
}
|
|
1247
|
+
|
|
1248
|
+
async function maybeAutoFastForwardIdleNode(components: DaemonComponents, args: {
|
|
1249
|
+
meshId: string;
|
|
1250
|
+
nodeId: string;
|
|
1251
|
+
sessionId?: string;
|
|
1252
|
+
providerType?: string;
|
|
1253
|
+
}): Promise<void> {
|
|
1254
|
+
const mesh = getMeshWithCache(components, args.meshId);
|
|
1255
|
+
const node = mesh?.nodes?.find((candidate: any) => meshNodeIdMatches(candidate, args.nodeId));
|
|
1256
|
+
const workspace = readNonEmptyString(node?.workspace);
|
|
1257
|
+
if (!workspace) return;
|
|
1258
|
+
if (!existsSync(workspace)) return;
|
|
1259
|
+
|
|
1260
|
+
const policy = resolveAutoFastForwardPolicy(mesh);
|
|
1261
|
+
if (!policy.enabled) return;
|
|
1262
|
+
if (nodeHasActiveMeshWork(components, args.meshId, args.nodeId, args.sessionId)) return;
|
|
1263
|
+
|
|
1264
|
+
const throttleKey = `${args.meshId}:${args.nodeId}`;
|
|
1265
|
+
const now = Date.now();
|
|
1266
|
+
const lastAttempt = idleAutoFastForwardLastAttempt.get(throttleKey) || 0;
|
|
1267
|
+
if (now - lastAttempt < IDLE_AUTO_FAST_FORWARD_THROTTLE_MS) return;
|
|
1268
|
+
idleAutoFastForwardLastAttempt.set(throttleKey, now);
|
|
1269
|
+
|
|
1270
|
+
const submoduleIgnorePaths = Array.isArray(node?.policy?.submoduleIgnorePaths)
|
|
1271
|
+
? node.policy.submoduleIgnorePaths.filter((value: unknown): value is string => typeof value === 'string')
|
|
1272
|
+
: undefined;
|
|
1273
|
+
try {
|
|
1274
|
+
const dryRun = await fastForwardMeshNode({
|
|
1275
|
+
meshId: args.meshId,
|
|
1276
|
+
nodeId: args.nodeId,
|
|
1277
|
+
workspace,
|
|
1278
|
+
execute: false,
|
|
1279
|
+
dryRun: true,
|
|
1280
|
+
updateSubmodules: false,
|
|
1281
|
+
submoduleIgnorePaths,
|
|
1282
|
+
trigger: 'idle_auto',
|
|
1283
|
+
});
|
|
1284
|
+
if (!dryRun || dryRun.code !== 'fast_forward_available' || dryRun.allowed !== true) return;
|
|
1285
|
+
const behind = Number(dryRun.current?.behind);
|
|
1286
|
+
if (policy.maxBehind !== undefined && Number.isFinite(behind) && behind > policy.maxBehind) return;
|
|
1287
|
+
if (policy.requireCleanSubmodules) {
|
|
1288
|
+
const submodules = Array.isArray(dryRun.current?.submodules) ? dryRun.current.submodules : [];
|
|
1289
|
+
if (submodules.some((submodule: any) => submodule?.dirty || submodule?.outOfSync || submodule?.error)) return;
|
|
1290
|
+
}
|
|
1291
|
+
await fastForwardMeshNode({
|
|
1292
|
+
meshId: args.meshId,
|
|
1293
|
+
nodeId: args.nodeId,
|
|
1294
|
+
workspace,
|
|
1295
|
+
execute: true,
|
|
1296
|
+
dryRun: false,
|
|
1297
|
+
updateSubmodules: false,
|
|
1298
|
+
submoduleIgnorePaths,
|
|
1299
|
+
trigger: 'idle_auto',
|
|
1300
|
+
});
|
|
1301
|
+
} catch (e: any) {
|
|
1302
|
+
LOG.warn('MeshFastForward', `Idle auto fast-forward check failed for ${args.nodeId}: ${e?.message || e}`);
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
|
|
1306
|
+
function runIdleMaintenanceThenAssignQueue(components: DaemonComponents, args: {
|
|
1307
|
+
meshId: string;
|
|
1308
|
+
nodeId: string;
|
|
1309
|
+
sessionId: string;
|
|
1310
|
+
providerType: string;
|
|
1311
|
+
}): void {
|
|
1312
|
+
setImmediate(() => {
|
|
1313
|
+
maybeAutoFastForwardIdleNode(components, args)
|
|
1314
|
+
.finally(() => {
|
|
1315
|
+
try {
|
|
1316
|
+
tryAssignQueueTask(components, args.meshId, args.nodeId, args.sessionId, args.providerType);
|
|
1317
|
+
} catch (e: any) {
|
|
1318
|
+
LOG.warn('MeshQueue', `Failed to assign idle queue task after maintenance for ${args.nodeId}: ${e?.message || e}`);
|
|
1319
|
+
}
|
|
1320
|
+
});
|
|
1321
|
+
});
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
// ---------------------------------------------------------------------------
|
|
1325
|
+
// Core event injection
|
|
1326
|
+
// ---------------------------------------------------------------------------
|
|
1327
|
+
|
|
1328
|
+
const MESH_COORDINATOR_EVENTS = new Set([
|
|
1329
|
+
'agent:generating_started',
|
|
1330
|
+
'agent:generating_completed',
|
|
1331
|
+
'agent:waiting_approval',
|
|
1332
|
+
'agent:stopped',
|
|
1333
|
+
'agent:ready',
|
|
1334
|
+
'monitor:no_progress',
|
|
1335
|
+
'refine:accepted',
|
|
1336
|
+
'refine:completed',
|
|
1337
|
+
'refine:failed',
|
|
1338
|
+
'worktree_bootstrap_complete',
|
|
1339
|
+
'worktree_bootstrap_failed',
|
|
1340
|
+
]);
|
|
1341
|
+
|
|
1342
|
+
const EVENT_TO_LEDGER_KIND: Record<string, MeshLedgerKind> = {
|
|
1343
|
+
'agent:generating_completed': 'task_completed',
|
|
1344
|
+
'agent:waiting_approval': 'task_approval_needed',
|
|
1345
|
+
'agent:stopped': 'task_failed',
|
|
1346
|
+
'monitor:no_progress': 'task_stalled',
|
|
1347
|
+
};
|
|
1348
|
+
|
|
1349
|
+
export function isMeshCoordinatorEvent(eventName: unknown): eventName is string {
|
|
1350
|
+
return typeof eventName === 'string' && MESH_COORDINATOR_EVENTS.has(eventName);
|
|
1351
|
+
}
|
|
1352
|
+
|
|
1353
|
+
// Terminal events that the coordinator is actively blocked waiting on. When the
|
|
1354
|
+
// coordinator CLI session dispatches a task (e.g. mesh_send_task) it stays in
|
|
1355
|
+
// `generating` until the result arrives — but a generating coordinator queues
|
|
1356
|
+
// incoming send_message calls into its adapter's pendingOutboundQueue, which is
|
|
1357
|
+
// only flushed on the coordinator's OWN idle transition. That transition can't
|
|
1358
|
+
// happen until it receives this very event → deadlock. We force-inject these so
|
|
1359
|
+
// they bypass the busy send-guard and land in the PTY while generating.
|
|
1360
|
+
export const MESH_FORCE_INJECT_EVENTS: ReadonlySet<string> = new Set([
|
|
1361
|
+
'agent:generating_completed',
|
|
1362
|
+
'agent:stopped',
|
|
1363
|
+
'agent:waiting_approval',
|
|
1364
|
+
'refine:completed',
|
|
1365
|
+
'refine:failed',
|
|
1366
|
+
'worktree_bootstrap_complete',
|
|
1367
|
+
'worktree_bootstrap_failed',
|
|
1368
|
+
]);
|
|
1369
|
+
|
|
1370
|
+
export function shouldForceInjectMeshEvent(eventName: unknown): boolean {
|
|
1371
|
+
return typeof eventName === 'string' && MESH_FORCE_INJECT_EVENTS.has(eventName);
|
|
1372
|
+
}
|
|
1373
|
+
|
|
1374
|
+
function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
1375
|
+
meshId: string;
|
|
1376
|
+
sourceInstanceId?: string;
|
|
1377
|
+
nodeId?: string;
|
|
1378
|
+
nodeLabel: string;
|
|
1379
|
+
event: string;
|
|
1380
|
+
metadataEvent: Record<string, unknown>;
|
|
1381
|
+
}) {
|
|
1382
|
+
const eventSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
1383
|
+
const eventNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1384
|
+
|
|
1385
|
+
const sourceSession = args.sourceInstanceId
|
|
1386
|
+
? components.instanceManager.getInstance(args.sourceInstanceId)
|
|
1387
|
+
: undefined;
|
|
1388
|
+
const workerCoordinatorDaemonId = readNonEmptyString(
|
|
1389
|
+
(sourceSession?.getState()?.settings as Record<string, unknown>)?.meshCoordinatorDaemonId,
|
|
1390
|
+
);
|
|
1391
|
+
|
|
1392
|
+
// R2: cloud P2P dashboard metadata sync. The cloud daemon used to do this from its own
|
|
1393
|
+
// relay listener; now the single core forwarder invokes the injected hook (no-op on
|
|
1394
|
+
// standalone) so the event path stays single-listener and the local code path is identical
|
|
1395
|
+
// across standalone and cloud.
|
|
1396
|
+
if (components.onMeshCoordinatorEventForwarded) {
|
|
1397
|
+
try {
|
|
1398
|
+
// T: the coordinator surfaces a remote worker's session but holds no local
|
|
1399
|
+
// instance for it, so the status snapshot can't derive a preview and the
|
|
1400
|
+
// mirror would stay stuck on the first dispatched user task. Resolve the
|
|
1401
|
+
// worker's latest assistant reply (carried on the completion event's
|
|
1402
|
+
// finalSummary / workerResult) into a preview the mirror can stamp, so the
|
|
1403
|
+
// mobile inbox reflects the assistant response. Only completion-style events
|
|
1404
|
+
// carry assistant text; for everything else this is undefined and the prior
|
|
1405
|
+
// surfaced preview is preserved downstream (no clobber).
|
|
1406
|
+
const surfacedPreview = resolveMeshSurfacedSessionPreview(args.metadataEvent);
|
|
1407
|
+
components.onMeshCoordinatorEventForwarded({
|
|
1408
|
+
event: args.event,
|
|
1409
|
+
meshId: args.meshId,
|
|
1410
|
+
nodeId: eventNodeId || undefined,
|
|
1411
|
+
...args.metadataEvent,
|
|
1412
|
+
// Ensure a `workspace` field reaches updateMeshOwnedSession even when the
|
|
1413
|
+
// worker provider event only carried `workspaceName`. The merge spread of
|
|
1414
|
+
// metadataEvent above wins when it already has a non-empty `workspace`.
|
|
1415
|
+
workspace: readNonEmptyString(args.metadataEvent.workspace)
|
|
1416
|
+
|| readNonEmptyString(args.metadataEvent.workspaceName)
|
|
1417
|
+
|| undefined,
|
|
1418
|
+
...(surfacedPreview ? {
|
|
1419
|
+
meshSessionLastMessagePreview: surfacedPreview.preview,
|
|
1420
|
+
meshSessionLastMessageRole: surfacedPreview.role,
|
|
1421
|
+
meshSessionLastMessageAt: surfacedPreview.receivedAt || undefined,
|
|
1422
|
+
} : {}),
|
|
1423
|
+
});
|
|
1424
|
+
} catch { /* dashboard metadata sync is best-effort */ }
|
|
1425
|
+
}
|
|
1426
|
+
|
|
1427
|
+
const intentionalCleanupStop = shouldSuppressIntentionalCleanupStop({
|
|
1428
|
+
event: args.event,
|
|
1429
|
+
meshId: args.meshId,
|
|
1430
|
+
metadataEvent: args.metadataEvent,
|
|
1431
|
+
sessionId: eventSessionId || undefined,
|
|
1432
|
+
nodeId: eventNodeId || undefined,
|
|
1433
|
+
});
|
|
1434
|
+
if (intentionalCleanupStop) {
|
|
1435
|
+
if (eventSessionId && eventNodeId) {
|
|
1436
|
+
try {
|
|
1437
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(eventNodeId, eventSessionId);
|
|
1438
|
+
} catch { /* best-effort */ }
|
|
1439
|
+
}
|
|
1440
|
+
LOG.info('MeshEvents', `Suppressed ${args.event} for intentionally cleanup-stopped session ${eventSessionId || '(unknown session)'}`);
|
|
1441
|
+
return { success: true, forwarded: 0, suppressed: true, intentionalCleanupStop: true };
|
|
1442
|
+
}
|
|
1443
|
+
|
|
1444
|
+
if (args.event === 'monitor:no_progress') {
|
|
1445
|
+
const reconciledCompletion = buildNoProgressCompletionReconciliation({
|
|
1446
|
+
meshId: args.meshId,
|
|
1447
|
+
nodeId: args.nodeId,
|
|
1448
|
+
nodeLabel: args.nodeLabel,
|
|
1449
|
+
metadataEvent: args.metadataEvent,
|
|
1450
|
+
sourceInstanceId: args.sourceInstanceId,
|
|
1451
|
+
});
|
|
1452
|
+
if (reconciledCompletion?.source === 'no_progress_reconciliation') {
|
|
1453
|
+
LOG.info('MeshEvents', `Reconciled no-progress monitor to completion for session ${eventSessionId || '(unknown session)'}`);
|
|
1454
|
+
return injectMeshSystemMessage(components, {
|
|
1455
|
+
...args,
|
|
1456
|
+
event: 'agent:generating_completed',
|
|
1457
|
+
metadataEvent: reconciledCompletion,
|
|
1458
|
+
});
|
|
1459
|
+
}
|
|
1460
|
+
if (reconciledCompletion?.source === 'no_progress_terminal_ledger_suppression') {
|
|
1461
|
+
LOG.info('MeshEvents', `Suppressed no-progress monitor because terminal ledger evidence already exists for session ${eventSessionId || '(unknown session)'}`);
|
|
1462
|
+
return {
|
|
1463
|
+
success: true,
|
|
1464
|
+
forwarded: 0,
|
|
1465
|
+
suppressed: true,
|
|
1466
|
+
terminalLedgerEvidence: true,
|
|
1467
|
+
terminalLedgerKind: reconciledCompletion.terminalLedgerKind,
|
|
1468
|
+
};
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
if (isDuplicateRefineTerminalEvent(args.meshId, args.event, args.metadataEvent)) {
|
|
1473
|
+
LOG.info('MeshEvents', `Suppressed duplicate ${args.event} for refine job ${readRefineJobId({ metadataEvent: args.metadataEvent })}`);
|
|
1474
|
+
return { success: true, forwarded: 0, suppressed: true, duplicateRefineTerminalEvent: true };
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
const eventTimestamp = readEventTimestamp(args.metadataEvent.timestamp);
|
|
1478
|
+
if (args.event === 'agent:waiting_approval' && eventSessionId) {
|
|
1479
|
+
const duplicateApproval = isDuplicateMeshApprovalEvent({
|
|
1480
|
+
meshId: args.meshId,
|
|
1481
|
+
sessionId: eventSessionId,
|
|
1482
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
1483
|
+
timestamp: eventTimestamp,
|
|
1484
|
+
modalMessage: readNonEmptyString(args.metadataEvent.modalMessage) || undefined,
|
|
1485
|
+
modalButtons: args.metadataEvent.modalButtons,
|
|
1486
|
+
});
|
|
1487
|
+
if (duplicateApproval) {
|
|
1488
|
+
LOG.info('MeshEvents', `Suppressed duplicate approval event for mesh ${args.meshId} session ${eventSessionId}`);
|
|
1489
|
+
return { success: true, forwarded: 0, suppressed: true, duplicateApproval: true };
|
|
1490
|
+
}
|
|
1491
|
+
}
|
|
1492
|
+
if (args.event === 'agent:generating_completed' && eventSessionId) {
|
|
1493
|
+
const terminal = findRecentTerminalLedgerEvidence({
|
|
1494
|
+
meshId: args.meshId,
|
|
1495
|
+
sessionId: eventSessionId,
|
|
1496
|
+
nodeId: eventNodeId || undefined,
|
|
1497
|
+
});
|
|
1498
|
+
if (terminal?.kind === 'task_completed' && !sessionHasActiveAssignment(args.meshId, eventSessionId)) {
|
|
1499
|
+
const newDispatchAfterTerminal = hasDispatchAfterTerminal(args.meshId, eventSessionId, terminal.id);
|
|
1500
|
+
if (!newDispatchAfterTerminal) {
|
|
1501
|
+
const terminalProviderSessionId = readNonEmptyString(terminal.payload.providerSessionId);
|
|
1502
|
+
const terminalFinalSummary = readNonEmptyString(terminal.payload.finalSummary);
|
|
1503
|
+
const eventProviderSessionId = readNonEmptyString(args.metadataEvent.providerSessionId);
|
|
1504
|
+
const eventFinalSummary = readNonEmptyString(args.metadataEvent.finalSummary);
|
|
1505
|
+
if (
|
|
1506
|
+
(terminalProviderSessionId && terminalProviderSessionId === eventProviderSessionId)
|
|
1507
|
+
|| (terminalFinalSummary && terminalFinalSummary === eventFinalSummary)
|
|
1508
|
+
|| args.metadataEvent.source === 'no_progress_reconciliation'
|
|
1509
|
+
) {
|
|
1510
|
+
LOG.info('MeshEvents', `Suppressed duplicate completion with existing terminal ledger evidence for mesh ${args.meshId} session ${eventSessionId}`);
|
|
1511
|
+
return { success: true, forwarded: 0, suppressed: true, duplicateCompletion: true, terminalLedgerEvidence: true };
|
|
1512
|
+
}
|
|
1513
|
+
}
|
|
1514
|
+
}
|
|
1515
|
+
const duplicateCompletion = isDuplicateMeshCompletionEvent({
|
|
1516
|
+
meshId: args.meshId,
|
|
1517
|
+
event: args.event,
|
|
1518
|
+
sessionId: eventSessionId,
|
|
1519
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
1520
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
1521
|
+
timestamp: eventTimestamp,
|
|
1522
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
1523
|
+
coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
|
|
1524
|
+
taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
|
|
1525
|
+
nodeId: eventNodeId || undefined,
|
|
1526
|
+
});
|
|
1527
|
+
if (duplicateCompletion) {
|
|
1528
|
+
LOG.info('MeshEvents', `Suppressed duplicate completion for mesh ${args.meshId} session ${eventSessionId}`);
|
|
1529
|
+
return { success: true, forwarded: 0, suppressed: true, duplicateCompletion: true };
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
if (args.event === 'agent:stopped' && eventSessionId) {
|
|
1533
|
+
const duplicateStopped = isDuplicateMeshCompletionEvent({
|
|
1534
|
+
meshId: args.meshId,
|
|
1535
|
+
event: args.event,
|
|
1536
|
+
sessionId: eventSessionId,
|
|
1537
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
1538
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
1539
|
+
timestamp: eventTimestamp,
|
|
1540
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
1541
|
+
coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
|
|
1542
|
+
taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
|
|
1543
|
+
nodeId: eventNodeId || undefined,
|
|
1544
|
+
});
|
|
1545
|
+
if (duplicateStopped) {
|
|
1546
|
+
LOG.info('MeshEvents', `Suppressed duplicate stopped event for mesh ${args.meshId} session ${eventSessionId}`);
|
|
1547
|
+
return { success: true, forwarded: 0, suppressed: true, duplicateStopped: true };
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
function markSessionTerminal(sessionId: string, outcome: 'completed' | 'failed', occurredAtMs?: number | null): { id?: string } | null {
|
|
1552
|
+
// C2: prefer an exact taskId match when the completion event carries one —
|
|
1553
|
+
// it's immune to coordinator↔worker clock skew that can hide the assigned row.
|
|
1554
|
+
const eventTaskId = readNonEmptyString(args.metadataEvent.taskId) || undefined;
|
|
1555
|
+
const task = updateSessionTaskStatus(args.meshId, sessionId, outcome, {
|
|
1556
|
+
occurredAt: occurredAtMs != null ? new Date(occurredAtMs).toISOString() : undefined,
|
|
1557
|
+
taskId: eventTaskId,
|
|
1558
|
+
});
|
|
1559
|
+
updateDirectDispatchStatus(args.meshId, sessionId, outcome);
|
|
1560
|
+
markSessionDeliveriesTerminal(args.meshId, sessionId, outcome);
|
|
1561
|
+
setImmediate(() => cleanupTerminalDirectDispatches());
|
|
1562
|
+
return task ? { id: task.id } : null;
|
|
1563
|
+
}
|
|
1564
|
+
|
|
1565
|
+
let completedTaskForLedger: { id?: string } | null = null;
|
|
1566
|
+
if (args.event === 'agent:generating_completed') {
|
|
1567
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
1568
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1569
|
+
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
1570
|
+
|
|
1571
|
+
if (sessionId) {
|
|
1572
|
+
completedTaskForLedger = markSessionTerminal(sessionId, 'completed', eventTimestamp);
|
|
1573
|
+
if (nodeId && providerType) {
|
|
1574
|
+
runIdleMaintenanceThenAssignQueue(components, { meshId: args.meshId, nodeId, sessionId, providerType });
|
|
1575
|
+
}
|
|
1576
|
+
// M1-3: wake dependents of the completed task. The maintenance path above
|
|
1577
|
+
// only assigns to the completing session; dependents may be claimable by
|
|
1578
|
+
// other idle sessions, so run a full queue trigger when any are waiting.
|
|
1579
|
+
const completedTaskId = completedTaskForLedger?.id;
|
|
1580
|
+
if (completedTaskId && hasPendingDependents(args.meshId, completedTaskId)) {
|
|
1581
|
+
setImmediate(() => {
|
|
1582
|
+
triggerMeshQueue(components, args.meshId).catch((e: any) => {
|
|
1583
|
+
LOG.warn('MeshQueue', `Dependent wake after task ${completedTaskId} failed: ${e?.message || e}`);
|
|
1584
|
+
});
|
|
1585
|
+
});
|
|
1586
|
+
}
|
|
1587
|
+
}
|
|
1588
|
+
} else if (args.event === 'agent:ready') {
|
|
1589
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
1590
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1591
|
+
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
1592
|
+
const providerSessionId = readNonEmptyString(args.metadataEvent.providerSessionId) || undefined;
|
|
1593
|
+
const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary) || undefined;
|
|
1594
|
+
const workerResult = readWorkerResultMetadata(args.metadataEvent);
|
|
1595
|
+
const hasCompletionEvidence = !!finalSummary || !!workerResult;
|
|
1596
|
+
if (sessionId && hasCompletionEvidence) {
|
|
1597
|
+
completedTaskForLedger = markSessionTerminal(sessionId, 'completed');
|
|
1598
|
+
if (completedTaskForLedger) {
|
|
1599
|
+
try {
|
|
1600
|
+
appendLedgerEntry(args.meshId, {
|
|
1601
|
+
kind: 'task_completed',
|
|
1602
|
+
nodeId: nodeId || undefined,
|
|
1603
|
+
sessionId,
|
|
1604
|
+
providerType: providerType || undefined,
|
|
1605
|
+
payload: {
|
|
1606
|
+
event: args.event,
|
|
1607
|
+
nodeLabel: args.nodeLabel,
|
|
1608
|
+
taskId: completedTaskForLedger.id,
|
|
1609
|
+
completedViaReady: true,
|
|
1610
|
+
providerSessionId,
|
|
1611
|
+
finalSummary,
|
|
1612
|
+
workerResult,
|
|
1613
|
+
evidence: buildTaskCompletionEvidence({
|
|
1614
|
+
event: 'agent:ready',
|
|
1615
|
+
nodeId,
|
|
1616
|
+
sessionId,
|
|
1617
|
+
providerType: providerType || undefined,
|
|
1618
|
+
providerSessionId,
|
|
1619
|
+
finalSummary,
|
|
1620
|
+
workerResult,
|
|
1621
|
+
}),
|
|
1622
|
+
},
|
|
1623
|
+
});
|
|
1624
|
+
} catch (e: any) {
|
|
1625
|
+
LOG.warn('MeshLedger', `Failed to record task_completed from ready: ${e?.message || e}`);
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
}
|
|
1629
|
+
|
|
1630
|
+
if (sessionId && nodeId && providerType) {
|
|
1631
|
+
sweepExpiredRemoteIdleSessions();
|
|
1632
|
+
try {
|
|
1633
|
+
MeshRuntimeStore.getInstance().setRemoteIdleSession(nodeId, sessionId, providerType, Date.now() + REMOTE_IDLE_SESSION_TTL_MS);
|
|
1634
|
+
} catch { /* best-effort */ }
|
|
1635
|
+
setImmediate(() => {
|
|
1636
|
+
maybeAutoFastForwardIdleNode(components, { meshId: args.meshId, nodeId, sessionId, providerType })
|
|
1637
|
+
.finally(() => {
|
|
1638
|
+
try {
|
|
1639
|
+
const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
|
|
1640
|
+
if (assigned) MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
1641
|
+
} catch (e: any) {
|
|
1642
|
+
LOG.warn('MeshQueue', `Failed to assign idle queue task after maintenance for ${nodeId}: ${e?.message || e}`);
|
|
1643
|
+
}
|
|
1644
|
+
});
|
|
1645
|
+
});
|
|
1646
|
+
}
|
|
1647
|
+
} else if (args.event === 'agent:generating_started') {
|
|
1648
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
1649
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1650
|
+
if (sessionId && nodeId) {
|
|
1651
|
+
try {
|
|
1652
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
1653
|
+
} catch { /* best-effort */ }
|
|
1654
|
+
}
|
|
1655
|
+
if (sessionId) {
|
|
1656
|
+
updateDirectDispatchStatus(args.meshId, sessionId, 'acked');
|
|
1657
|
+
const activeDeliveries = ((): { id: string }[] => {
|
|
1658
|
+
try { return MeshRuntimeStore.getInstance().getActiveSessionDeliveries(args.meshId, sessionId); }
|
|
1659
|
+
catch { return []; }
|
|
1660
|
+
})();
|
|
1661
|
+
for (const d of activeDeliveries) {
|
|
1662
|
+
updateSessionDeliveryStatus(d.id, 'acked');
|
|
1663
|
+
}
|
|
1664
|
+
}
|
|
1665
|
+
} else if (args.event === 'agent:stopped') {
|
|
1666
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
1667
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1668
|
+
if (sessionId && nodeId) {
|
|
1669
|
+
try {
|
|
1670
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
1671
|
+
} catch { /* best-effort */ }
|
|
1672
|
+
}
|
|
1673
|
+
if (sessionId) {
|
|
1674
|
+
completedTaskForLedger = markSessionTerminal(sessionId, 'failed');
|
|
1675
|
+
}
|
|
1676
|
+
}
|
|
1677
|
+
|
|
1678
|
+
const ledgerKind = EVENT_TO_LEDGER_KIND[args.event];
|
|
1679
|
+
if (ledgerKind) {
|
|
1680
|
+
try {
|
|
1681
|
+
const ledgerNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined;
|
|
1682
|
+
const ledgerSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined;
|
|
1683
|
+
const ledgerProviderType = readNonEmptyString(args.metadataEvent.providerType) || undefined;
|
|
1684
|
+
const providerSessionId = readNonEmptyString(args.metadataEvent.providerSessionId) || undefined;
|
|
1685
|
+
const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary) || undefined;
|
|
1686
|
+
const workerResult = readWorkerResultMetadata(args.metadataEvent);
|
|
1687
|
+
const completionEvidence = ledgerKind === 'task_completed' && ledgerNodeId && ledgerSessionId
|
|
1688
|
+
? buildTaskCompletionEvidence({
|
|
1689
|
+
event: 'agent:generating_completed',
|
|
1690
|
+
nodeId: ledgerNodeId,
|
|
1691
|
+
sessionId: ledgerSessionId,
|
|
1692
|
+
providerType: ledgerProviderType,
|
|
1693
|
+
providerSessionId,
|
|
1694
|
+
finalSummary,
|
|
1695
|
+
workerResult,
|
|
1696
|
+
})
|
|
1697
|
+
: undefined;
|
|
1698
|
+
appendLedgerEntry(args.meshId, {
|
|
1699
|
+
kind: ledgerKind,
|
|
1700
|
+
nodeId: ledgerNodeId,
|
|
1701
|
+
sessionId: ledgerSessionId,
|
|
1702
|
+
providerType: ledgerProviderType,
|
|
1703
|
+
payload: {
|
|
1704
|
+
event: args.event,
|
|
1705
|
+
nodeLabel: args.nodeLabel,
|
|
1706
|
+
taskId: completedTaskForLedger?.id || undefined,
|
|
1707
|
+
providerSessionId,
|
|
1708
|
+
finalSummary,
|
|
1709
|
+
workerResult,
|
|
1710
|
+
completionDiagnostic: args.metadataEvent.completionDiagnostic && typeof args.metadataEvent.completionDiagnostic === 'object'
|
|
1711
|
+
? args.metadataEvent.completionDiagnostic
|
|
1712
|
+
: undefined,
|
|
1713
|
+
evidence: completionEvidence,
|
|
1714
|
+
// B2: evidenceLevel lets coordinator know when completion evidence is insufficient.
|
|
1715
|
+
...(completionEvidence
|
|
1716
|
+
? completionEvidence.workerResult.source === 'default'
|
|
1717
|
+
? { evidenceLevel: 'insufficient', reviewRecommended: true }
|
|
1718
|
+
: { evidenceLevel: 'sufficient' }
|
|
1719
|
+
: {}),
|
|
1720
|
+
},
|
|
1721
|
+
});
|
|
1722
|
+
} catch (e: any) {
|
|
1723
|
+
LOG.warn('MeshLedger', `Failed to record ${ledgerKind}: ${e?.message || e}`);
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
|
|
1727
|
+
let recoveryContext: SessionRecoveryContext | null = null;
|
|
1728
|
+
if (args.event === 'agent:stopped') {
|
|
1729
|
+
try {
|
|
1730
|
+
const mesh = getMesh(args.meshId);
|
|
1731
|
+
const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
|
|
1732
|
+
|
|
1733
|
+
recoveryContext = getSessionRecoveryContext(args.meshId, {
|
|
1734
|
+
sessionId: resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined,
|
|
1735
|
+
nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
|
|
1736
|
+
maxRetries,
|
|
1737
|
+
});
|
|
1738
|
+
recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
|
|
1739
|
+
|
|
1740
|
+
if (recoveryContext.retryRecommended && recoveryContext.consecutiveNodeFailures > 0) {
|
|
1741
|
+
appendLedgerEntry(args.meshId, {
|
|
1742
|
+
kind: 'recovery_attempted',
|
|
1743
|
+
nodeId: recoveryContext.failedNodeId || undefined,
|
|
1744
|
+
sessionId: recoveryContext.failedSessionId || undefined,
|
|
1745
|
+
providerType: recoveryContext.failedProviderType || undefined,
|
|
1746
|
+
payload: {
|
|
1747
|
+
consecutiveFailures: recoveryContext.consecutiveNodeFailures,
|
|
1748
|
+
taskAttemptCount: recoveryContext.taskAttemptCount,
|
|
1749
|
+
retryRecommended: recoveryContext.retryRecommended,
|
|
1750
|
+
advice: recoveryContext.advice,
|
|
1751
|
+
},
|
|
1752
|
+
});
|
|
1753
|
+
|
|
1754
|
+
if (recoveryContext.lastTaskMessage && recoveryContext.failedNodeId && recoveryContext.failedProviderType) {
|
|
1755
|
+
const autoNodeId = recoveryContext.failedNodeId;
|
|
1756
|
+
try {
|
|
1757
|
+
const task = enqueueTask(args.meshId, recoveryContext.lastTaskMessage, {
|
|
1758
|
+
targetNodeId: autoNodeId
|
|
1759
|
+
});
|
|
1760
|
+
LOG.info('MeshRecovery', `Auto-requeued failed task: ${task.id} for node ${autoNodeId}`);
|
|
1761
|
+
|
|
1762
|
+
const node = mesh?.nodes.find((n: any) => n.id === autoNodeId);
|
|
1763
|
+
if (node) {
|
|
1764
|
+
components.cliManager.handleCliCommand('launch_cli', {
|
|
1765
|
+
cliType: recoveryContext.failedProviderType,
|
|
1766
|
+
dir: node.workspace,
|
|
1767
|
+
settings: {
|
|
1768
|
+
role: 'worker',
|
|
1769
|
+
meshNodeFor: args.meshId,
|
|
1770
|
+
meshNodeId: node.id,
|
|
1771
|
+
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
1772
|
+
// Coordinator-dispatched recovery relaunch: same auto-approve
|
|
1773
|
+
// policy as the primary worker launch path.
|
|
1774
|
+
autoApprove: resolveDelegatedWorkerAutoApprove(mesh?.policy, node?.policy),
|
|
1775
|
+
launchedByCoordinator: true,
|
|
1776
|
+
}
|
|
1777
|
+
}).catch((e: any) => LOG.error('MeshRecovery', `Failed to auto-relaunch session for ${node.id}: ${e?.message}`));
|
|
1778
|
+
}
|
|
1779
|
+
} catch (e: any) {
|
|
1780
|
+
LOG.warn('MeshRecovery', `Failed to execute auto-recovery: ${e?.message}`);
|
|
1781
|
+
}
|
|
1782
|
+
}
|
|
1783
|
+
}
|
|
1784
|
+
|
|
1785
|
+
LOG.info('MeshRecovery', `Recovery context for ${args.nodeLabel}: ${recoveryContext.advice}`);
|
|
1786
|
+
} catch (e: any) {
|
|
1787
|
+
LOG.warn('MeshRecovery', `Failed to build recovery context: ${e?.message || e}`);
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1791
|
+
const messageText = buildMeshSystemMessage({
|
|
1792
|
+
event: args.event,
|
|
1793
|
+
nodeLabel: args.nodeLabel,
|
|
1794
|
+
metadataEvent: args.metadataEvent,
|
|
1795
|
+
recoveryContext,
|
|
1796
|
+
});
|
|
1797
|
+
if (!messageText) {
|
|
1798
|
+
// Lifecycle events that carry no coordinator-facing message (agent:ready /
|
|
1799
|
+
// agent:generating_started) still drive the remote-claim state machine: the
|
|
1800
|
+
// coordinator's agent:ready branch above runs setRemoteIdleSession +
|
|
1801
|
+
// tryAssignQueueTask, and agent:generating_started clears the remote-idle entry.
|
|
1802
|
+
// For a LOCAL worker whose coordinator is a REMOTE daemon those side effects ran
|
|
1803
|
+
// on the wrong daemon (this worker's empty queue / store), so the coordinator never
|
|
1804
|
+
// learns the auto-launched session went idle and re-auto-launches it forever
|
|
1805
|
+
// (queue task stuck pending). Queue the silent event so the coordinator pulls it
|
|
1806
|
+
// (PHASE 1 pullRemoteNodeQueues → handleMeshForwardEvent) and re-runs the claim on
|
|
1807
|
+
// the daemon that actually owns the queue. Gate strictly on a present, REMOTE
|
|
1808
|
+
// coordinator daemon id: a co-located worker already ran the claim on the right
|
|
1809
|
+
// daemon, and a coordinator processing a *pulled* event has no sourceSession so
|
|
1810
|
+
// workerCoordinatorDaemonId is empty — neither re-queues, so there is no loop.
|
|
1811
|
+
const isSilentClaimRelevantEvent = args.event === 'agent:ready' || args.event === 'agent:generating_started';
|
|
1812
|
+
const coordinatorIsRemote = !!workerCoordinatorDaemonId
|
|
1813
|
+
&& !resolveCoordinatorDrainDaemonIds(components).includes(workerCoordinatorDaemonId);
|
|
1814
|
+
if (!(isSilentClaimRelevantEvent && coordinatorIsRemote)) {
|
|
1815
|
+
return { success: false, error: 'unsupported mesh event' };
|
|
1816
|
+
}
|
|
1817
|
+
}
|
|
1818
|
+
|
|
1819
|
+
// ── Queue-only delivery (single-model: queue + periodic poll) ──────────────
|
|
1820
|
+
// Every mesh coordinator event — terminal or not, local-coordinator or
|
|
1821
|
+
// remote — is persisted to the pending-events queue (SQLite + JSONL) and
|
|
1822
|
+
// NOTHING is pushed here. The old spontaneous-forward paths were removed:
|
|
1823
|
+
// - F1 remote P2P `mesh_forward_event` dispatch (network/stamp-dependent,
|
|
1824
|
+
// silently dropped on P2P failure or missing meshCoordinatorDaemonId)
|
|
1825
|
+
// - F3 live-CLI PTY `send_message` fire-and-forget inject (silently
|
|
1826
|
+
// dropped when the coordinator was generating)
|
|
1827
|
+
// Delivery to a live CLI coordinator now happens via setupMeshReconcileLoop,
|
|
1828
|
+
// which drains this queue on a fixed interval and injects into the coordinator
|
|
1829
|
+
// only when it is idle. A pure stdio MCP (LLM) coordinator — which has no live
|
|
1830
|
+
// CLI session to inject into — drains the queue itself when it calls a mesh
|
|
1831
|
+
// tool (mesh_status / mesh_read_chat). Either way the queue is the single
|
|
1832
|
+
// source of truth and the only thing this function writes to.
|
|
1833
|
+
//
|
|
1834
|
+
// targetCoordinatorDaemonId scopes the event to a specific coordinator daemon
|
|
1835
|
+
// (unicast) when the worker carries one, so the reconcile loop on the right
|
|
1836
|
+
// daemon drains it and other daemons skip it. Absent → broadcast/backfill.
|
|
1837
|
+
const pendingEvent = {
|
|
1838
|
+
event: args.event,
|
|
1839
|
+
meshId: args.meshId,
|
|
1840
|
+
nodeLabel: args.nodeLabel,
|
|
1841
|
+
nodeId: args.nodeId || undefined,
|
|
1842
|
+
workspace: readNonEmptyString(args.metadataEvent.workspace)
|
|
1843
|
+
|| readNonEmptyString(args.metadataEvent.workspaceName),
|
|
1844
|
+
metadataEvent: {
|
|
1845
|
+
...args.metadataEvent,
|
|
1846
|
+
...(recoveryContext ? { recoveryContext } : {}),
|
|
1847
|
+
},
|
|
1848
|
+
// Silent lifecycle events (agent:ready / agent:generating_started) carry no
|
|
1849
|
+
// coordinator message; they are queued only so the coordinator re-runs the
|
|
1850
|
+
// remote-claim state machine on pull. injectPendingIntoCoordinator skips
|
|
1851
|
+
// entries without a coordinatorMessage, so a live CLI coordinator is not spammed.
|
|
1852
|
+
...(messageText ? { coordinatorMessage: messageText } : {}),
|
|
1853
|
+
queuedAt: Date.now(),
|
|
1854
|
+
...(workerCoordinatorDaemonId ? { targetCoordinatorDaemonId: workerCoordinatorDaemonId } : {}),
|
|
1855
|
+
};
|
|
1856
|
+
if (queuePendingMeshCoordinatorEvent(pendingEvent)) {
|
|
1857
|
+
LOG.info('MeshEvents', `Queued ${args.event} for coordinator (mesh ${args.meshId}${workerCoordinatorDaemonId ? `, coordinator daemon ${workerCoordinatorDaemonId}` : ''})`);
|
|
1858
|
+
}
|
|
1859
|
+
return { success: true, forwarded: 0 };
|
|
1860
|
+
}
|
|
1861
|
+
|
|
1862
|
+
export function handleMeshForwardEvent(components: DaemonComponents, payload: Record<string, unknown>) {
|
|
1863
|
+
const eventName = readNonEmptyString(payload.event);
|
|
1864
|
+
if (!isMeshCoordinatorEvent(eventName)) {
|
|
1865
|
+
return { success: false, error: 'unsupported mesh event' };
|
|
1866
|
+
}
|
|
1867
|
+
const nodeId = readNonEmptyString(payload.nodeId);
|
|
1868
|
+
const workspace = readNonEmptyString(payload.workspace);
|
|
1869
|
+
|
|
1870
|
+
// The fallback worker-forward path (forwardUnresolvedDelegateEvent) cannot resolve a
|
|
1871
|
+
// mesh id locally on the remote worker, so it forwards the event with workspace only.
|
|
1872
|
+
// The coordinator hosting the mesh CAN resolve it: recover the mesh id by workspace
|
|
1873
|
+
// when the payload doesn't carry one.
|
|
1874
|
+
const meshId = readNonEmptyString(payload.meshId)
|
|
1875
|
+
|| (workspace ? readNonEmptyString(getCachedMeshByWorkspace(workspace)?.id) : '');
|
|
1876
|
+
if (!meshId) return { success: false, error: 'meshId required' };
|
|
1877
|
+
const nodeLabel = nodeId ? `Node '${nodeId}'` : workspace ? `Agent at ${workspace}` : 'Remote agent';
|
|
1878
|
+
const relayModalMessage = readNonEmptyString(payload.modalMessage);
|
|
1879
|
+
const relayModalButtons = Array.isArray(payload.modalButtons)
|
|
1880
|
+
? (payload.modalButtons as unknown[]).filter((b): b is string => typeof b === 'string' && b.trim().length > 0)
|
|
1881
|
+
: null;
|
|
1882
|
+
|
|
1883
|
+
return injectMeshSystemMessage(components, {
|
|
1884
|
+
meshId,
|
|
1885
|
+
nodeId,
|
|
1886
|
+
nodeLabel,
|
|
1887
|
+
event: eventName,
|
|
1888
|
+
metadataEvent: {
|
|
1889
|
+
targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.instanceId),
|
|
1890
|
+
providerType: readNonEmptyString(payload.providerType),
|
|
1891
|
+
providerSessionId: readNonEmptyString(payload.providerSessionId),
|
|
1892
|
+
// Carry the session identity fields the worker provider event emits so the
|
|
1893
|
+
// coordinator's mirror (updateMeshOwnedSession) gets a real workspace/title/
|
|
1894
|
+
// settings. Without these the remote-relay hop reconstructs metadataEvent with
|
|
1895
|
+
// an empty workspace, and the dashboard flaps to the generic
|
|
1896
|
+
// "Terminal (Mesh Node)" title (and degrades the provider label) between live
|
|
1897
|
+
// events and the periodic get_status_metadata snapshot. The local in-process
|
|
1898
|
+
// forward path (onMeshCoordinatorEventForwarded) already preserves these; this
|
|
1899
|
+
// mirrors them for the remote-only relay path.
|
|
1900
|
+
workspace: readNonEmptyString(payload.workspace) || readNonEmptyString(payload.workspaceName),
|
|
1901
|
+
workspaceName: readNonEmptyString(payload.workspaceName) || readNonEmptyString(payload.workspace),
|
|
1902
|
+
sessionTitle: readNonEmptyString(payload.sessionTitle),
|
|
1903
|
+
sessionStatus: readNonEmptyString(payload.sessionStatus),
|
|
1904
|
+
sessionChatStatus: readNonEmptyString(payload.sessionChatStatus),
|
|
1905
|
+
providerName: readNonEmptyString(payload.providerName),
|
|
1906
|
+
...(payload.sessionSettings && typeof payload.sessionSettings === 'object' && !Array.isArray(payload.sessionSettings) ? { sessionSettings: payload.sessionSettings } : {}),
|
|
1907
|
+
finalSummary: readNonEmptyString(payload.finalSummary) || readNonEmptyString(payload.summary),
|
|
1908
|
+
jobId: readNonEmptyString(payload.jobId),
|
|
1909
|
+
interactionId: readNonEmptyString(payload.interactionId),
|
|
1910
|
+
status: readNonEmptyString(payload.status),
|
|
1911
|
+
targetDaemonId: readNonEmptyString(payload.targetDaemonId),
|
|
1912
|
+
startedAt: readNonEmptyString(payload.startedAt),
|
|
1913
|
+
completedAt: readNonEmptyString(payload.completedAt),
|
|
1914
|
+
retryOfJobId: readNonEmptyString(payload.retryOfJobId),
|
|
1915
|
+
...(relayModalMessage ? { modalMessage: relayModalMessage } : {}),
|
|
1916
|
+
...(relayModalButtons && relayModalButtons.length > 0 ? { modalButtons: relayModalButtons } : {}),
|
|
1917
|
+
...(payload.result && typeof payload.result === 'object' && !Array.isArray(payload.result) ? { result: payload.result } : {}),
|
|
1918
|
+
...(payload.completionDiagnostic && typeof payload.completionDiagnostic === 'object' && !Array.isArray(payload.completionDiagnostic) ? { completionDiagnostic: payload.completionDiagnostic } : {}),
|
|
1919
|
+
...(payload.workerResult && typeof payload.workerResult === 'object' && !Array.isArray(payload.workerResult) ? { workerResult: payload.workerResult } : {}),
|
|
1920
|
+
...(payload.meshWorkerResult && typeof payload.meshWorkerResult === 'object' && !Array.isArray(payload.meshWorkerResult) ? { meshWorkerResult: payload.meshWorkerResult } : {}),
|
|
1921
|
+
...(payload.structuredResult && typeof payload.structuredResult === 'object' && !Array.isArray(payload.structuredResult) ? { structuredResult: payload.structuredResult } : {}),
|
|
1922
|
+
...(payload.timestamp !== undefined ? { timestamp: payload.timestamp } : {}),
|
|
1923
|
+
intentional: payload.intentional === true,
|
|
1924
|
+
intentionalStop: payload.intentionalStop === true,
|
|
1925
|
+
operatorCleanup: payload.operatorCleanup === true,
|
|
1926
|
+
reason: readNonEmptyString(payload.reason),
|
|
1927
|
+
stopReason: readNonEmptyString(payload.stopReason),
|
|
1928
|
+
cleanupReason: readNonEmptyString(payload.cleanupReason),
|
|
1929
|
+
source: readNonEmptyString(payload.source),
|
|
1930
|
+
},
|
|
1931
|
+
});
|
|
1932
|
+
}
|
|
1933
|
+
|
|
1934
|
+
// ---------------------------------------------------------------------------
|
|
1935
|
+
// Worker-side fallback forward for unresolved-mesh delegates.
|
|
1936
|
+
//
|
|
1937
|
+
// A REMOTE worker daemon that is being P2P-remote-controlled by a coordinator is
|
|
1938
|
+
// NOT a member of the coordinator's mesh — it has no local mesh record. So when its
|
|
1939
|
+
// completion event reaches the forwarder, resolveWorkerDelegateRouting() resolves the
|
|
1940
|
+
// coordinator anchor (meshCoordinatorDaemonId) from the worker envelope but cannot
|
|
1941
|
+
// resolve the mesh id (neither meshNodeFor nor a workspace→mesh lookup yields one) and
|
|
1942
|
+
// returns isDelegate=false / mesh_unresolved. Before this fallback the event was dropped
|
|
1943
|
+
// (delivery_unroutable) and only recovered later when the coordinator happened to pull
|
|
1944
|
+
// the worker's queue — which it can't, because the worker never queued an unroutable
|
|
1945
|
+
// event. Live symptom: `WARN [MeshEvents] delivery_unroutable: ... mesh unresolved`.
|
|
1946
|
+
//
|
|
1947
|
+
// The fix: the routing object still carries coordinatorDaemonId. Forward the raw event
|
|
1948
|
+
// straight to that coordinator daemon over P2P (mesh_forward_event). The coordinator
|
|
1949
|
+
// hosts the mesh, so it recovers the mesh id by workspace in handleMeshForwardEvent and
|
|
1950
|
+
// injects/queues it normally. meshId is intentionally omitted from the payload (the
|
|
1951
|
+
// worker has none); workspace is the routing anchor the coordinator resolves from.
|
|
1952
|
+
//
|
|
1953
|
+
// No loop / no double-delivery:
|
|
1954
|
+
// - This only fires on the WORKER (the coordinator-own session is rejected by the
|
|
1955
|
+
// resolver before reaching here), and the coordinator merely injects — it does not
|
|
1956
|
+
// re-enter this forwarder for the relayed event.
|
|
1957
|
+
// - It fires only when the normal queue path did NOT run (isDelegate=false), so the
|
|
1958
|
+
// event is never both queued locally and forwarded.
|
|
1959
|
+
//
|
|
1960
|
+
// Returns true when the event was durably accepted for delivery to the coordinator
|
|
1961
|
+
// daemon (so the caller skips the delivery_unroutable diagnostic); false when no
|
|
1962
|
+
// fallback was possible (no coordinator anchor / no dispatch transport).
|
|
1963
|
+
//
|
|
1964
|
+
// Durability: the directed push to the coordinator is the ONLY delivery route for an
|
|
1965
|
+
// unresolved-mesh worker (it is in no mesh.node the coordinator can pull). So instead
|
|
1966
|
+
// of a fire-and-forget push that drops on one transient P2P failure, the event is
|
|
1967
|
+
// persisted to the worker-side outbox FIRST and only acked after a successful push.
|
|
1968
|
+
// A best-effort immediate push keeps latency low on the happy path; a failed or
|
|
1969
|
+
// un-acked push leaves the durable row for setupMeshReconcileLoop's PHASE 0 to retry.
|
|
1970
|
+
function forwardUnresolvedDelegateEvent(
|
|
1971
|
+
components: DaemonComponents,
|
|
1972
|
+
routing: ReturnType<typeof resolveWorkerDelegateRouting>,
|
|
1973
|
+
event: Record<string, unknown>,
|
|
1974
|
+
): boolean {
|
|
1975
|
+
const coordinatorDaemonId = readNonEmptyString(routing.coordinatorDaemonId);
|
|
1976
|
+
if (!coordinatorDaemonId) return false;
|
|
1977
|
+
if (!components.dispatchMeshCommand) return false;
|
|
1978
|
+
|
|
1979
|
+
const eventName = readNonEmptyString(event.event);
|
|
1980
|
+
if (!eventName) return false;
|
|
1981
|
+
|
|
1982
|
+
// Flat payload mirroring buildForwardPayloadFromPending / what handleMeshForwardEvent
|
|
1983
|
+
// reads. meshId is omitted on purpose — the worker can't resolve it; the coordinator
|
|
1984
|
+
// recovers it from workspace. nodeId/workspace come from the worker envelope so the
|
|
1985
|
+
// coordinator can name and locate the node.
|
|
1986
|
+
const payload: Record<string, unknown> = {
|
|
1987
|
+
...event,
|
|
1988
|
+
event: eventName,
|
|
1989
|
+
nodeId: readNonEmptyString(routing.nodeId) || readNonEmptyString(event.meshNodeId) || undefined,
|
|
1990
|
+
workspace: readNonEmptyString(routing.workspace) || readNonEmptyString(event.workspace) || undefined,
|
|
1991
|
+
};
|
|
1992
|
+
|
|
1993
|
+
// 1) Persist durably FIRST. Idempotent on fingerprint, so a re-fired completion
|
|
1994
|
+
// does not duplicate the outbox row. If persistence fails we still attempt the
|
|
1995
|
+
// push below (degrades to the old at-most-once behaviour rather than dropping
|
|
1996
|
+
// the chance entirely).
|
|
1997
|
+
const persisted = enqueueUnresolvedDelegateForward(coordinatorDaemonId, eventName, payload);
|
|
1998
|
+
|
|
1999
|
+
// 2) Best-effort immediate push for low latency. On success, ack the outbox row so
|
|
2000
|
+
// the retry loop won't re-send it. On failure, leave it queued — PHASE 0 retries.
|
|
2001
|
+
Promise.resolve(components.dispatchMeshCommand(coordinatorDaemonId, 'mesh_forward_event', payload))
|
|
2002
|
+
.then((result: any) => {
|
|
2003
|
+
if (result && result.success === false) {
|
|
2004
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} rejected (${readNonEmptyString(result.error) || 'no reason'}) — left queued for retry`);
|
|
2005
|
+
return;
|
|
2006
|
+
}
|
|
2007
|
+
// Acked. Mark the durable copy delivered so the retry loop skips it.
|
|
2008
|
+
if (persisted) ackUnresolvedDelegateForwardByFingerprint(coordinatorDaemonId, eventName, payload);
|
|
2009
|
+
})
|
|
2010
|
+
.catch((e: any) => {
|
|
2011
|
+
// Coordinator momentarily unreachable; the durable row stays queued and the
|
|
2012
|
+
// reconcile loop retries it. Trace so the relay attempt is visible.
|
|
2013
|
+
LOG.warn('MeshEvents', `Immediate forward of ${eventName} to coordinator ${coordinatorDaemonId} failed: ${e?.message || e} — left queued for retry`);
|
|
2014
|
+
});
|
|
2015
|
+
LOG.info('MeshEvents', `Durably forwarded ${eventName} for unresolved-mesh worker at ${routing.workspace || '(no workspace)'} to coordinator daemon ${coordinatorDaemonId}`);
|
|
2016
|
+
return true;
|
|
2017
|
+
}
|
|
2018
|
+
|
|
2019
|
+
// Ack a just-pushed outbox entry by re-deriving its row from the same coordinator +
|
|
2020
|
+
// event + payload. We don't thread the row id back from enqueue (the immediate push is
|
|
2021
|
+
// fire-then-ack), so locate it among the undrained entries by matching coordinator and
|
|
2022
|
+
// the flat payload's forward identity. A miss is harmless — the retry loop's own
|
|
2023
|
+
// receiver-side dedup suppresses a duplicate delivery.
|
|
2024
|
+
function ackUnresolvedDelegateForwardByFingerprint(
|
|
2025
|
+
coordinatorDaemonId: string,
|
|
2026
|
+
eventName: string,
|
|
2027
|
+
payload: Record<string, unknown>,
|
|
2028
|
+
): void {
|
|
2029
|
+
const match = peekUnresolvedDelegateForwards().find(entry =>
|
|
2030
|
+
entry.coordinatorDaemonId === coordinatorDaemonId
|
|
2031
|
+
&& readNonEmptyString(entry.payload.event) === eventName
|
|
2032
|
+
&& readNonEmptyString(entry.payload.targetSessionId || entry.payload.sessionId || entry.payload.instanceId)
|
|
2033
|
+
=== readNonEmptyString(payload.targetSessionId || payload.sessionId || payload.instanceId)
|
|
2034
|
+
&& readNonEmptyString(entry.payload.workspace) === readNonEmptyString(payload.workspace),
|
|
2035
|
+
);
|
|
2036
|
+
if (match) ackUnresolvedDelegateForward(match.id);
|
|
2037
|
+
}
|
|
2038
|
+
|
|
2039
|
+
export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
2040
|
+
components.instanceManager.onEvent((event) => {
|
|
2041
|
+
// --- Coordinator idle auto-flush (fast path) ---
|
|
2042
|
+
// When a coordinator session becomes idle, immediately flush any pending
|
|
2043
|
+
// coordinator events that accumulated while it was generating, rather than
|
|
2044
|
+
// waiting up to one reconcile interval for setupMeshReconcileLoop to do it.
|
|
2045
|
+
// Both paths drain the SAME queue via drainPendingMeshCoordinatorEvents,
|
|
2046
|
+
// whose SQLite drained=1 marking is atomic — whichever fires first consumes
|
|
2047
|
+
// the events and the other gets nothing, so there is no double-delivery.
|
|
2048
|
+
// This runs before the delegate routing below so that coordinator-own idle
|
|
2049
|
+
// transitions are handled first.
|
|
2050
|
+
// Exception: a coordinator that is itself a direct-dispatch target still needs
|
|
2051
|
+
// to go through delegate routing so that the dispatching coordinator receives a
|
|
2052
|
+
// pendingCoordinatorEvents entry for the completion.
|
|
2053
|
+
if (event.event === 'agent:ready' || event.event === 'agent:generating_completed') {
|
|
2054
|
+
const flushInstanceId = readNonEmptyString(event.instanceId);
|
|
2055
|
+
if (flushInstanceId) {
|
|
2056
|
+
const flushSource = components.instanceManager.getInstance(flushInstanceId);
|
|
2057
|
+
if (flushSource && flushSource.category === 'cli') {
|
|
2058
|
+
const flushState = flushSource.getState();
|
|
2059
|
+
const flushSettings = flushState.settings && typeof flushState.settings === 'object' ? flushState.settings as Record<string, unknown> : {};
|
|
2060
|
+
const coordinatorMeshId = readNonEmptyString(flushSettings.meshCoordinatorFor);
|
|
2061
|
+
if (coordinatorMeshId) {
|
|
2062
|
+
const status = readNonEmptyString(flushState.status).toLowerCase();
|
|
2063
|
+
if (status === 'idle') {
|
|
2064
|
+
try {
|
|
2065
|
+
// Drain with the daemon's full coordinator-id set (status id + machineId).
|
|
2066
|
+
// The MCP layer stamps the prefixed status id (`standalone_<machineId>` /
|
|
2067
|
+
// `daemon_<machineId>`) as the worker's meshCoordinatorDaemonId; draining
|
|
2068
|
+
// with bare machineId alone would miss those unicast events. Mirrors
|
|
2069
|
+
// resolveCoordinatorDaemonIds in mesh-reconcile-loop.
|
|
2070
|
+
const drainDaemonIds = resolveCoordinatorDrainDaemonIds(components);
|
|
2071
|
+
const pendingEvents = drainPendingMeshCoordinatorEvents(coordinatorMeshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
|
|
2072
|
+
if (pendingEvents.length > 0) {
|
|
2073
|
+
LOG.info('MeshEvents', `Auto-flushing ${pendingEvents.length} pending coordinator event(s) for mesh ${coordinatorMeshId} on coordinator idle`);
|
|
2074
|
+
for (const pending of pendingEvents) {
|
|
2075
|
+
if (!pending.coordinatorMessage) continue;
|
|
2076
|
+
const forcePending = shouldForceInjectMeshEvent(pending.event);
|
|
2077
|
+
flushSource.onEvent('send_message', {
|
|
2078
|
+
input: { text: pending.coordinatorMessage, textFallback: pending.coordinatorMessage },
|
|
2079
|
+
...(forcePending ? { force: true } : {}),
|
|
2080
|
+
});
|
|
2081
|
+
}
|
|
2082
|
+
}
|
|
2083
|
+
} catch (e: any) {
|
|
2084
|
+
LOG.warn('MeshEvents', `Failed to auto-flush pending coordinator events: ${e?.message || e}`);
|
|
2085
|
+
}
|
|
2086
|
+
}
|
|
2087
|
+
// Skip delegate routing unless this coordinator session is itself
|
|
2088
|
+
// a direct-dispatch target — in that case fall through so the
|
|
2089
|
+
// dispatching coordinator gets a pendingCoordinatorEvents entry.
|
|
2090
|
+
let hasDirectDispatch = false;
|
|
2091
|
+
try {
|
|
2092
|
+
hasDirectDispatch =
|
|
2093
|
+
getActiveDirectDispatches(coordinatorMeshId).some(d => d.sessionId === flushInstanceId)
|
|
2094
|
+
|| hasUnterminalDirectDispatchLedgerEntry(coordinatorMeshId, flushInstanceId);
|
|
2095
|
+
} catch { /* best-effort */ }
|
|
2096
|
+
if (!hasDirectDispatch) return;
|
|
2097
|
+
}
|
|
2098
|
+
}
|
|
2099
|
+
}
|
|
2100
|
+
}
|
|
2101
|
+
|
|
2102
|
+
// --- Delegate event routing ---
|
|
2103
|
+
if (!isMeshCoordinatorEvent(event.event)) return;
|
|
2104
|
+
|
|
2105
|
+
const instanceId = readNonEmptyString(event.instanceId);
|
|
2106
|
+
if (!instanceId) return;
|
|
2107
|
+
|
|
2108
|
+
// R1: all session→node→mesh→coordinator interpretation is folded into the single
|
|
2109
|
+
// resolveWorkerDelegateRouting() resolver. No stamp (meshNodeFor / meshNodeId /
|
|
2110
|
+
// meshCoordinatorDaemonId / meshCoordinatorNodeId / launchedByCoordinator) is read
|
|
2111
|
+
// here to make a routing decision — the resolver is the one authority, and the
|
|
2112
|
+
// forwarder consumes its typed result only.
|
|
2113
|
+
const routing = resolveWorkerDelegateRouting(components, instanceId, {
|
|
2114
|
+
getMeshById: (meshId) => getMeshWithCache(components, meshId),
|
|
2115
|
+
getMeshByWorkspace: (workspace) => getCachedMeshByWorkspace(workspace),
|
|
2116
|
+
});
|
|
2117
|
+
if (!routing.isDelegate) {
|
|
2118
|
+
// Fallback: a REMOTE worker that isn't a member of the coordinator's mesh can't
|
|
2119
|
+
// resolve a mesh id locally (mesh_unresolved), but it still carries the coordinator
|
|
2120
|
+
// daemon anchor. Forward the event straight to that coordinator over P2P instead of
|
|
2121
|
+
// dropping it — the coordinator hosts the mesh and recovers the id by workspace.
|
|
2122
|
+
if (isUnroutableDelegateRejection(routing)
|
|
2123
|
+
&& forwardUnresolvedDelegateEvent(components, routing, event)) {
|
|
2124
|
+
return;
|
|
2125
|
+
}
|
|
2126
|
+
// R4: a worker that presented a valid envelope but resolved to no mesh (and could
|
|
2127
|
+
// not be fallback-forwarded — e.g. no coordinator anchor) used to be dropped
|
|
2128
|
+
// silently. Leave a fail-loud diagnostic so the missing completion is traceable.
|
|
2129
|
+
// Benign non-delegate rejections (not_cli / no_workspace / etc.) are no-ops inside
|
|
2130
|
+
// recordUnroutableDelegateEvent.
|
|
2131
|
+
recordUnroutableDelegateEvent(routing, event.event);
|
|
2132
|
+
return;
|
|
2133
|
+
}
|
|
2134
|
+
|
|
2135
|
+
injectMeshSystemMessage(components, {
|
|
2136
|
+
meshId: routing.meshId,
|
|
2137
|
+
sourceInstanceId: instanceId,
|
|
2138
|
+
nodeId: routing.nodeId,
|
|
2139
|
+
nodeLabel: routing.nodeLabel,
|
|
2140
|
+
event: event.event,
|
|
2141
|
+
metadataEvent: event,
|
|
2142
|
+
});
|
|
2143
|
+
});
|
|
2144
|
+
}
|