@adhdev/daemon-core 0.9.82-rc.21 → 0.9.82-rc.211
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/daemon-lifecycle.d.ts +1 -0
- package/dist/boot/process-hardening.d.ts +50 -0
- package/dist/chat/source-machine.d.ts +166 -0
- package/dist/chat/source-resolver.d.ts +104 -0
- package/dist/chat/subscription-updates.d.ts +1 -0
- package/dist/cli-adapter-types.d.ts +23 -2
- package/dist/cli-adapters/cli-script-runner.d.ts +117 -0
- package/dist/cli-adapters/cli-state-engine.d.ts +178 -0
- package/dist/cli-adapters/provider-cli-adapter.d.ts +106 -63
- package/dist/cli-adapters/provider-cli-parse.d.ts +4 -0
- package/dist/cli-adapters/provider-cli-shared.d.ts +29 -0
- package/dist/cli-adapters/raw-terminal-io.d.ts +37 -0
- package/dist/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +3 -3
- package/dist/cli-adapters/terminal-backends/types.d.ts +1 -2
- package/dist/cli-adapters/terminal-screen.d.ts +2 -11
- package/dist/commands/cli-manager.d.ts +2 -1
- package/dist/commands/handler.d.ts +110 -0
- package/dist/commands/mesh-coordinator.d.ts +85 -1
- package/dist/commands/router.d.ts +32 -1
- package/dist/config/chat-history.d.ts +9 -0
- package/dist/config/config.d.ts +5 -0
- package/dist/config/mesh-config.d.ts +72 -1
- package/dist/git/git-commands.d.ts +9 -1
- package/dist/git/git-diff.d.ts +6 -0
- package/dist/git/git-types.d.ts +2 -0
- package/dist/index.d.ts +46 -11
- package/dist/index.js +35173 -15756
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +34880 -15556
- package/dist/index.mjs.map +1 -1
- package/dist/installer.d.ts +1 -4
- package/dist/ipc/local-ipc-server.d.ts +91 -0
- package/dist/launch.d.ts +1 -1
- package/dist/logging/async-batch-writer.d.ts +10 -0
- package/dist/mesh/contracts.d.ts +164 -0
- package/dist/mesh/coordinator-prompt.d.ts +36 -0
- package/dist/mesh/coordinator-registry.d.ts +59 -0
- package/dist/mesh/mesh-active-work.d.ts +90 -0
- package/dist/mesh/mesh-delivery-policy.d.ts +131 -0
- package/dist/mesh/mesh-events-coordinator.d.ts +151 -0
- package/dist/mesh/mesh-events-pending.d.ts +33 -0
- package/dist/mesh/mesh-events-stale.d.ts +40 -0
- package/dist/mesh/mesh-events-utils.d.ts +14 -0
- package/dist/mesh/mesh-events.d.ts +5 -40
- package/dist/mesh/mesh-fast-forward.d.ts +41 -0
- package/dist/mesh/mesh-host-ownership.d.ts +9 -0
- package/dist/mesh/mesh-ledger-reconciliation.d.ts +23 -3
- package/dist/mesh/mesh-ledger.d.ts +77 -1
- package/dist/mesh/mesh-missions.d.ts +58 -0
- package/dist/mesh/mesh-refine-status.d.ts +26 -0
- package/dist/mesh/mesh-review-inbox.d.ts +90 -0
- package/dist/mesh/mesh-runtime-store.d.ts +324 -0
- package/dist/mesh/mesh-task-stats.d.ts +49 -0
- package/dist/mesh/mesh-work-queue.d.ts +137 -6
- package/dist/mesh/preview-freshness.d.ts +18 -0
- package/dist/mesh/refine-config.d.ts +215 -0
- package/dist/mesh/worktree-bootstrap-config.d.ts +135 -0
- package/dist/providers/acp-provider-instance.d.ts +2 -0
- package/dist/providers/approval-utils.d.ts +13 -0
- package/dist/providers/cli-provider-instance.d.ts +39 -3
- package/dist/providers/contracts.d.ts +130 -6
- package/dist/providers/external-sources.d.ts +71 -0
- package/dist/providers/native-history/antigravity-cli-transcript.d.ts +100 -0
- package/dist/providers/native-history/claude-cli-transcript.d.ts +70 -0
- package/dist/providers/native-history/codex-cli-transcript.d.ts +73 -0
- package/dist/providers/native-history/dispatcher.d.ts +27 -0
- package/dist/providers/native-history/hermes-cli-transcript.d.ts +30 -0
- package/dist/providers/native-history/index.d.ts +13 -0
- package/dist/providers/provider-instance-manager.d.ts +12 -0
- package/dist/providers/provider-instance.d.ts +13 -1
- package/dist/providers/provider-loader.d.ts +26 -4
- package/dist/providers/provider-trust.d.ts +31 -0
- package/dist/providers/read-chat-contract.d.ts +29 -0
- package/dist/providers/sdk/v1/builders/acp/detect-status.d.ts +68 -0
- package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +85 -0
- package/dist/providers/sdk/v1/builders/cli/parse-approval-squash.d.ts +59 -0
- package/dist/providers/sdk/v1/builders/cli/parse-approval.d.ts +65 -0
- package/dist/providers/sdk/v1/builders/cli/parse-session.d.ts +91 -0
- package/dist/providers/sdk/v1/builders/cli/visible-region.d.ts +42 -0
- package/dist/providers/sdk/v1/fixture-tooling/format.d.ts +126 -0
- package/dist/providers/sdk/v1/fixture-tooling/index.d.ts +8 -0
- package/dist/providers/sdk/v1/fixture-tooling/replay.d.ts +38 -0
- package/dist/providers/sdk/v1/index.d.ts +30 -0
- package/dist/providers/sdk/v1/sandbox/README-design.d.ts +193 -0
- package/dist/providers/sdk/v1/sandbox/require-whitelist.d.ts +74 -0
- package/dist/providers/sdk/v1/sandbox/script-runner.d.ts +98 -0
- package/dist/providers/sdk/v1/types/cli/index.d.ts +277 -0
- package/dist/providers/sdk/v1/types/common/index.d.ts +203 -0
- package/dist/providers/sdk/v1/validators/index.d.ts +5 -0
- package/dist/providers/sdk/v1/validators/manifest.d.ts +40 -0
- package/dist/providers/sdk/v1/validators/taint.d.ts +52 -0
- package/dist/providers/spec/adapter.d.ts +52 -0
- package/dist/providers/spec/cli-adapter.d.ts +101 -0
- package/dist/providers/spec/driver.d.ts +225 -0
- package/dist/providers/spec/evaluator.d.ts +54 -0
- package/dist/providers/spec/loader.d.ts +15 -0
- package/dist/providers/spec/native-history-executor.d.ts +41 -0
- package/dist/providers/spec/route.d.ts +4 -0
- package/dist/providers/spec/schema.gen.d.ts +2025 -0
- package/dist/providers/spec/types.d.ts +302 -0
- package/dist/providers/transcript-v2.d.ts +176 -0
- package/dist/providers/types/interactive-prompt.d.ts +48 -0
- package/dist/repo-mesh-types.d.ts +109 -1
- package/dist/sessions/registry.d.ts +3 -0
- package/dist/shared-types-extra.d.ts +2 -4
- package/dist/shared-types.d.ts +42 -1
- package/dist/status/normalize.d.ts +1 -1
- package/dist/status/normalize.js +1 -0
- package/dist/status/normalize.js.map +1 -1
- package/dist/status/normalize.mjs +1 -0
- package/dist/status/normalize.mjs.map +1 -1
- package/dist/status/reporter.d.ts +2 -0
- package/dist/status/snapshot.d.ts +1 -0
- package/dist/types.d.ts +5 -0
- package/package.json +6 -3
- package/src/agent-stream/poller.ts +2 -3
- package/src/boot/daemon-lifecycle.ts +22 -10
- package/src/boot/process-hardening.ts +89 -0
- package/src/chat/source-machine.ts +534 -0
- package/src/chat/source-resolver.ts +0 -0
- package/src/chat/subscription-updates.ts +20 -1
- package/src/cli-adapter-types.d.ts +1 -0
- package/src/cli-adapter-types.ts +22 -2
- package/src/cli-adapters/cli-script-runner.ts +421 -0
- package/src/cli-adapters/cli-state-engine.ts +1138 -0
- package/src/cli-adapters/provider-cli-adapter.d.ts +1 -1
- package/src/cli-adapters/provider-cli-adapter.ts +673 -1139
- package/src/cli-adapters/provider-cli-parse.d.ts +1 -0
- package/src/cli-adapters/provider-cli-parse.ts +13 -0
- package/src/cli-adapters/provider-cli-runtime.ts +3 -1
- package/src/cli-adapters/provider-cli-shared.d.ts +2 -0
- package/src/cli-adapters/provider-cli-shared.ts +59 -11
- package/src/cli-adapters/raw-terminal-io.ts +252 -0
- package/src/cli-adapters/terminal-backends/ghostty-vt-backend.ts +35 -29
- package/src/cli-adapters/terminal-backends/types.ts +1 -3
- package/src/cli-adapters/terminal-screen.ts +11 -81
- package/src/commands/chat-commands.ts +1787 -60
- package/src/commands/cli-manager.ts +283 -14
- package/src/commands/handler.ts +809 -2
- package/src/commands/mesh-coordinator.ts +332 -122
- package/src/commands/router.ts +4198 -557
- package/src/config/chat-history.ts +95 -24
- package/src/config/config.ts +12 -0
- package/src/config/mesh-config.ts +280 -2
- package/src/config/recent-activity.ts +8 -2
- package/src/daemon/dev-auto-implement.ts +3 -2
- package/src/daemon/dev-cli-debug.ts +10 -1
- package/src/detection/ide-detector.ts +26 -16
- package/src/git/git-commands.ts +42 -12
- package/src/git/git-diff.ts +53 -0
- package/src/git/git-status.ts +35 -6
- package/src/git/git-types.ts +2 -0
- package/src/git/git-worktree.ts +8 -1
- package/src/index.ts +126 -10
- package/src/installer.d.ts +1 -1
- package/src/installer.ts +8 -6
- package/src/ipc/local-ipc-server.ts +278 -0
- package/src/launch.d.ts +1 -1
- package/src/launch.ts +37 -28
- package/src/logging/async-batch-writer.ts +55 -0
- package/src/logging/logger.ts +2 -1
- package/src/mesh/contracts.ts +329 -0
- package/src/mesh/coordinator-prompt.ts +218 -31
- package/src/mesh/coordinator-registry.ts +121 -0
- package/src/mesh/mesh-active-work.ts +437 -0
- package/src/mesh/mesh-delivery-policy.ts +315 -0
- package/src/mesh/mesh-events-coordinator.ts +1404 -0
- package/src/mesh/mesh-events-pending.ts +371 -0
- package/src/mesh/mesh-events-stale.ts +283 -0
- package/src/mesh/mesh-events-utils.ts +161 -0
- package/src/mesh/mesh-events.ts +24 -908
- package/src/mesh/mesh-fast-forward.ts +438 -0
- package/src/mesh/mesh-host-ownership.ts +73 -0
- package/src/mesh/mesh-ledger-reconciliation.ts +12 -5
- package/src/mesh/mesh-ledger.ts +587 -102
- package/src/mesh/mesh-missions.ts +151 -0
- package/src/mesh/mesh-refine-status.ts +144 -0
- package/src/mesh/mesh-review-inbox.ts +307 -0
- package/src/mesh/mesh-runtime-store.ts +1305 -0
- package/src/mesh/mesh-task-stats.ts +154 -0
- package/src/mesh/mesh-work-queue.ts +490 -166
- package/src/mesh/preview-freshness.ts +118 -0
- package/src/mesh/refine-config.ts +403 -0
- package/src/mesh/worktree-bootstrap-config.ts +326 -0
- package/src/providers/acp-provider-instance.ts +15 -1
- package/src/providers/approval-utils.d.ts +4 -0
- package/src/providers/approval-utils.ts +47 -5
- package/src/providers/chat-message-normalization.ts +4 -11
- package/src/providers/cli-provider-instance.ts +833 -64
- package/src/providers/contracts.d.ts +55 -0
- package/src/providers/contracts.ts +141 -6
- package/src/providers/external-sources.ts +218 -0
- package/src/providers/ide-provider-instance.ts +19 -5
- package/src/providers/native-history/antigravity-cli-transcript.ts +643 -0
- package/src/providers/native-history/claude-cli-transcript.ts +396 -0
- package/src/providers/native-history/codex-cli-transcript.ts +419 -0
- package/src/providers/native-history/dispatcher.ts +340 -0
- package/src/providers/native-history/hermes-cli-transcript.ts +230 -0
- package/src/providers/native-history/index.ts +30 -0
- package/src/providers/provider-instance-manager.ts +30 -0
- package/src/providers/provider-instance.ts +10 -1
- package/src/providers/provider-loader.ts +582 -50
- package/src/providers/provider-schema.ts +87 -14
- package/src/providers/provider-trust.ts +114 -0
- package/src/providers/read-chat-contract.ts +76 -16
- package/src/providers/sdk/README.md +49 -0
- package/src/providers/sdk/v1/builders/acp/detect-status.ts +144 -0
- package/src/providers/sdk/v1/builders/cli/detect-status.ts +262 -0
- package/src/providers/sdk/v1/builders/cli/parse-approval-squash.ts +158 -0
- package/src/providers/sdk/v1/builders/cli/parse-approval.ts +250 -0
- package/src/providers/sdk/v1/builders/cli/parse-session.ts +276 -0
- package/src/providers/sdk/v1/builders/cli/visible-region.ts +143 -0
- package/src/providers/sdk/v1/fixture-tooling/format.ts +130 -0
- package/src/providers/sdk/v1/fixture-tooling/index.ts +22 -0
- package/src/providers/sdk/v1/fixture-tooling/replay.ts +352 -0
- package/src/providers/sdk/v1/index.ts +152 -0
- package/src/providers/sdk/v1/sandbox/README-design.ts +195 -0
- package/src/providers/sdk/v1/sandbox/require-whitelist.ts +472 -0
- package/src/providers/sdk/v1/sandbox/script-runner.ts +150 -0
- package/src/providers/sdk/v1/schemas/cli/provider.schema.json +504 -0
- package/src/providers/sdk/v1/schemas/primitives/acp-session-protocol-v1.json +131 -0
- package/src/providers/sdk/v1/schemas/primitives/native-history-codex-rollout-v1.json +66 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-approval-squash-v1.json +91 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-assistant-block-v1.json +91 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-cue-ordering-v1.json +47 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-dispatch-order-v1.json +32 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-footer-chrome-v1.json +42 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-index-finder-v1.json +27 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-modal-v1.json +125 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-prompt-marker-v1.json +45 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-session-id-extraction-v1.json +46 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-settled-prompt-v1.json +71 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-spinner-v1.json +83 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-transcript-pty-v1.json +83 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-visible-region-v1.json +57 -0
- package/src/providers/sdk/v1/schemas/primitives/tui-welcome-screen-v1.json +35 -0
- package/src/providers/sdk/v1/types/cli/index.ts +365 -0
- package/src/providers/sdk/v1/types/common/index.ts +229 -0
- package/src/providers/sdk/v1/validators/index.ts +19 -0
- package/src/providers/sdk/v1/validators/manifest.ts +110 -0
- package/src/providers/sdk/v1/validators/taint.ts +309 -0
- package/src/providers/spec/adapter.ts +138 -0
- package/src/providers/spec/cli-adapter.ts +626 -0
- package/src/providers/spec/driver.ts +922 -0
- package/src/providers/spec/evaluator.ts +597 -0
- package/src/providers/spec/loader.ts +324 -0
- package/src/providers/spec/native-history-executor.ts +939 -0
- package/src/providers/spec/route.ts +51 -0
- package/src/providers/spec/schema.gen.ts +691 -0
- package/src/providers/spec/schema.json +341 -0
- package/src/providers/spec/types.ts +356 -0
- package/src/providers/transcript-v2.ts +567 -0
- package/src/providers/types/interactive-prompt.ts +425 -0
- package/src/providers/version-archive.ts +38 -20
- package/src/repo-mesh-types.ts +119 -1
- package/src/sessions/registry.ts +3 -0
- package/src/shared-types-extra.ts +2 -4
- package/src/shared-types.ts +45 -1
- package/src/status/builders.ts +24 -6
- package/src/status/normalize.ts +2 -0
- package/src/status/reporter.ts +15 -0
- package/src/status/snapshot.ts +84 -25
- package/src/system/host-memory.ts +29 -12
- package/src/types.ts +5 -0
- package/dist/cli-adapters/terminal-backends/xterm-backend.d.ts +0 -17
- package/dist/mesh/mesh-sync.d.ts +0 -53
- package/src/cli-adapters/terminal-backends/ghostty-vt-backend.d.ts +0 -16
- package/src/cli-adapters/terminal-backends/xterm-backend.ts +0 -97
- package/src/mesh/mesh-sync.ts +0 -111
|
@@ -0,0 +1,1404 @@
|
|
|
1
|
+
import { existsSync } from 'fs';
|
|
2
|
+
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
3
|
+
import { loadConfig } from '../config/config.js';
|
|
4
|
+
import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
|
|
5
|
+
import { detectCLI } from '../detection/cli-detector.js';
|
|
6
|
+
import { LOG } from '../logging/logger.js';
|
|
7
|
+
import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
|
|
8
|
+
import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
|
|
9
|
+
import { buildMeshNodeCapabilityTags, claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents } from './mesh-work-queue.js';
|
|
10
|
+
import { fastForwardMeshNode } from './mesh-fast-forward.js';
|
|
11
|
+
import { createSessionDelivery, markSessionDeliveriesTerminal, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
|
|
12
|
+
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
13
|
+
import { queuePendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
14
|
+
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
15
|
+
import {
|
|
16
|
+
findRecentTerminalLedgerEvidence,
|
|
17
|
+
hasDispatchAfterTerminal,
|
|
18
|
+
hasUnterminalDirectDispatchLedgerEntry,
|
|
19
|
+
buildLongGeneratingCompletionReconciliation,
|
|
20
|
+
} from './mesh-events-stale.js';
|
|
21
|
+
import {
|
|
22
|
+
buildMeshSystemMessage,
|
|
23
|
+
readNonEmptyString,
|
|
24
|
+
readRecord,
|
|
25
|
+
resolveEventSessionId,
|
|
26
|
+
readRefineJobId,
|
|
27
|
+
readWorkerResultMetadata,
|
|
28
|
+
} from './mesh-events-utils.js';
|
|
29
|
+
|
|
30
|
+
// ---------------------------------------------------------------------------
|
|
31
|
+
// Remote Node Idle Session Tracking
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
const REMOTE_IDLE_SESSION_TTL_MS = 5 * 60 * 1000; // 5 minutes
|
|
34
|
+
|
|
35
|
+
// ---------------------------------------------------------------------------
|
|
36
|
+
// Workspace-to-mesh lookup cache
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
const meshByWorkspaceCache = new Map<string, { mesh: any; cachedAt: number }>();
|
|
39
|
+
const MESH_WORKSPACE_CACHE_TTL_MS = 5_000;
|
|
40
|
+
const IDLE_AUTO_FAST_FORWARD_THROTTLE_MS = 30 * 60 * 1000;
|
|
41
|
+
const idleAutoFastForwardLastAttempt = new Map<string, number>();
|
|
42
|
+
|
|
43
|
+
function getCachedMeshByWorkspace(workspace: string): any {
|
|
44
|
+
const now = Date.now();
|
|
45
|
+
const cached = meshByWorkspaceCache.get(workspace);
|
|
46
|
+
if (cached && now - cached.cachedAt < MESH_WORKSPACE_CACHE_TTL_MS) return cached.mesh;
|
|
47
|
+
const mesh = getMeshByRepo(workspace);
|
|
48
|
+
meshByWorkspaceCache.set(workspace, { mesh, cachedAt: now });
|
|
49
|
+
return mesh;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export function __resetIdleAutoFastForwardForTests(): void {
|
|
53
|
+
idleAutoFastForwardLastAttempt.clear();
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function sweepExpiredRemoteIdleSessions(): void {
|
|
57
|
+
try {
|
|
58
|
+
MeshRuntimeStore.getInstance().pruneExpiredRemoteIdleSessions();
|
|
59
|
+
} catch { /* best-effort */ }
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function getMeshWithCache(components: DaemonComponents, meshId: string): any | undefined {
|
|
63
|
+
const localMesh = getMesh(meshId);
|
|
64
|
+
if (localMesh) return localMesh;
|
|
65
|
+
return components.router?.getCachedInlineMesh(meshId);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS = 30 * 60 * 1000;
|
|
69
|
+
|
|
70
|
+
function isIntentionalCleanupStopMetadata(event: Record<string, unknown>): boolean {
|
|
71
|
+
return event.intentional === true
|
|
72
|
+
|| event.intentionalStop === true
|
|
73
|
+
|| event.operatorCleanup === true
|
|
74
|
+
|| event.reason === 'operator_cleanup'
|
|
75
|
+
|| event.stopReason === 'operator_cleanup'
|
|
76
|
+
|| event.cleanupReason === 'operator_cleanup'
|
|
77
|
+
|| event.source === 'mesh_cleanup_sessions'
|
|
78
|
+
|| event.source === 'mesh_remove_node';
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function hasRecentIntentionalCleanupStop(meshId: string, sessionId?: string, nodeId?: string): boolean {
|
|
82
|
+
if (!sessionId && !nodeId) return false;
|
|
83
|
+
const cutoff = Date.now() - INTENTIONAL_CLEANUP_STOP_SUPPRESSION_MS;
|
|
84
|
+
const entries = readLedgerEntries(meshId, { tail: 200 });
|
|
85
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
86
|
+
const entry = entries[i];
|
|
87
|
+
const timestamp = new Date(entry.timestamp).getTime();
|
|
88
|
+
if (!Number.isNaN(timestamp) && timestamp < cutoff) break;
|
|
89
|
+
if (!isIntentionalCleanupStopEntry(entry)) continue;
|
|
90
|
+
if (sessionId && entry.sessionId === sessionId) return true;
|
|
91
|
+
if (!sessionId && nodeId && entry.nodeId === nodeId) return true;
|
|
92
|
+
}
|
|
93
|
+
return false;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function shouldSuppressIntentionalCleanupStop(args: {
|
|
97
|
+
event: string;
|
|
98
|
+
meshId: string;
|
|
99
|
+
metadataEvent: Record<string, unknown>;
|
|
100
|
+
sessionId?: string;
|
|
101
|
+
nodeId?: string;
|
|
102
|
+
}): boolean {
|
|
103
|
+
if (args.event !== 'agent:stopped' && args.event !== 'monitor:long_generating') return false;
|
|
104
|
+
if (isIntentionalCleanupStopMetadata(args.metadataEvent)) return true;
|
|
105
|
+
return hasRecentIntentionalCleanupStop(args.meshId, args.sessionId, args.nodeId);
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const RECENT_COMPLETION_FINGERPRINT_TTL_MS = 10 * 60 * 1000;
|
|
109
|
+
|
|
110
|
+
function hasFingerprintSeen(fingerprint: string): boolean {
|
|
111
|
+
try {
|
|
112
|
+
return MeshRuntimeStore.getInstance().hasCompletionFingerprint(fingerprint);
|
|
113
|
+
} catch {
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function recordFingerprintSeen(fingerprint: string): void {
|
|
119
|
+
try {
|
|
120
|
+
const db = MeshRuntimeStore.getInstance();
|
|
121
|
+
db.recordCompletionFingerprint(fingerprint, RECENT_COMPLETION_FINGERPRINT_TTL_MS);
|
|
122
|
+
db.sweepExpiredFingerprints();
|
|
123
|
+
} catch { /* best-effort; duplicate events are preferable to a crash */ }
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
function readEventTimestamp(value: unknown): number | null {
|
|
127
|
+
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
|
128
|
+
if (typeof value === 'string' && value.trim()) {
|
|
129
|
+
const numeric = Number(value);
|
|
130
|
+
if (Number.isFinite(numeric)) return numeric;
|
|
131
|
+
const parsed = Date.parse(value);
|
|
132
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
133
|
+
}
|
|
134
|
+
return null;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
function buildMeshCompletionFingerprint(args: {
|
|
138
|
+
meshId: string;
|
|
139
|
+
event: string;
|
|
140
|
+
sessionId: string;
|
|
141
|
+
providerType?: string;
|
|
142
|
+
providerSessionId?: string;
|
|
143
|
+
timestamp?: number | null;
|
|
144
|
+
finalSummary?: string;
|
|
145
|
+
coordinatorDaemonId?: string;
|
|
146
|
+
}): string {
|
|
147
|
+
const timestampPart = Number.isFinite(args.timestamp)
|
|
148
|
+
? String(args.timestamp)
|
|
149
|
+
: readNonEmptyString(args.finalSummary).slice(0, 200);
|
|
150
|
+
return [
|
|
151
|
+
args.meshId,
|
|
152
|
+
args.event,
|
|
153
|
+
args.sessionId,
|
|
154
|
+
args.providerType || '',
|
|
155
|
+
args.providerSessionId || '',
|
|
156
|
+
timestampPart,
|
|
157
|
+
args.coordinatorDaemonId || '',
|
|
158
|
+
].join('::');
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function isDuplicateMeshCompletionEvent(args: {
|
|
162
|
+
meshId: string;
|
|
163
|
+
event: string;
|
|
164
|
+
sessionId: string;
|
|
165
|
+
providerType?: string;
|
|
166
|
+
providerSessionId?: string;
|
|
167
|
+
timestamp?: number | null;
|
|
168
|
+
finalSummary?: string;
|
|
169
|
+
coordinatorDaemonId?: string;
|
|
170
|
+
taskId?: string;
|
|
171
|
+
nodeId?: string;
|
|
172
|
+
}): boolean {
|
|
173
|
+
const fingerprint = buildMeshCompletionFingerprint(args);
|
|
174
|
+
if (!fingerprint) return false;
|
|
175
|
+
if (hasFingerprintSeen(fingerprint)) {
|
|
176
|
+
if (args.taskId) {
|
|
177
|
+
recordCompletionConflict({
|
|
178
|
+
meshId: args.meshId,
|
|
179
|
+
fingerprint,
|
|
180
|
+
conflictingTaskId: args.taskId,
|
|
181
|
+
conflictingSessionId: args.sessionId,
|
|
182
|
+
event: args.event,
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
return true;
|
|
186
|
+
}
|
|
187
|
+
recordFingerprintSeen(fingerprint);
|
|
188
|
+
return false;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function isDuplicateMeshApprovalEvent(args: {
|
|
192
|
+
meshId: string;
|
|
193
|
+
sessionId: string;
|
|
194
|
+
providerType?: string;
|
|
195
|
+
timestamp?: number | null;
|
|
196
|
+
modalMessage?: string;
|
|
197
|
+
modalButtons?: unknown;
|
|
198
|
+
}): boolean {
|
|
199
|
+
const modalButtons = Array.isArray(args.modalButtons)
|
|
200
|
+
? args.modalButtons.map(button => String(button).trim()).filter(Boolean)
|
|
201
|
+
: [];
|
|
202
|
+
const approvalIdentity = Number.isFinite(args.timestamp)
|
|
203
|
+
? String(args.timestamp)
|
|
204
|
+
: JSON.stringify({ message: args.modalMessage || '', buttons: modalButtons });
|
|
205
|
+
if (!approvalIdentity || approvalIdentity === '{"message":"","buttons":[]}') return false;
|
|
206
|
+
const fingerprint = [
|
|
207
|
+
args.meshId,
|
|
208
|
+
'agent:waiting_approval',
|
|
209
|
+
args.sessionId,
|
|
210
|
+
args.providerType || '',
|
|
211
|
+
approvalIdentity,
|
|
212
|
+
].join('::');
|
|
213
|
+
if (hasFingerprintSeen(fingerprint)) return true;
|
|
214
|
+
recordFingerprintSeen(fingerprint);
|
|
215
|
+
return false;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function isDuplicateRefineTerminalEvent(meshId: string, eventName: string, metadataEvent: Record<string, unknown>): boolean {
|
|
219
|
+
const jobId = readRefineJobId({ metadataEvent });
|
|
220
|
+
const fingerprint = jobId && new Set(['refine:completed', 'refine:failed']).has(eventName) ? `${meshId}::${eventName}::${jobId}` : '';
|
|
221
|
+
if (!fingerprint) return false;
|
|
222
|
+
if (hasFingerprintSeen(fingerprint)) return true;
|
|
223
|
+
recordFingerprintSeen(fingerprint);
|
|
224
|
+
return false;
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
// ---------------------------------------------------------------------------
|
|
228
|
+
// Queue assignment
|
|
229
|
+
// ---------------------------------------------------------------------------
|
|
230
|
+
|
|
231
|
+
export function tryAssignQueueTask(
|
|
232
|
+
components: DaemonComponents,
|
|
233
|
+
meshId: string,
|
|
234
|
+
nodeId: string,
|
|
235
|
+
sessionId: string,
|
|
236
|
+
providerType: string
|
|
237
|
+
): boolean {
|
|
238
|
+
const mesh = getMeshWithCache(components, meshId);
|
|
239
|
+
const node = mesh?.nodes.find((n: any) => n.id === nodeId);
|
|
240
|
+
const capabilityTags = buildMeshNodeCapabilityTags(node, providerType);
|
|
241
|
+
const task = claimNextTask(meshId, nodeId, sessionId, capabilityTags);
|
|
242
|
+
if (!task) {
|
|
243
|
+
return false;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
|
|
247
|
+
|
|
248
|
+
if (node?.daemonId && components.dispatchMeshCommand) {
|
|
249
|
+
const isLocalNode = components.cliManager.adapters.has(sessionId);
|
|
250
|
+
if (!isLocalNode) {
|
|
251
|
+
const delivery = createSessionDelivery({
|
|
252
|
+
meshId,
|
|
253
|
+
nodeId,
|
|
254
|
+
sessionId,
|
|
255
|
+
providerType,
|
|
256
|
+
taskId: task.id,
|
|
257
|
+
kind: 'task',
|
|
258
|
+
message: task.message,
|
|
259
|
+
status: 'delivering',
|
|
260
|
+
});
|
|
261
|
+
components.dispatchMeshCommand(node.daemonId, 'agent_command', {
|
|
262
|
+
targetSessionId: sessionId,
|
|
263
|
+
cliType: providerType,
|
|
264
|
+
action: 'send_chat',
|
|
265
|
+
message: task.message,
|
|
266
|
+
}).then(() => {
|
|
267
|
+
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
268
|
+
}).catch((e: any) => {
|
|
269
|
+
LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
|
|
270
|
+
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
271
|
+
updateTaskStatus(meshId, task.id, 'pending');
|
|
272
|
+
try {
|
|
273
|
+
appendLedgerEntry(meshId, {
|
|
274
|
+
kind: 'dispatch_failed' as any,
|
|
275
|
+
nodeId,
|
|
276
|
+
sessionId,
|
|
277
|
+
payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
|
|
278
|
+
});
|
|
279
|
+
} catch { /* ledger write is best-effort */ }
|
|
280
|
+
});
|
|
281
|
+
return true;
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
const delivery = createSessionDelivery({
|
|
286
|
+
meshId,
|
|
287
|
+
nodeId,
|
|
288
|
+
sessionId,
|
|
289
|
+
providerType,
|
|
290
|
+
taskId: task.id,
|
|
291
|
+
kind: 'task',
|
|
292
|
+
message: task.message,
|
|
293
|
+
status: 'delivering',
|
|
294
|
+
});
|
|
295
|
+
components.cliManager.handleCliCommand('agent_command', {
|
|
296
|
+
targetSessionId: sessionId,
|
|
297
|
+
cliType: providerType,
|
|
298
|
+
action: 'send_chat',
|
|
299
|
+
message: task.message,
|
|
300
|
+
}).then(() => {
|
|
301
|
+
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
302
|
+
}).catch((e: any) => {
|
|
303
|
+
LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
|
|
304
|
+
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
305
|
+
updateTaskStatus(meshId, task.id, 'failed');
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
return true;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
const autoLaunchInProgress = new Set<string>();
|
|
312
|
+
const autoLaunchCooldownUntil = new Map<string, number>();
|
|
313
|
+
const AUTO_LAUNCH_COOLDOWN_MS = 5_000;
|
|
314
|
+
|
|
315
|
+
function sweepExpiredCooldowns(): void {
|
|
316
|
+
const now = Date.now();
|
|
317
|
+
for (const [key, until] of autoLaunchCooldownUntil) {
|
|
318
|
+
if (now >= until) autoLaunchCooldownUntil.delete(key);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function normalizeProviderPriority(policy: unknown): string[] {
|
|
323
|
+
const raw = policy && typeof policy === 'object' && !Array.isArray(policy)
|
|
324
|
+
? (policy as Record<string, unknown>).providerPriority
|
|
325
|
+
: undefined;
|
|
326
|
+
if (!Array.isArray(raw)) return [];
|
|
327
|
+
const seen = new Set<string>();
|
|
328
|
+
return raw
|
|
329
|
+
.map(type => typeof type === 'string' ? type.trim() : '')
|
|
330
|
+
.filter(Boolean)
|
|
331
|
+
.filter(type => {
|
|
332
|
+
if (seen.has(type)) return false;
|
|
333
|
+
seen.add(type);
|
|
334
|
+
return true;
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
function isTerminalSessionStatus(status: string): boolean {
|
|
339
|
+
return ['stopped', 'failed', 'terminated', 'exited', 'closed'].includes(status);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
function isIdleSessionState(state: any): boolean {
|
|
343
|
+
const status = readNonEmptyString(state?.status).toLowerCase();
|
|
344
|
+
if (isTerminalSessionStatus(status)) return false;
|
|
345
|
+
return status === 'idle' || state?.activeChat?.status === 'waiting_input';
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
function isDirtyNode(node: any): boolean {
|
|
349
|
+
return node?.health === 'dirty' || node?.git?.dirty === true;
|
|
350
|
+
}
|
|
351
|
+
|
|
352
|
+
function isLaunchableNode(node: any): boolean {
|
|
353
|
+
if (!node || node.status === 'disabled' || node.status === 'removed') return false;
|
|
354
|
+
const health = readNonEmptyString(node.health).toLowerCase();
|
|
355
|
+
if (!health) return true;
|
|
356
|
+
return health === 'online' || health === 'unknown';
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
function localAutoLaunchSkipReason(node: any): string | null {
|
|
360
|
+
const daemonId = readNonEmptyString(node?.daemonId);
|
|
361
|
+
const machineId = readNonEmptyString(node?.machineId);
|
|
362
|
+
const appConfig = loadConfig();
|
|
363
|
+
const localMachineId = readNonEmptyString(appConfig.machineId) || readNonEmptyString(appConfig.registeredMachineId);
|
|
364
|
+
const cloudDaemonId = localMachineId ? `daemon_${localMachineId}` : '';
|
|
365
|
+
const standaloneDaemonId = localMachineId ? `standalone_${localMachineId}` : '';
|
|
366
|
+
|
|
367
|
+
const daemonMatchesLocal = !daemonId || daemonId === cloudDaemonId || daemonId === standaloneDaemonId;
|
|
368
|
+
const machineMatchesLocal = !machineId || (localMachineId && machineId === localMachineId);
|
|
369
|
+
|
|
370
|
+
if (node?.isLocalWorktree === true) {
|
|
371
|
+
return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
if (daemonId || machineId) {
|
|
375
|
+
return daemonMatchesLocal && machineMatchesLocal ? null : 'remote_auto_launch_unsupported';
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
return null;
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
function activeAssignedCount(meshId: string): number {
|
|
382
|
+
return getQueue(meshId, { status: ['assigned'] as any }).length;
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
function nodeHasActiveAssignment(meshId: string, nodeId: string): boolean {
|
|
386
|
+
return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedNodeId === nodeId);
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
function sessionHasActiveAssignment(meshId: string, sessionId: string): boolean {
|
|
390
|
+
return getQueue(meshId, { status: ['assigned'] as any }).some(task => task.assignedSessionId === sessionId);
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
function liveSessionCountForNode(components: DaemonComponents, meshId: string, nodeId: string): number {
|
|
394
|
+
return components.instanceManager.getByCategory('cli').filter((inst: any) => {
|
|
395
|
+
const state = inst.getState();
|
|
396
|
+
const settings = state.settings as Record<string, unknown> || {};
|
|
397
|
+
if (readNonEmptyString(settings.meshNodeFor) !== meshId) return false;
|
|
398
|
+
const instNodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
399
|
+
if (instNodeId !== nodeId) return false;
|
|
400
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
401
|
+
return !isTerminalSessionStatus(status);
|
|
402
|
+
}).length;
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
function recordAutoLaunchEvent(meshId: string, args: {
|
|
406
|
+
phase: 'skipped' | 'started' | 'failed' | 'completed';
|
|
407
|
+
taskId: string;
|
|
408
|
+
nodeId?: string;
|
|
409
|
+
providerType?: string;
|
|
410
|
+
sessionId?: string;
|
|
411
|
+
reason?: string;
|
|
412
|
+
error?: string;
|
|
413
|
+
}) {
|
|
414
|
+
try {
|
|
415
|
+
appendLedgerEntry(meshId, {
|
|
416
|
+
kind: 'session_auto_launch',
|
|
417
|
+
nodeId: args.nodeId,
|
|
418
|
+
sessionId: args.sessionId,
|
|
419
|
+
providerType: args.providerType,
|
|
420
|
+
payload: {
|
|
421
|
+
phase: args.phase,
|
|
422
|
+
taskId: args.taskId,
|
|
423
|
+
reason: args.reason,
|
|
424
|
+
error: args.error,
|
|
425
|
+
},
|
|
426
|
+
});
|
|
427
|
+
} catch (e: any) {
|
|
428
|
+
LOG.warn('MeshQueue', `Failed to record auto-launch ledger event: ${e?.message || e}`);
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function markAutoLaunch(meshId: string, taskId: string, args: {
|
|
433
|
+
status: 'skipped' | 'started' | 'failed' | 'completed';
|
|
434
|
+
reason?: string;
|
|
435
|
+
nodeId?: string;
|
|
436
|
+
providerType?: string;
|
|
437
|
+
sessionId?: string;
|
|
438
|
+
error?: string;
|
|
439
|
+
}) {
|
|
440
|
+
recordTaskAutoLaunch(meshId, taskId, {
|
|
441
|
+
status: args.status,
|
|
442
|
+
reason: args.reason || args.error,
|
|
443
|
+
nodeId: args.nodeId,
|
|
444
|
+
providerType: args.providerType,
|
|
445
|
+
sessionId: args.sessionId,
|
|
446
|
+
});
|
|
447
|
+
recordAutoLaunchEvent(meshId, {
|
|
448
|
+
phase: args.status,
|
|
449
|
+
taskId,
|
|
450
|
+
nodeId: args.nodeId,
|
|
451
|
+
providerType: args.providerType,
|
|
452
|
+
sessionId: args.sessionId,
|
|
453
|
+
reason: args.reason,
|
|
454
|
+
error: args.error,
|
|
455
|
+
});
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
async function resolveUsableProvider(components: DaemonComponents, nodeId: string, node: any): Promise<{ providerType?: string; reason?: string }> {
|
|
459
|
+
const providerPriority = normalizeProviderPriority(node?.policy);
|
|
460
|
+
if (!providerPriority.length) return { reason: 'missing_provider_priority' };
|
|
461
|
+
const providerLoader = components.providerLoader;
|
|
462
|
+
if (!providerLoader) return { reason: 'provider_loader_unavailable' };
|
|
463
|
+
|
|
464
|
+
const failed: string[] = [];
|
|
465
|
+
for (const requestedType of providerPriority) {
|
|
466
|
+
const normalizedType = typeof providerLoader.resolveAlias === 'function'
|
|
467
|
+
? providerLoader.resolveAlias(requestedType)
|
|
468
|
+
: requestedType;
|
|
469
|
+
if (typeof providerLoader.isMachineProviderEnabled === 'function' && !providerLoader.isMachineProviderEnabled(normalizedType)) {
|
|
470
|
+
failed.push(`${requestedType}: disabled`);
|
|
471
|
+
continue;
|
|
472
|
+
}
|
|
473
|
+
let detected: any;
|
|
474
|
+
try {
|
|
475
|
+
detected = await detectCLI(normalizedType, providerLoader, { includeVersion: false });
|
|
476
|
+
} catch (e: any) {
|
|
477
|
+
failed.push(`${requestedType}: detect failed: ${e?.message || e}`);
|
|
478
|
+
continue;
|
|
479
|
+
}
|
|
480
|
+
if (typeof providerLoader.setCliDetectionResults === 'function') {
|
|
481
|
+
providerLoader.setCliDetectionResults([{
|
|
482
|
+
id: normalizedType,
|
|
483
|
+
installed: !!detected,
|
|
484
|
+
path: detected?.path,
|
|
485
|
+
}], false);
|
|
486
|
+
}
|
|
487
|
+
(components as any).onStatusChange?.();
|
|
488
|
+
if (detected) return { providerType: normalizedType };
|
|
489
|
+
failed.push(`${requestedType}: not detected`);
|
|
490
|
+
}
|
|
491
|
+
return { reason: `provider_priority_unusable: ${failed.join('; ') || nodeId}` };
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, meshId: string, mesh: any): Promise<boolean> {
|
|
495
|
+
const queue = getQueue(meshId);
|
|
496
|
+
const pending = queue.filter(task => task.status === 'pending');
|
|
497
|
+
if (!pending.length) return false;
|
|
498
|
+
|
|
499
|
+
const maxParallelTasks = Math.max(1, Math.floor(Number(mesh?.policy?.maxParallelTasks) || 2));
|
|
500
|
+
for (const task of pending) {
|
|
501
|
+
if (activeAssignedCount(meshId) >= maxParallelTasks) {
|
|
502
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_parallel_tasks_reached' });
|
|
503
|
+
return false;
|
|
504
|
+
}
|
|
505
|
+
if (task.targetSessionId) {
|
|
506
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'target_session_constraint' });
|
|
507
|
+
continue;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
const candidateNodes = Array.isArray(mesh?.nodes)
|
|
511
|
+
? mesh.nodes.filter((node: any) => task.targetNodeId ? node?.id === task.targetNodeId : true)
|
|
512
|
+
: [];
|
|
513
|
+
if (!candidateNodes.length) {
|
|
514
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'no_matching_node', nodeId: task.targetNodeId });
|
|
515
|
+
continue;
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
for (const node of candidateNodes) {
|
|
519
|
+
const nodeId = readNonEmptyString(node?.id);
|
|
520
|
+
if (!nodeId) continue;
|
|
521
|
+
const launchKey = `${meshId}:${nodeId}`;
|
|
522
|
+
const now = Date.now();
|
|
523
|
+
const cooldownUntil = autoLaunchCooldownUntil.get(launchKey) || 0;
|
|
524
|
+
if (cooldownUntil > 0 && now >= cooldownUntil) autoLaunchCooldownUntil.delete(launchKey);
|
|
525
|
+
if (autoLaunchInProgress.has(launchKey)) {
|
|
526
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_in_progress', nodeId });
|
|
527
|
+
continue;
|
|
528
|
+
}
|
|
529
|
+
if (now < cooldownUntil) {
|
|
530
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'auto_launch_cooldown', nodeId });
|
|
531
|
+
continue;
|
|
532
|
+
}
|
|
533
|
+
if (isDirtyNode(node)) {
|
|
534
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'dirty_workspace', nodeId });
|
|
535
|
+
continue;
|
|
536
|
+
}
|
|
537
|
+
if (!isLaunchableNode(node)) {
|
|
538
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_not_launch_ready', nodeId });
|
|
539
|
+
continue;
|
|
540
|
+
}
|
|
541
|
+
const localSkipReason = localAutoLaunchSkipReason(node);
|
|
542
|
+
if (localSkipReason) {
|
|
543
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: localSkipReason, nodeId });
|
|
544
|
+
continue;
|
|
545
|
+
}
|
|
546
|
+
if (nodeHasActiveAssignment(meshId, nodeId)) {
|
|
547
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'node_has_active_assignment', nodeId });
|
|
548
|
+
continue;
|
|
549
|
+
}
|
|
550
|
+
const maxConcurrentSessions = Number(node?.policy?.maxConcurrentSessions);
|
|
551
|
+
if (Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0 && liveSessionCountForNode(components, meshId, nodeId) >= maxConcurrentSessions) {
|
|
552
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'max_concurrent_sessions_reached', nodeId });
|
|
553
|
+
continue;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
autoLaunchInProgress.add(launchKey);
|
|
557
|
+
try {
|
|
558
|
+
const resolved = await resolveUsableProvider(components, nodeId, node);
|
|
559
|
+
if (!resolved.providerType) {
|
|
560
|
+
markAutoLaunch(meshId, task.id, { status: 'skipped', reason: resolved.reason || 'provider_unusable', nodeId });
|
|
561
|
+
continue;
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
markAutoLaunch(meshId, task.id, { status: 'started', nodeId, providerType: resolved.providerType });
|
|
565
|
+
const launchResult: any = await components.cliManager.handleCliCommand('launch_cli', {
|
|
566
|
+
cliType: resolved.providerType,
|
|
567
|
+
dir: node.workspace,
|
|
568
|
+
settings: {
|
|
569
|
+
// Worker launch envelope: role + mesh context so worker can route completion events.
|
|
570
|
+
role: 'worker',
|
|
571
|
+
meshNodeFor: meshId,
|
|
572
|
+
meshNodeId: nodeId,
|
|
573
|
+
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
574
|
+
launchedByCoordinator: true,
|
|
575
|
+
autoLaunchedForQueueTaskId: task.id,
|
|
576
|
+
},
|
|
577
|
+
});
|
|
578
|
+
if (!launchResult?.success) {
|
|
579
|
+
const reason = launchResult?.error || 'launch_cli_failed';
|
|
580
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason, nodeId, providerType: resolved.providerType });
|
|
581
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
582
|
+
return false;
|
|
583
|
+
}
|
|
584
|
+
const sessionId = readNonEmptyString(launchResult.sessionId) || readNonEmptyString(launchResult.id) || readNonEmptyString(launchResult.runtimeSessionId);
|
|
585
|
+
if (!sessionId) {
|
|
586
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', reason: 'launch_missing_session_id', nodeId, providerType: resolved.providerType });
|
|
587
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS); sweepExpiredCooldowns();
|
|
588
|
+
return false;
|
|
589
|
+
}
|
|
590
|
+
markAutoLaunch(meshId, task.id, { status: 'completed', nodeId, providerType: resolved.providerType, sessionId });
|
|
591
|
+
tryAssignQueueTask(components, meshId, nodeId, sessionId, resolved.providerType);
|
|
592
|
+
return true;
|
|
593
|
+
} catch (e: any) {
|
|
594
|
+
markAutoLaunch(meshId, task.id, { status: 'failed', error: e?.message || String(e), nodeId });
|
|
595
|
+
autoLaunchCooldownUntil.set(launchKey, Date.now() + AUTO_LAUNCH_COOLDOWN_MS);
|
|
596
|
+
return false;
|
|
597
|
+
} finally {
|
|
598
|
+
autoLaunchInProgress.delete(launchKey);
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
return false;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
export interface MeshQueueTriggerResult {
|
|
606
|
+
success: true;
|
|
607
|
+
meshId: string;
|
|
608
|
+
pendingBefore: number;
|
|
609
|
+
assignedBefore: number;
|
|
610
|
+
pendingAfter: number;
|
|
611
|
+
assignedAfter: number;
|
|
612
|
+
claimed: boolean;
|
|
613
|
+
newlyAssignedTasks: Array<{
|
|
614
|
+
id: string;
|
|
615
|
+
nodeId?: string;
|
|
616
|
+
sessionId?: string;
|
|
617
|
+
}>;
|
|
618
|
+
localIdleSessionsChecked: number;
|
|
619
|
+
remoteIdleSessionsChecked: number;
|
|
620
|
+
skippedSessions: Array<{
|
|
621
|
+
nodeId?: string;
|
|
622
|
+
sessionId?: string;
|
|
623
|
+
reason: string;
|
|
624
|
+
status?: string;
|
|
625
|
+
}>;
|
|
626
|
+
autoLaunchStarted: boolean;
|
|
627
|
+
noIdleMeshSessionAvailable?: boolean;
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
function countQueueStatus(meshId: string, status: 'pending' | 'assigned'): number {
|
|
631
|
+
return getQueue(meshId, { status: [status] as any }).length;
|
|
632
|
+
}
|
|
633
|
+
|
|
634
|
+
function getQueueStatusById(meshId: string): Map<string, string> {
|
|
635
|
+
return new Map(getQueue(meshId).map(task => [task.id, task.status]));
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
export async function triggerMeshQueue(components: DaemonComponents, meshId: string): Promise<MeshQueueTriggerResult> {
|
|
639
|
+
const mesh = getMeshWithCache(components, meshId);
|
|
640
|
+
const pendingBefore = countQueueStatus(meshId, 'pending');
|
|
641
|
+
const assignedBefore = countQueueStatus(meshId, 'assigned');
|
|
642
|
+
const beforeStatus = getQueueStatusById(meshId);
|
|
643
|
+
const skippedSessions: MeshQueueTriggerResult['skippedSessions'] = [];
|
|
644
|
+
let localIdleSessionsChecked = 0;
|
|
645
|
+
let remoteIdleSessionsChecked = 0;
|
|
646
|
+
let autoLaunchStarted = false;
|
|
647
|
+
if (!mesh) {
|
|
648
|
+
return {
|
|
649
|
+
success: true,
|
|
650
|
+
meshId,
|
|
651
|
+
pendingBefore,
|
|
652
|
+
assignedBefore,
|
|
653
|
+
pendingAfter: pendingBefore,
|
|
654
|
+
assignedAfter: assignedBefore,
|
|
655
|
+
claimed: false,
|
|
656
|
+
newlyAssignedTasks: [],
|
|
657
|
+
localIdleSessionsChecked,
|
|
658
|
+
remoteIdleSessionsChecked,
|
|
659
|
+
skippedSessions: [{ reason: 'mesh_not_found' }],
|
|
660
|
+
autoLaunchStarted,
|
|
661
|
+
noIdleMeshSessionAvailable: true,
|
|
662
|
+
};
|
|
663
|
+
}
|
|
664
|
+
|
|
665
|
+
const cliInstances = components.instanceManager.getByCategory('cli');
|
|
666
|
+
for (const inst of cliInstances) {
|
|
667
|
+
const state = inst.getState();
|
|
668
|
+
const settings = state.settings as Record<string, unknown> || {};
|
|
669
|
+
|
|
670
|
+
const instMeshId = readNonEmptyString(settings.meshNodeFor);
|
|
671
|
+
if (instMeshId !== meshId) continue;
|
|
672
|
+
|
|
673
|
+
const nodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
|
|
674
|
+
if (!nodeId) continue;
|
|
675
|
+
|
|
676
|
+
if (!isIdleSessionState(state)) {
|
|
677
|
+
const status = readNonEmptyString(state.status).toLowerCase();
|
|
678
|
+
skippedSessions.push({
|
|
679
|
+
nodeId,
|
|
680
|
+
sessionId: readNonEmptyString(state.instanceId),
|
|
681
|
+
reason: isTerminalSessionStatus(status) ? 'terminal_session' : 'session_not_idle',
|
|
682
|
+
status: status || undefined,
|
|
683
|
+
});
|
|
684
|
+
continue;
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
const sessionId = state.instanceId;
|
|
688
|
+
const providerType = state.type || readNonEmptyString(settings.providerType);
|
|
689
|
+
|
|
690
|
+
if (providerType) {
|
|
691
|
+
localIdleSessionsChecked += 1;
|
|
692
|
+
tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
|
|
693
|
+
} else {
|
|
694
|
+
skippedSessions.push({
|
|
695
|
+
nodeId,
|
|
696
|
+
sessionId,
|
|
697
|
+
reason: 'provider_type_missing',
|
|
698
|
+
});
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
let remoteSessions: Array<{ nodeId: string; sessionId: string; providerType: string }> = [];
|
|
703
|
+
try {
|
|
704
|
+
remoteSessions = MeshRuntimeStore.getInstance().getRemoteIdleSessions();
|
|
705
|
+
} catch { /* best-effort */ }
|
|
706
|
+
|
|
707
|
+
for (const idle of remoteSessions) {
|
|
708
|
+
const node = mesh.nodes.find((n: any) => n.id === idle.nodeId);
|
|
709
|
+
if (node) {
|
|
710
|
+
remoteIdleSessionsChecked += 1;
|
|
711
|
+
const assigned = tryAssignQueueTask(components, meshId, idle.nodeId, idle.sessionId, idle.providerType);
|
|
712
|
+
if (assigned) {
|
|
713
|
+
try {
|
|
714
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(idle.nodeId, idle.sessionId);
|
|
715
|
+
} catch { /* best-effort */ }
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
autoLaunchStarted = await maybeAutoLaunchOneQueueSession(components, meshId, mesh);
|
|
721
|
+
const afterQueue = getQueue(meshId);
|
|
722
|
+
const pendingAfter = afterQueue.filter(task => task.status === 'pending').length;
|
|
723
|
+
const assignedAfter = afterQueue.filter(task => task.status === 'assigned').length;
|
|
724
|
+
const newlyAssignedTasks = afterQueue
|
|
725
|
+
.filter(task => task.status === 'assigned' && beforeStatus.get(task.id) !== 'assigned')
|
|
726
|
+
.map(task => ({
|
|
727
|
+
id: task.id,
|
|
728
|
+
nodeId: task.assignedNodeId,
|
|
729
|
+
sessionId: task.assignedSessionId,
|
|
730
|
+
}));
|
|
731
|
+
return {
|
|
732
|
+
success: true,
|
|
733
|
+
meshId,
|
|
734
|
+
pendingBefore,
|
|
735
|
+
assignedBefore,
|
|
736
|
+
pendingAfter,
|
|
737
|
+
assignedAfter,
|
|
738
|
+
claimed: newlyAssignedTasks.length > 0,
|
|
739
|
+
newlyAssignedTasks,
|
|
740
|
+
localIdleSessionsChecked,
|
|
741
|
+
remoteIdleSessionsChecked,
|
|
742
|
+
skippedSessions,
|
|
743
|
+
autoLaunchStarted,
|
|
744
|
+
...(pendingAfter > 0 && newlyAssignedTasks.length === 0 && localIdleSessionsChecked === 0 && remoteIdleSessionsChecked === 0 && !autoLaunchStarted
|
|
745
|
+
? { noIdleMeshSessionAvailable: true }
|
|
746
|
+
: {}),
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
|
|
750
|
+
async function maybeAutoFastForwardIdleNode(components: DaemonComponents, args: {
|
|
751
|
+
meshId: string;
|
|
752
|
+
nodeId: string;
|
|
753
|
+
sessionId?: string;
|
|
754
|
+
providerType?: string;
|
|
755
|
+
}): Promise<void> {
|
|
756
|
+
const mesh = getMeshWithCache(components, args.meshId);
|
|
757
|
+
const node = mesh?.nodes?.find((candidate: any) => candidate?.id === args.nodeId || candidate?.nodeId === args.nodeId);
|
|
758
|
+
const workspace = readNonEmptyString(node?.workspace);
|
|
759
|
+
if (!workspace) return;
|
|
760
|
+
if (!existsSync(workspace)) return;
|
|
761
|
+
|
|
762
|
+
const throttleKey = `${args.meshId}:${args.nodeId}`;
|
|
763
|
+
const now = Date.now();
|
|
764
|
+
const lastAttempt = idleAutoFastForwardLastAttempt.get(throttleKey) || 0;
|
|
765
|
+
if (now - lastAttempt < IDLE_AUTO_FAST_FORWARD_THROTTLE_MS) return;
|
|
766
|
+
idleAutoFastForwardLastAttempt.set(throttleKey, now);
|
|
767
|
+
|
|
768
|
+
const submoduleIgnorePaths = Array.isArray(node?.policy?.submoduleIgnorePaths)
|
|
769
|
+
? node.policy.submoduleIgnorePaths.filter((value: unknown): value is string => typeof value === 'string')
|
|
770
|
+
: undefined;
|
|
771
|
+
try {
|
|
772
|
+
const dryRun = await fastForwardMeshNode({
|
|
773
|
+
meshId: args.meshId,
|
|
774
|
+
nodeId: args.nodeId,
|
|
775
|
+
workspace,
|
|
776
|
+
execute: false,
|
|
777
|
+
dryRun: true,
|
|
778
|
+
updateSubmodules: false,
|
|
779
|
+
submoduleIgnorePaths,
|
|
780
|
+
trigger: 'idle_auto',
|
|
781
|
+
});
|
|
782
|
+
if (!dryRun || dryRun.code !== 'fast_forward_available' || dryRun.allowed !== true) return;
|
|
783
|
+
await fastForwardMeshNode({
|
|
784
|
+
meshId: args.meshId,
|
|
785
|
+
nodeId: args.nodeId,
|
|
786
|
+
workspace,
|
|
787
|
+
execute: true,
|
|
788
|
+
dryRun: false,
|
|
789
|
+
updateSubmodules: false,
|
|
790
|
+
submoduleIgnorePaths,
|
|
791
|
+
trigger: 'idle_auto',
|
|
792
|
+
});
|
|
793
|
+
} catch (e: any) {
|
|
794
|
+
LOG.warn('MeshFastForward', `Idle auto fast-forward check failed for ${args.nodeId}: ${e?.message || e}`);
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
function runIdleMaintenanceThenAssignQueue(components: DaemonComponents, args: {
|
|
799
|
+
meshId: string;
|
|
800
|
+
nodeId: string;
|
|
801
|
+
sessionId: string;
|
|
802
|
+
providerType: string;
|
|
803
|
+
}): void {
|
|
804
|
+
setImmediate(() => {
|
|
805
|
+
maybeAutoFastForwardIdleNode(components, args)
|
|
806
|
+
.finally(() => {
|
|
807
|
+
try {
|
|
808
|
+
tryAssignQueueTask(components, args.meshId, args.nodeId, args.sessionId, args.providerType);
|
|
809
|
+
} catch (e: any) {
|
|
810
|
+
LOG.warn('MeshQueue', `Failed to assign idle queue task after maintenance for ${args.nodeId}: ${e?.message || e}`);
|
|
811
|
+
}
|
|
812
|
+
});
|
|
813
|
+
});
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
// ---------------------------------------------------------------------------
|
|
817
|
+
// Core event injection
|
|
818
|
+
// ---------------------------------------------------------------------------
|
|
819
|
+
|
|
820
|
+
const MESH_COORDINATOR_EVENTS = new Set([
|
|
821
|
+
'agent:generating_started',
|
|
822
|
+
'agent:generating_completed',
|
|
823
|
+
'agent:waiting_approval',
|
|
824
|
+
'agent:stopped',
|
|
825
|
+
'agent:ready',
|
|
826
|
+
'monitor:long_generating',
|
|
827
|
+
'refine:accepted',
|
|
828
|
+
'refine:completed',
|
|
829
|
+
'refine:failed',
|
|
830
|
+
]);
|
|
831
|
+
|
|
832
|
+
const EVENT_TO_LEDGER_KIND: Record<string, MeshLedgerKind> = {
|
|
833
|
+
'agent:generating_completed': 'task_completed',
|
|
834
|
+
'agent:waiting_approval': 'task_approval_needed',
|
|
835
|
+
'agent:stopped': 'task_failed',
|
|
836
|
+
'monitor:long_generating': 'task_stalled',
|
|
837
|
+
};
|
|
838
|
+
|
|
839
|
+
export function isMeshCoordinatorEvent(eventName: unknown): eventName is string {
|
|
840
|
+
return typeof eventName === 'string' && MESH_COORDINATOR_EVENTS.has(eventName);
|
|
841
|
+
}
|
|
842
|
+
|
|
843
|
+
function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
844
|
+
meshId: string;
|
|
845
|
+
sourceInstanceId?: string;
|
|
846
|
+
nodeId?: string;
|
|
847
|
+
nodeLabel: string;
|
|
848
|
+
event: string;
|
|
849
|
+
metadataEvent: Record<string, unknown>;
|
|
850
|
+
}) {
|
|
851
|
+
const eventSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
852
|
+
const eventNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
853
|
+
|
|
854
|
+
const sourceSession = args.sourceInstanceId
|
|
855
|
+
? components.instanceManager.getInstance(args.sourceInstanceId)
|
|
856
|
+
: undefined;
|
|
857
|
+
const workerCoordinatorDaemonId = readNonEmptyString(
|
|
858
|
+
(sourceSession?.getState()?.settings as Record<string, unknown>)?.meshCoordinatorDaemonId,
|
|
859
|
+
);
|
|
860
|
+
const localDaemonId = readNonEmptyString(loadConfig().machineId);
|
|
861
|
+
const intentionalCleanupStop = shouldSuppressIntentionalCleanupStop({
|
|
862
|
+
event: args.event,
|
|
863
|
+
meshId: args.meshId,
|
|
864
|
+
metadataEvent: args.metadataEvent,
|
|
865
|
+
sessionId: eventSessionId || undefined,
|
|
866
|
+
nodeId: eventNodeId || undefined,
|
|
867
|
+
});
|
|
868
|
+
if (intentionalCleanupStop) {
|
|
869
|
+
if (eventSessionId && eventNodeId) {
|
|
870
|
+
try {
|
|
871
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(eventNodeId, eventSessionId);
|
|
872
|
+
} catch { /* best-effort */ }
|
|
873
|
+
}
|
|
874
|
+
LOG.info('MeshEvents', `Suppressed ${args.event} for intentionally cleanup-stopped session ${eventSessionId || '(unknown session)'}`);
|
|
875
|
+
return { success: true, forwarded: 0, suppressed: true, intentionalCleanupStop: true };
|
|
876
|
+
}
|
|
877
|
+
|
|
878
|
+
if (args.event === 'monitor:long_generating') {
|
|
879
|
+
const reconciledCompletion = buildLongGeneratingCompletionReconciliation({
|
|
880
|
+
meshId: args.meshId,
|
|
881
|
+
nodeId: args.nodeId,
|
|
882
|
+
nodeLabel: args.nodeLabel,
|
|
883
|
+
metadataEvent: args.metadataEvent,
|
|
884
|
+
sourceInstanceId: args.sourceInstanceId,
|
|
885
|
+
});
|
|
886
|
+
if (reconciledCompletion?.source === 'long_generating_reconciliation') {
|
|
887
|
+
LOG.info('MeshEvents', `Reconciled long-generating monitor to completion for session ${eventSessionId || '(unknown session)'}`);
|
|
888
|
+
return injectMeshSystemMessage(components, {
|
|
889
|
+
...args,
|
|
890
|
+
event: 'agent:generating_completed',
|
|
891
|
+
metadataEvent: reconciledCompletion,
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
if (reconciledCompletion?.source === 'long_generating_terminal_ledger_suppression') {
|
|
895
|
+
LOG.info('MeshEvents', `Suppressed long-generating monitor because terminal ledger evidence already exists for session ${eventSessionId || '(unknown session)'}`);
|
|
896
|
+
return {
|
|
897
|
+
success: true,
|
|
898
|
+
forwarded: 0,
|
|
899
|
+
suppressed: true,
|
|
900
|
+
terminalLedgerEvidence: true,
|
|
901
|
+
terminalLedgerKind: reconciledCompletion.terminalLedgerKind,
|
|
902
|
+
};
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
if (isDuplicateRefineTerminalEvent(args.meshId, args.event, args.metadataEvent)) {
|
|
907
|
+
LOG.info('MeshEvents', `Suppressed duplicate ${args.event} for refine job ${readRefineJobId({ metadataEvent: args.metadataEvent })}`);
|
|
908
|
+
return { success: true, forwarded: 0, suppressed: true, duplicateRefineTerminalEvent: true };
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
const eventTimestamp = readEventTimestamp(args.metadataEvent.timestamp);
|
|
912
|
+
if (args.event === 'agent:waiting_approval' && eventSessionId) {
|
|
913
|
+
const duplicateApproval = isDuplicateMeshApprovalEvent({
|
|
914
|
+
meshId: args.meshId,
|
|
915
|
+
sessionId: eventSessionId,
|
|
916
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
917
|
+
timestamp: eventTimestamp,
|
|
918
|
+
modalMessage: readNonEmptyString(args.metadataEvent.modalMessage) || undefined,
|
|
919
|
+
modalButtons: args.metadataEvent.modalButtons,
|
|
920
|
+
});
|
|
921
|
+
if (duplicateApproval) {
|
|
922
|
+
LOG.info('MeshEvents', `Suppressed duplicate approval event for mesh ${args.meshId} session ${eventSessionId}`);
|
|
923
|
+
return { success: true, forwarded: 0, suppressed: true, duplicateApproval: true };
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
if (args.event === 'agent:generating_completed' && eventSessionId) {
|
|
927
|
+
const terminal = findRecentTerminalLedgerEvidence({
|
|
928
|
+
meshId: args.meshId,
|
|
929
|
+
sessionId: eventSessionId,
|
|
930
|
+
nodeId: eventNodeId || undefined,
|
|
931
|
+
});
|
|
932
|
+
if (terminal?.kind === 'task_completed' && !sessionHasActiveAssignment(args.meshId, eventSessionId)) {
|
|
933
|
+
const newDispatchAfterTerminal = hasDispatchAfterTerminal(args.meshId, eventSessionId, terminal.id);
|
|
934
|
+
if (!newDispatchAfterTerminal) {
|
|
935
|
+
const terminalProviderSessionId = readNonEmptyString(terminal.payload.providerSessionId);
|
|
936
|
+
const terminalFinalSummary = readNonEmptyString(terminal.payload.finalSummary);
|
|
937
|
+
const eventProviderSessionId = readNonEmptyString(args.metadataEvent.providerSessionId);
|
|
938
|
+
const eventFinalSummary = readNonEmptyString(args.metadataEvent.finalSummary);
|
|
939
|
+
if (
|
|
940
|
+
(terminalProviderSessionId && terminalProviderSessionId === eventProviderSessionId)
|
|
941
|
+
|| (terminalFinalSummary && terminalFinalSummary === eventFinalSummary)
|
|
942
|
+
|| args.metadataEvent.source === 'long_generating_reconciliation'
|
|
943
|
+
) {
|
|
944
|
+
LOG.info('MeshEvents', `Suppressed duplicate completion with existing terminal ledger evidence for mesh ${args.meshId} session ${eventSessionId}`);
|
|
945
|
+
return { success: true, forwarded: 0, suppressed: true, duplicateCompletion: true, terminalLedgerEvidence: true };
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
const duplicateCompletion = isDuplicateMeshCompletionEvent({
|
|
950
|
+
meshId: args.meshId,
|
|
951
|
+
event: args.event,
|
|
952
|
+
sessionId: eventSessionId,
|
|
953
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
954
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
955
|
+
timestamp: eventTimestamp,
|
|
956
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
957
|
+
coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
|
|
958
|
+
taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
|
|
959
|
+
nodeId: eventNodeId || undefined,
|
|
960
|
+
});
|
|
961
|
+
if (duplicateCompletion) {
|
|
962
|
+
LOG.info('MeshEvents', `Suppressed duplicate completion for mesh ${args.meshId} session ${eventSessionId}`);
|
|
963
|
+
return { success: true, forwarded: 0, suppressed: true, duplicateCompletion: true };
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
if (args.event === 'agent:stopped' && eventSessionId) {
|
|
967
|
+
const duplicateStopped = isDuplicateMeshCompletionEvent({
|
|
968
|
+
meshId: args.meshId,
|
|
969
|
+
event: args.event,
|
|
970
|
+
sessionId: eventSessionId,
|
|
971
|
+
providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
|
|
972
|
+
providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
|
|
973
|
+
timestamp: eventTimestamp,
|
|
974
|
+
finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
|
|
975
|
+
coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
|
|
976
|
+
taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
|
|
977
|
+
nodeId: eventNodeId || undefined,
|
|
978
|
+
});
|
|
979
|
+
if (duplicateStopped) {
|
|
980
|
+
LOG.info('MeshEvents', `Suppressed duplicate stopped event for mesh ${args.meshId} session ${eventSessionId}`);
|
|
981
|
+
return { success: true, forwarded: 0, suppressed: true, duplicateStopped: true };
|
|
982
|
+
}
|
|
983
|
+
}
|
|
984
|
+
|
|
985
|
+
function markSessionTerminal(sessionId: string, outcome: 'completed' | 'failed', occurredAtMs?: number | null): { id?: string } | null {
|
|
986
|
+
const task = updateSessionTaskStatus(args.meshId, sessionId, outcome, {
|
|
987
|
+
occurredAt: occurredAtMs != null ? new Date(occurredAtMs).toISOString() : undefined,
|
|
988
|
+
});
|
|
989
|
+
updateDirectDispatchStatus(args.meshId, sessionId, outcome);
|
|
990
|
+
markSessionDeliveriesTerminal(args.meshId, sessionId, outcome);
|
|
991
|
+
setImmediate(() => cleanupTerminalDirectDispatches());
|
|
992
|
+
return task ? { id: task.id } : null;
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
let completedTaskForLedger: { id?: string } | null = null;
|
|
996
|
+
if (args.event === 'agent:generating_completed') {
|
|
997
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
998
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
999
|
+
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
1000
|
+
|
|
1001
|
+
if (sessionId) {
|
|
1002
|
+
completedTaskForLedger = markSessionTerminal(sessionId, 'completed', eventTimestamp);
|
|
1003
|
+
if (nodeId && providerType) {
|
|
1004
|
+
runIdleMaintenanceThenAssignQueue(components, { meshId: args.meshId, nodeId, sessionId, providerType });
|
|
1005
|
+
}
|
|
1006
|
+
// M1-3: wake dependents of the completed task. The maintenance path above
|
|
1007
|
+
// only assigns to the completing session; dependents may be claimable by
|
|
1008
|
+
// other idle sessions, so run a full queue trigger when any are waiting.
|
|
1009
|
+
const completedTaskId = completedTaskForLedger?.id;
|
|
1010
|
+
if (completedTaskId && hasPendingDependents(args.meshId, completedTaskId)) {
|
|
1011
|
+
setImmediate(() => {
|
|
1012
|
+
triggerMeshQueue(components, args.meshId).catch((e: any) => {
|
|
1013
|
+
LOG.warn('MeshQueue', `Dependent wake after task ${completedTaskId} failed: ${e?.message || e}`);
|
|
1014
|
+
});
|
|
1015
|
+
});
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
} else if (args.event === 'agent:ready') {
|
|
1019
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
1020
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1021
|
+
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
1022
|
+
const providerSessionId = readNonEmptyString(args.metadataEvent.providerSessionId) || undefined;
|
|
1023
|
+
const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary) || undefined;
|
|
1024
|
+
const workerResult = readWorkerResultMetadata(args.metadataEvent);
|
|
1025
|
+
const hasCompletionEvidence = !!finalSummary || !!workerResult;
|
|
1026
|
+
if (sessionId && hasCompletionEvidence) {
|
|
1027
|
+
completedTaskForLedger = markSessionTerminal(sessionId, 'completed');
|
|
1028
|
+
if (completedTaskForLedger) {
|
|
1029
|
+
try {
|
|
1030
|
+
appendLedgerEntry(args.meshId, {
|
|
1031
|
+
kind: 'task_completed',
|
|
1032
|
+
nodeId: nodeId || undefined,
|
|
1033
|
+
sessionId,
|
|
1034
|
+
providerType: providerType || undefined,
|
|
1035
|
+
payload: {
|
|
1036
|
+
event: args.event,
|
|
1037
|
+
nodeLabel: args.nodeLabel,
|
|
1038
|
+
taskId: completedTaskForLedger.id,
|
|
1039
|
+
completedViaReady: true,
|
|
1040
|
+
providerSessionId,
|
|
1041
|
+
finalSummary,
|
|
1042
|
+
workerResult,
|
|
1043
|
+
evidence: buildTaskCompletionEvidence({
|
|
1044
|
+
event: 'agent:ready',
|
|
1045
|
+
nodeId,
|
|
1046
|
+
sessionId,
|
|
1047
|
+
providerType: providerType || undefined,
|
|
1048
|
+
providerSessionId,
|
|
1049
|
+
finalSummary,
|
|
1050
|
+
workerResult,
|
|
1051
|
+
}),
|
|
1052
|
+
},
|
|
1053
|
+
});
|
|
1054
|
+
} catch (e: any) {
|
|
1055
|
+
LOG.warn('MeshLedger', `Failed to record task_completed from ready: ${e?.message || e}`);
|
|
1056
|
+
}
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
if (sessionId && nodeId && providerType) {
|
|
1061
|
+
sweepExpiredRemoteIdleSessions();
|
|
1062
|
+
try {
|
|
1063
|
+
MeshRuntimeStore.getInstance().setRemoteIdleSession(nodeId, sessionId, providerType, Date.now() + REMOTE_IDLE_SESSION_TTL_MS);
|
|
1064
|
+
} catch { /* best-effort */ }
|
|
1065
|
+
setImmediate(() => {
|
|
1066
|
+
maybeAutoFastForwardIdleNode(components, { meshId: args.meshId, nodeId, sessionId, providerType })
|
|
1067
|
+
.finally(() => {
|
|
1068
|
+
try {
|
|
1069
|
+
const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
|
|
1070
|
+
if (assigned) MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
1071
|
+
} catch (e: any) {
|
|
1072
|
+
LOG.warn('MeshQueue', `Failed to assign idle queue task after maintenance for ${nodeId}: ${e?.message || e}`);
|
|
1073
|
+
}
|
|
1074
|
+
});
|
|
1075
|
+
});
|
|
1076
|
+
}
|
|
1077
|
+
} else if (args.event === 'agent:generating_started') {
|
|
1078
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
1079
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1080
|
+
if (sessionId && nodeId) {
|
|
1081
|
+
try {
|
|
1082
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
1083
|
+
} catch { /* best-effort */ }
|
|
1084
|
+
}
|
|
1085
|
+
if (sessionId) {
|
|
1086
|
+
updateDirectDispatchStatus(args.meshId, sessionId, 'acked');
|
|
1087
|
+
const activeDeliveries = ((): { id: string }[] => {
|
|
1088
|
+
try { return MeshRuntimeStore.getInstance().getActiveSessionDeliveries(args.meshId, sessionId); }
|
|
1089
|
+
catch { return []; }
|
|
1090
|
+
})();
|
|
1091
|
+
for (const d of activeDeliveries) {
|
|
1092
|
+
updateSessionDeliveryStatus(d.id, 'acked');
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
} else if (args.event === 'agent:stopped') {
|
|
1096
|
+
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
1097
|
+
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1098
|
+
if (sessionId && nodeId) {
|
|
1099
|
+
try {
|
|
1100
|
+
MeshRuntimeStore.getInstance().deleteRemoteIdleSession(nodeId, sessionId);
|
|
1101
|
+
} catch { /* best-effort */ }
|
|
1102
|
+
}
|
|
1103
|
+
if (sessionId) {
|
|
1104
|
+
completedTaskForLedger = markSessionTerminal(sessionId, 'failed');
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1108
|
+
const ledgerKind = EVENT_TO_LEDGER_KIND[args.event];
|
|
1109
|
+
if (ledgerKind) {
|
|
1110
|
+
try {
|
|
1111
|
+
const ledgerNodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined;
|
|
1112
|
+
const ledgerSessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined;
|
|
1113
|
+
const ledgerProviderType = readNonEmptyString(args.metadataEvent.providerType) || undefined;
|
|
1114
|
+
const providerSessionId = readNonEmptyString(args.metadataEvent.providerSessionId) || undefined;
|
|
1115
|
+
const finalSummary = readNonEmptyString(args.metadataEvent.finalSummary) || undefined;
|
|
1116
|
+
const workerResult = readWorkerResultMetadata(args.metadataEvent);
|
|
1117
|
+
const completionEvidence = ledgerKind === 'task_completed' && ledgerNodeId && ledgerSessionId
|
|
1118
|
+
? buildTaskCompletionEvidence({
|
|
1119
|
+
event: 'agent:generating_completed',
|
|
1120
|
+
nodeId: ledgerNodeId,
|
|
1121
|
+
sessionId: ledgerSessionId,
|
|
1122
|
+
providerType: ledgerProviderType,
|
|
1123
|
+
providerSessionId,
|
|
1124
|
+
finalSummary,
|
|
1125
|
+
workerResult,
|
|
1126
|
+
})
|
|
1127
|
+
: undefined;
|
|
1128
|
+
appendLedgerEntry(args.meshId, {
|
|
1129
|
+
kind: ledgerKind,
|
|
1130
|
+
nodeId: ledgerNodeId,
|
|
1131
|
+
sessionId: ledgerSessionId,
|
|
1132
|
+
providerType: ledgerProviderType,
|
|
1133
|
+
payload: {
|
|
1134
|
+
event: args.event,
|
|
1135
|
+
nodeLabel: args.nodeLabel,
|
|
1136
|
+
taskId: completedTaskForLedger?.id || undefined,
|
|
1137
|
+
providerSessionId,
|
|
1138
|
+
finalSummary,
|
|
1139
|
+
workerResult,
|
|
1140
|
+
completionDiagnostic: args.metadataEvent.completionDiagnostic && typeof args.metadataEvent.completionDiagnostic === 'object'
|
|
1141
|
+
? args.metadataEvent.completionDiagnostic
|
|
1142
|
+
: undefined,
|
|
1143
|
+
evidence: completionEvidence,
|
|
1144
|
+
// B2: evidenceLevel lets coordinator know when completion evidence is insufficient.
|
|
1145
|
+
...(completionEvidence
|
|
1146
|
+
? completionEvidence.workerResult.source === 'default'
|
|
1147
|
+
? { evidenceLevel: 'insufficient', reviewRecommended: true }
|
|
1148
|
+
: { evidenceLevel: 'sufficient' }
|
|
1149
|
+
: {}),
|
|
1150
|
+
},
|
|
1151
|
+
});
|
|
1152
|
+
} catch (e: any) {
|
|
1153
|
+
LOG.warn('MeshLedger', `Failed to record ${ledgerKind}: ${e?.message || e}`);
|
|
1154
|
+
}
|
|
1155
|
+
}
|
|
1156
|
+
|
|
1157
|
+
let recoveryContext: SessionRecoveryContext | null = null;
|
|
1158
|
+
if (args.event === 'agent:stopped') {
|
|
1159
|
+
try {
|
|
1160
|
+
const mesh = getMesh(args.meshId);
|
|
1161
|
+
const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
|
|
1162
|
+
|
|
1163
|
+
recoveryContext = getSessionRecoveryContext(args.meshId, {
|
|
1164
|
+
sessionId: resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined,
|
|
1165
|
+
nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
|
|
1166
|
+
maxRetries,
|
|
1167
|
+
});
|
|
1168
|
+
recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
|
|
1169
|
+
|
|
1170
|
+
if (recoveryContext.retryRecommended && recoveryContext.consecutiveNodeFailures > 0) {
|
|
1171
|
+
appendLedgerEntry(args.meshId, {
|
|
1172
|
+
kind: 'recovery_attempted',
|
|
1173
|
+
nodeId: recoveryContext.failedNodeId || undefined,
|
|
1174
|
+
sessionId: recoveryContext.failedSessionId || undefined,
|
|
1175
|
+
providerType: recoveryContext.failedProviderType || undefined,
|
|
1176
|
+
payload: {
|
|
1177
|
+
consecutiveFailures: recoveryContext.consecutiveNodeFailures,
|
|
1178
|
+
taskAttemptCount: recoveryContext.taskAttemptCount,
|
|
1179
|
+
retryRecommended: recoveryContext.retryRecommended,
|
|
1180
|
+
advice: recoveryContext.advice,
|
|
1181
|
+
},
|
|
1182
|
+
});
|
|
1183
|
+
|
|
1184
|
+
if (recoveryContext.lastTaskMessage && recoveryContext.failedNodeId && recoveryContext.failedProviderType) {
|
|
1185
|
+
const autoNodeId = recoveryContext.failedNodeId;
|
|
1186
|
+
try {
|
|
1187
|
+
const task = enqueueTask(args.meshId, recoveryContext.lastTaskMessage, {
|
|
1188
|
+
targetNodeId: autoNodeId
|
|
1189
|
+
});
|
|
1190
|
+
LOG.info('MeshRecovery', `Auto-requeued failed task: ${task.id} for node ${autoNodeId}`);
|
|
1191
|
+
|
|
1192
|
+
const node = mesh?.nodes.find((n: any) => n.id === autoNodeId);
|
|
1193
|
+
if (node) {
|
|
1194
|
+
components.cliManager.handleCliCommand('launch_cli', {
|
|
1195
|
+
cliType: recoveryContext.failedProviderType,
|
|
1196
|
+
dir: node.workspace,
|
|
1197
|
+
settings: {
|
|
1198
|
+
role: 'worker',
|
|
1199
|
+
meshNodeFor: args.meshId,
|
|
1200
|
+
meshNodeId: node.id,
|
|
1201
|
+
spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
|
|
1202
|
+
launchedByCoordinator: true,
|
|
1203
|
+
}
|
|
1204
|
+
}).catch((e: any) => LOG.error('MeshRecovery', `Failed to auto-relaunch session for ${node.id}: ${e?.message}`));
|
|
1205
|
+
}
|
|
1206
|
+
} catch (e: any) {
|
|
1207
|
+
LOG.warn('MeshRecovery', `Failed to execute auto-recovery: ${e?.message}`);
|
|
1208
|
+
}
|
|
1209
|
+
}
|
|
1210
|
+
}
|
|
1211
|
+
|
|
1212
|
+
LOG.info('MeshRecovery', `Recovery context for ${args.nodeLabel}: ${recoveryContext.advice}`);
|
|
1213
|
+
} catch (e: any) {
|
|
1214
|
+
LOG.warn('MeshRecovery', `Failed to build recovery context: ${e?.message || e}`);
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
const messageText = buildMeshSystemMessage({
|
|
1219
|
+
event: args.event,
|
|
1220
|
+
nodeLabel: args.nodeLabel,
|
|
1221
|
+
metadataEvent: args.metadataEvent,
|
|
1222
|
+
recoveryContext,
|
|
1223
|
+
});
|
|
1224
|
+
if (!messageText) return { success: false, error: 'unsupported mesh event' };
|
|
1225
|
+
|
|
1226
|
+
const coordinatorInstances = components.instanceManager.getByCategory('cli').filter((inst) => {
|
|
1227
|
+
const instState = inst.getState();
|
|
1228
|
+
if (instState.settings?.meshCoordinatorFor !== args.meshId) return false;
|
|
1229
|
+
if (args.sourceInstanceId && instState.instanceId === args.sourceInstanceId) return false;
|
|
1230
|
+
if (workerCoordinatorDaemonId && localDaemonId && workerCoordinatorDaemonId !== localDaemonId) return false;
|
|
1231
|
+
return true;
|
|
1232
|
+
});
|
|
1233
|
+
|
|
1234
|
+
const isRefineTerminalEvent = new Set(['refine:completed', 'refine:failed']).has(args.event);
|
|
1235
|
+
|
|
1236
|
+
if (coordinatorInstances.length === 0) {
|
|
1237
|
+
if (queuePendingMeshCoordinatorEvent({
|
|
1238
|
+
event: args.event,
|
|
1239
|
+
meshId: args.meshId,
|
|
1240
|
+
nodeLabel: args.nodeLabel,
|
|
1241
|
+
nodeId: args.nodeId || undefined,
|
|
1242
|
+
workspace: readNonEmptyString(args.metadataEvent.workspace),
|
|
1243
|
+
metadataEvent: {
|
|
1244
|
+
...args.metadataEvent,
|
|
1245
|
+
...(recoveryContext ? { recoveryContext } : {}),
|
|
1246
|
+
},
|
|
1247
|
+
coordinatorMessage: messageText,
|
|
1248
|
+
queuedAt: Date.now(),
|
|
1249
|
+
...(workerCoordinatorDaemonId ? { targetCoordinatorDaemonId: workerCoordinatorDaemonId } : {}),
|
|
1250
|
+
})) {
|
|
1251
|
+
LOG.info('MeshEvents', `Queued ${args.event} for MCP coordinator (mesh ${args.meshId}${workerCoordinatorDaemonId ? `, coordinator daemon ${workerCoordinatorDaemonId}` : ''})`);
|
|
1252
|
+
}
|
|
1253
|
+
return { success: true, forwarded: 0 };
|
|
1254
|
+
}
|
|
1255
|
+
|
|
1256
|
+
const allCoordinatorsGenerating = isRefineTerminalEvent && coordinatorInstances.every((inst) => {
|
|
1257
|
+
const s = inst.getState();
|
|
1258
|
+
const status = readNonEmptyString(s.status).toLowerCase();
|
|
1259
|
+
const activeChatStatus = readNonEmptyString(s.activeChat?.status).toLowerCase();
|
|
1260
|
+
return status === 'generating' || status === 'streaming' || status === 'long_generating'
|
|
1261
|
+
|| activeChatStatus === 'generating' || activeChatStatus === 'streaming';
|
|
1262
|
+
});
|
|
1263
|
+
|
|
1264
|
+
if (!isRefineTerminalEvent || allCoordinatorsGenerating) {
|
|
1265
|
+
if (queuePendingMeshCoordinatorEvent({
|
|
1266
|
+
event: args.event,
|
|
1267
|
+
meshId: args.meshId,
|
|
1268
|
+
nodeLabel: args.nodeLabel,
|
|
1269
|
+
nodeId: args.nodeId || undefined,
|
|
1270
|
+
workspace: readNonEmptyString(args.metadataEvent.workspace),
|
|
1271
|
+
metadataEvent: {
|
|
1272
|
+
...args.metadataEvent,
|
|
1273
|
+
...(recoveryContext ? { recoveryContext } : {}),
|
|
1274
|
+
},
|
|
1275
|
+
coordinatorMessage: messageText,
|
|
1276
|
+
queuedAt: Date.now(),
|
|
1277
|
+
...(workerCoordinatorDaemonId ? { targetCoordinatorDaemonId: workerCoordinatorDaemonId } : {}),
|
|
1278
|
+
})) {
|
|
1279
|
+
if (allCoordinatorsGenerating) {
|
|
1280
|
+
LOG.info('MeshEvents', `Queued ${args.event} for generating CLI coordinator (mesh ${args.meshId}) — will be delivered via get_pending_mesh_events when coordinator returns to idle`);
|
|
1281
|
+
} else {
|
|
1282
|
+
LOG.info('MeshEvents', `Queued ${args.event} for MCP coordinator (mesh ${args.meshId})`);
|
|
1283
|
+
}
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1287
|
+
if (allCoordinatorsGenerating) {
|
|
1288
|
+
return { success: true, forwarded: 0, bufferedForGeneratingCoordinator: true };
|
|
1289
|
+
}
|
|
1290
|
+
|
|
1291
|
+
for (const coord of coordinatorInstances) {
|
|
1292
|
+
const coordState = coord.getState();
|
|
1293
|
+
LOG.info('MeshEvents', `Forwarding mesh event to coordinator ${coordState.instanceId}`);
|
|
1294
|
+
coord.onEvent('send_message', { input: { text: messageText, textFallback: messageText } });
|
|
1295
|
+
}
|
|
1296
|
+
return { success: true, forwarded: coordinatorInstances.length };
|
|
1297
|
+
}
|
|
1298
|
+
|
|
1299
|
+
export function handleMeshForwardEvent(components: DaemonComponents, payload: Record<string, unknown>) {
|
|
1300
|
+
const eventName = readNonEmptyString(payload.event);
|
|
1301
|
+
if (!isMeshCoordinatorEvent(eventName)) {
|
|
1302
|
+
return { success: false, error: 'unsupported mesh event' };
|
|
1303
|
+
}
|
|
1304
|
+
const meshId = readNonEmptyString(payload.meshId);
|
|
1305
|
+
if (!meshId) return { success: false, error: 'meshId required' };
|
|
1306
|
+
|
|
1307
|
+
const nodeId = readNonEmptyString(payload.nodeId);
|
|
1308
|
+
const workspace = readNonEmptyString(payload.workspace);
|
|
1309
|
+
const nodeLabel = nodeId ? `Node '${nodeId}'` : workspace ? `Agent at ${workspace}` : 'Remote agent';
|
|
1310
|
+
const relayModalMessage = readNonEmptyString(payload.modalMessage);
|
|
1311
|
+
const relayModalButtons = Array.isArray(payload.modalButtons)
|
|
1312
|
+
? (payload.modalButtons as unknown[]).filter((b): b is string => typeof b === 'string' && b.trim().length > 0)
|
|
1313
|
+
: null;
|
|
1314
|
+
|
|
1315
|
+
return injectMeshSystemMessage(components, {
|
|
1316
|
+
meshId,
|
|
1317
|
+
nodeId,
|
|
1318
|
+
nodeLabel,
|
|
1319
|
+
event: eventName,
|
|
1320
|
+
metadataEvent: {
|
|
1321
|
+
targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.instanceId),
|
|
1322
|
+
providerType: readNonEmptyString(payload.providerType),
|
|
1323
|
+
providerSessionId: readNonEmptyString(payload.providerSessionId),
|
|
1324
|
+
finalSummary: readNonEmptyString(payload.finalSummary) || readNonEmptyString(payload.summary),
|
|
1325
|
+
jobId: readNonEmptyString(payload.jobId),
|
|
1326
|
+
interactionId: readNonEmptyString(payload.interactionId),
|
|
1327
|
+
status: readNonEmptyString(payload.status),
|
|
1328
|
+
targetDaemonId: readNonEmptyString(payload.targetDaemonId),
|
|
1329
|
+
startedAt: readNonEmptyString(payload.startedAt),
|
|
1330
|
+
completedAt: readNonEmptyString(payload.completedAt),
|
|
1331
|
+
retryOfJobId: readNonEmptyString(payload.retryOfJobId),
|
|
1332
|
+
...(relayModalMessage ? { modalMessage: relayModalMessage } : {}),
|
|
1333
|
+
...(relayModalButtons && relayModalButtons.length > 0 ? { modalButtons: relayModalButtons } : {}),
|
|
1334
|
+
...(payload.result && typeof payload.result === 'object' && !Array.isArray(payload.result) ? { result: payload.result } : {}),
|
|
1335
|
+
...(payload.completionDiagnostic && typeof payload.completionDiagnostic === 'object' && !Array.isArray(payload.completionDiagnostic) ? { completionDiagnostic: payload.completionDiagnostic } : {}),
|
|
1336
|
+
...(payload.workerResult && typeof payload.workerResult === 'object' && !Array.isArray(payload.workerResult) ? { workerResult: payload.workerResult } : {}),
|
|
1337
|
+
...(payload.meshWorkerResult && typeof payload.meshWorkerResult === 'object' && !Array.isArray(payload.meshWorkerResult) ? { meshWorkerResult: payload.meshWorkerResult } : {}),
|
|
1338
|
+
...(payload.structuredResult && typeof payload.structuredResult === 'object' && !Array.isArray(payload.structuredResult) ? { structuredResult: payload.structuredResult } : {}),
|
|
1339
|
+
...(payload.timestamp !== undefined ? { timestamp: payload.timestamp } : {}),
|
|
1340
|
+
intentional: payload.intentional === true,
|
|
1341
|
+
intentionalStop: payload.intentionalStop === true,
|
|
1342
|
+
operatorCleanup: payload.operatorCleanup === true,
|
|
1343
|
+
reason: readNonEmptyString(payload.reason),
|
|
1344
|
+
stopReason: readNonEmptyString(payload.stopReason),
|
|
1345
|
+
cleanupReason: readNonEmptyString(payload.cleanupReason),
|
|
1346
|
+
source: readNonEmptyString(payload.source),
|
|
1347
|
+
},
|
|
1348
|
+
});
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
1352
|
+
components.instanceManager.onEvent((event) => {
|
|
1353
|
+
if (!isMeshCoordinatorEvent(event.event)) return;
|
|
1354
|
+
|
|
1355
|
+
const instanceId = readNonEmptyString(event.instanceId);
|
|
1356
|
+
if (!instanceId) return;
|
|
1357
|
+
|
|
1358
|
+
const sourceInstance = components.instanceManager.getInstance(instanceId);
|
|
1359
|
+
if (!sourceInstance || sourceInstance.category !== 'cli') return;
|
|
1360
|
+
const state = sourceInstance.getState();
|
|
1361
|
+
const workspace = readNonEmptyString(state.workspace);
|
|
1362
|
+
if (!workspace) return;
|
|
1363
|
+
const settings = state.settings && typeof state.settings === 'object' ? state.settings as Record<string, unknown> : {};
|
|
1364
|
+
|
|
1365
|
+
const coordinatorMeshId = readNonEmptyString(settings.meshCoordinatorFor);
|
|
1366
|
+
let meshIdFromDirectDispatch = '';
|
|
1367
|
+
if (coordinatorMeshId) {
|
|
1368
|
+
try {
|
|
1369
|
+
const hasActiveDispatch =
|
|
1370
|
+
getActiveDirectDispatches(coordinatorMeshId).some(d => d.sessionId === instanceId)
|
|
1371
|
+
|| hasUnterminalDirectDispatchLedgerEntry(coordinatorMeshId, instanceId);
|
|
1372
|
+
if (hasActiveDispatch) meshIdFromDirectDispatch = coordinatorMeshId;
|
|
1373
|
+
} catch { /* best-effort */ }
|
|
1374
|
+
if (!meshIdFromDirectDispatch) return;
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
const meshIdFromRuntime = readNonEmptyString(settings.meshNodeFor) || meshIdFromDirectDispatch;
|
|
1378
|
+
|
|
1379
|
+
const isMeshDelegate = Boolean(meshIdFromRuntime || settings.launchedByCoordinator);
|
|
1380
|
+
if (!isMeshDelegate) return;
|
|
1381
|
+
|
|
1382
|
+
const mesh = meshIdFromRuntime ? getMeshWithCache(components, meshIdFromRuntime) : getCachedMeshByWorkspace(workspace);
|
|
1383
|
+
const meshId = meshIdFromRuntime || readNonEmptyString(mesh?.id);
|
|
1384
|
+
if (!meshId) return;
|
|
1385
|
+
|
|
1386
|
+
const targetNode = mesh?.nodes?.find((n: any) => n.workspace === workspace);
|
|
1387
|
+
const runtimeNodeId = readNonEmptyString(settings.meshNodeId);
|
|
1388
|
+
const resolvedNodeId = targetNode?.id || runtimeNodeId;
|
|
1389
|
+
const nodeLabel = targetNode
|
|
1390
|
+
? `Node '${targetNode.id}'`
|
|
1391
|
+
: runtimeNodeId
|
|
1392
|
+
? `Node '${runtimeNodeId}'`
|
|
1393
|
+
: `Agent at ${workspace}`;
|
|
1394
|
+
|
|
1395
|
+
injectMeshSystemMessage(components, {
|
|
1396
|
+
meshId,
|
|
1397
|
+
sourceInstanceId: instanceId,
|
|
1398
|
+
nodeId: resolvedNodeId,
|
|
1399
|
+
nodeLabel,
|
|
1400
|
+
event: event.event,
|
|
1401
|
+
metadataEvent: event,
|
|
1402
|
+
});
|
|
1403
|
+
});
|
|
1404
|
+
}
|