@adhdev/daemon-core 0.9.82-rc.256 → 0.9.82-rc.258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/boot/daemon-lifecycle.d.ts +3 -0
- package/dist/index.js +287 -43
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +287 -43
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-events-coordinator.d.ts +22 -0
- package/dist/mesh/mesh-events-pending.d.ts +7 -0
- package/dist/mesh/mesh-events-utils.d.ts +2 -0
- package/dist/mesh/mesh-events.d.ts +2 -2
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/mesh/mesh-routing.d.ts +70 -0
- package/dist/mesh/mesh-runtime-store.d.ts +3 -0
- package/package.json +1 -1
- package/src/boot/daemon-lifecycle.ts +8 -0
- package/src/commands/router.ts +12 -4
- package/src/mesh/mesh-events-coordinator.ts +110 -54
- package/src/mesh/mesh-events-pending.ts +57 -3
- package/src/mesh/mesh-events-utils.ts +20 -0
- package/src/mesh/mesh-events.ts +2 -0
- package/src/mesh/mesh-ledger.ts +1 -0
- package/src/mesh/mesh-routing.ts +272 -0
- package/src/mesh/mesh-runtime-store.ts +37 -0
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
2
|
export declare function __resetIdleAutoFastForwardForTests(): void;
|
|
3
|
+
export declare function __resetMeshWorkspaceCacheForTests(): void;
|
|
3
4
|
export declare function tryAssignQueueTask(components: DaemonComponents, meshId: string, nodeId: string, sessionId: string, providerType: string): boolean;
|
|
4
5
|
export interface MeshQueueTriggerResult {
|
|
5
6
|
success: true;
|
|
@@ -39,6 +40,7 @@ export declare function handleMeshForwardEvent(components: DaemonComponents, pay
|
|
|
39
40
|
duplicateCompletion?: undefined;
|
|
40
41
|
duplicateStopped?: undefined;
|
|
41
42
|
error?: undefined;
|
|
43
|
+
remoteForwarded?: undefined;
|
|
42
44
|
} | {
|
|
43
45
|
success: boolean;
|
|
44
46
|
forwarded: number;
|
|
@@ -51,6 +53,7 @@ export declare function handleMeshForwardEvent(components: DaemonComponents, pay
|
|
|
51
53
|
duplicateCompletion?: undefined;
|
|
52
54
|
duplicateStopped?: undefined;
|
|
53
55
|
error?: undefined;
|
|
56
|
+
remoteForwarded?: undefined;
|
|
54
57
|
} | {
|
|
55
58
|
success: boolean;
|
|
56
59
|
forwarded: number;
|
|
@@ -63,6 +66,7 @@ export declare function handleMeshForwardEvent(components: DaemonComponents, pay
|
|
|
63
66
|
duplicateCompletion?: undefined;
|
|
64
67
|
duplicateStopped?: undefined;
|
|
65
68
|
error?: undefined;
|
|
69
|
+
remoteForwarded?: undefined;
|
|
66
70
|
} | {
|
|
67
71
|
success: boolean;
|
|
68
72
|
forwarded: number;
|
|
@@ -75,6 +79,7 @@ export declare function handleMeshForwardEvent(components: DaemonComponents, pay
|
|
|
75
79
|
duplicateCompletion?: undefined;
|
|
76
80
|
duplicateStopped?: undefined;
|
|
77
81
|
error?: undefined;
|
|
82
|
+
remoteForwarded?: undefined;
|
|
78
83
|
} | {
|
|
79
84
|
success: boolean;
|
|
80
85
|
forwarded: number;
|
|
@@ -87,6 +92,7 @@ export declare function handleMeshForwardEvent(components: DaemonComponents, pay
|
|
|
87
92
|
duplicateApproval?: undefined;
|
|
88
93
|
duplicateStopped?: undefined;
|
|
89
94
|
error?: undefined;
|
|
95
|
+
remoteForwarded?: undefined;
|
|
90
96
|
} | {
|
|
91
97
|
success: boolean;
|
|
92
98
|
forwarded: number;
|
|
@@ -99,6 +105,7 @@ export declare function handleMeshForwardEvent(components: DaemonComponents, pay
|
|
|
99
105
|
duplicateApproval?: undefined;
|
|
100
106
|
duplicateStopped?: undefined;
|
|
101
107
|
error?: undefined;
|
|
108
|
+
remoteForwarded?: undefined;
|
|
102
109
|
} | {
|
|
103
110
|
success: boolean;
|
|
104
111
|
forwarded: number;
|
|
@@ -111,9 +118,11 @@ export declare function handleMeshForwardEvent(components: DaemonComponents, pay
|
|
|
111
118
|
duplicateApproval?: undefined;
|
|
112
119
|
duplicateCompletion?: undefined;
|
|
113
120
|
error?: undefined;
|
|
121
|
+
remoteForwarded?: undefined;
|
|
114
122
|
} | {
|
|
115
123
|
success: boolean;
|
|
116
124
|
forwarded: number;
|
|
125
|
+
remoteForwarded: boolean;
|
|
117
126
|
suppressed?: undefined;
|
|
118
127
|
intentionalCleanupStop?: undefined;
|
|
119
128
|
terminalLedgerEvidence?: undefined;
|
|
@@ -123,6 +132,19 @@ export declare function handleMeshForwardEvent(components: DaemonComponents, pay
|
|
|
123
132
|
duplicateCompletion?: undefined;
|
|
124
133
|
duplicateStopped?: undefined;
|
|
125
134
|
error?: undefined;
|
|
135
|
+
} | {
|
|
136
|
+
success: boolean;
|
|
137
|
+
forwarded: number;
|
|
138
|
+
suppressed?: undefined;
|
|
139
|
+
intentionalCleanupStop?: undefined;
|
|
140
|
+
terminalLedgerEvidence?: undefined;
|
|
141
|
+
terminalLedgerKind?: undefined;
|
|
142
|
+
duplicateRefineTerminalEvent?: undefined;
|
|
143
|
+
duplicateApproval?: undefined;
|
|
144
|
+
duplicateCompletion?: undefined;
|
|
145
|
+
duplicateStopped?: undefined;
|
|
146
|
+
error?: undefined;
|
|
147
|
+
remoteForwarded?: undefined;
|
|
126
148
|
} | {
|
|
127
149
|
success: boolean;
|
|
128
150
|
error: string;
|
|
@@ -18,6 +18,13 @@ export declare function readRefineJobId(event: {
|
|
|
18
18
|
metadataEvent?: Record<string, unknown>;
|
|
19
19
|
} | Record<string, unknown>): string;
|
|
20
20
|
export declare function buildPendingEventFingerprint(event: PendingMeshCoordinatorEvent): string;
|
|
21
|
+
/**
|
|
22
|
+
* R3: record that an event was direct-injected into a live coordinator on `coordinatorDaemonId`.
|
|
23
|
+
* That coordinator's own drain (get_pending_mesh_events with the same coordinatorDaemonId) will
|
|
24
|
+
* skip the queued copy, so it receives the event exactly once instead of twice (PTY + poll).
|
|
25
|
+
* Other consumers (unscoped drainers, other daemons) are unaffected — they did not get the inject.
|
|
26
|
+
*/
|
|
27
|
+
export declare function markMeshCoordinatorEventDirectDelivered(coordinatorDaemonId: string, event: PendingMeshCoordinatorEvent): void;
|
|
21
28
|
export declare function hasPendingCoordinatorEventDuplicate(event: PendingMeshCoordinatorEvent): boolean;
|
|
22
29
|
export declare function queuePendingMeshCoordinatorEvent(event: PendingMeshCoordinatorEvent): boolean;
|
|
23
30
|
/** Drain and return all pending coordinator events for meshId, removing them from disk. */
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import type { SessionRecoveryContext } from './mesh-ledger.js';
|
|
2
2
|
export declare function readNonEmptyString(value: unknown): string;
|
|
3
3
|
export declare function readRecord(value: unknown): Record<string, unknown> | undefined;
|
|
4
|
+
export declare function canonicalDaemonId(value: unknown): string;
|
|
5
|
+
export declare function sameDaemonId(a: unknown, b: unknown): boolean;
|
|
4
6
|
export declare function resolveEventSessionId(event: Record<string, unknown>, fallback?: unknown): string;
|
|
5
7
|
export declare function readRefineJobId(event: {
|
|
6
8
|
metadataEvent?: Record<string, unknown>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
2
|
-
export { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, clearPendingMeshCoordinatorEvents, } from './mesh-events-pending.js';
|
|
2
|
+
export { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, clearPendingMeshCoordinatorEvents, markMeshCoordinatorEventDirectDelivered, } from './mesh-events-pending.js';
|
|
3
3
|
export { reconcileDirectDispatchCompletionFromTranscript, } from './mesh-events-stale.js';
|
|
4
4
|
export type { MeshQueueTriggerResult } from './mesh-events-coordinator.js';
|
|
5
|
-
export { tryAssignQueueTask, triggerMeshQueue, handleMeshForwardEvent, setupMeshEventForwarding, isMeshCoordinatorEvent, __resetIdleAutoFastForwardForTests, } from './mesh-events-coordinator.js';
|
|
5
|
+
export { tryAssignQueueTask, triggerMeshQueue, handleMeshForwardEvent, setupMeshEventForwarding, isMeshCoordinatorEvent, __resetIdleAutoFastForwardForTests, __resetMeshWorkspaceCacheForTests, } from './mesh-events-coordinator.js';
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { EventEmitter } from 'events';
|
|
16
16
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
17
|
-
export type MeshLedgerKind = 'task_dispatched' | 'task_completed' | 'task_failed' | 'task_stalled' | 'task_approval_needed' | 'p2p_dispatch_failed' | 'session_launched' | 'session_auto_launch' | 'session_stopped' | 'checkpoint_created' | 'node_cloned' | 'node_joined' | 'node_removed' | 'coordinator_started' | 'recovery_attempted' | 'ledger_replicated' | 'ledger_reconciled' | 'direct_fast_forward';
|
|
17
|
+
export type MeshLedgerKind = 'task_dispatched' | 'task_completed' | 'task_failed' | 'task_stalled' | 'task_approval_needed' | 'p2p_dispatch_failed' | 'session_launched' | 'session_auto_launch' | 'session_stopped' | 'checkpoint_created' | 'node_cloned' | 'node_joined' | 'node_removed' | 'coordinator_started' | 'recovery_attempted' | 'ledger_replicated' | 'ledger_reconciled' | 'direct_fast_forward' | 'delivery_unroutable';
|
|
18
18
|
export interface MeshLedgerEntry {
|
|
19
19
|
id: string;
|
|
20
20
|
meshId: string;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
|
|
2
|
+
export type WorkerDelegateRejectionReason = 'not_cli' | 'no_workspace' | 'no_worker_envelope' | 'coordinator_not_dispatch_target' | 'mesh_unresolved';
|
|
3
|
+
export interface WorkerDelegateRouting {
|
|
4
|
+
/** True when the source session is a mesh worker whose events must route to a coordinator. */
|
|
5
|
+
isDelegate: boolean;
|
|
6
|
+
/** Resolved mesh id (runtime stamp, direct-dispatch recovery, or workspace lookup). */
|
|
7
|
+
meshId: string;
|
|
8
|
+
/** Resolved node id within the mesh, or '' when it cannot be determined. */
|
|
9
|
+
nodeId: string;
|
|
10
|
+
/** Human-readable node label for the coordinator system message. */
|
|
11
|
+
nodeLabel: string;
|
|
12
|
+
/** Coordinator daemon anchor (prefixed id) when the worker carries one; '' otherwise. */
|
|
13
|
+
coordinatorDaemonId: string;
|
|
14
|
+
/** Source session workspace (best-effort; '' when the session/workspace is missing). */
|
|
15
|
+
workspace: string;
|
|
16
|
+
/** Source session id echoed back, so diagnostics can name the dropped event's origin. */
|
|
17
|
+
sessionId: string;
|
|
18
|
+
/** Why routing was rejected (isDelegate=false), for fail-loud diagnostics (R4). */
|
|
19
|
+
rejectionReason?: WorkerDelegateRejectionReason;
|
|
20
|
+
}
|
|
21
|
+
interface ResolveDeps {
|
|
22
|
+
/** Resolve a mesh by id (local config + inline cache). */
|
|
23
|
+
getMeshById: (meshId: string) => any | undefined;
|
|
24
|
+
/** Resolve a mesh by workspace path (cached repo lookup). */
|
|
25
|
+
getMeshByWorkspace: (workspace: string) => any | undefined;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Resolve how a source CLI session's mesh events should route to a coordinator.
|
|
29
|
+
*
|
|
30
|
+
* This is the single authoritative interpretation of the worker envelope. It folds
|
|
31
|
+
* together every signal that previously lived in inline branches:
|
|
32
|
+
* - runtime stamps (meshNodeFor / meshNodeId)
|
|
33
|
+
* - the coordinator anchor (meshCoordinatorDaemonId) and launch marker (launchedByCoordinator,
|
|
34
|
+
* meshCoordinatorNodeId) — any one proves delegation even if the node/mesh stamp was dropped
|
|
35
|
+
* - direct-dispatch recovery: a coordinator session that is itself a dispatch target
|
|
36
|
+
* - workspace→mesh fallback when no runtime mesh id survived
|
|
37
|
+
*
|
|
38
|
+
* Returns isDelegate=false (with a rejectionReason) instead of throwing, so the caller
|
|
39
|
+
* can either ignore non-delegate events or surface a diagnostic.
|
|
40
|
+
*/
|
|
41
|
+
export declare function resolveWorkerDelegateRouting(components: DaemonComponents, instanceId: string, deps: ResolveDeps): WorkerDelegateRouting;
|
|
42
|
+
/** Ledger stream that collects delivery_unroutable diagnostics (no real mesh id exists for them). */
|
|
43
|
+
export declare const UNROUTABLE_DIAGNOSTIC_STREAM = "__unroutable__";
|
|
44
|
+
export declare function __resetUnroutableDiagnosticsForTests(): void;
|
|
45
|
+
/** True only for rejections that represent a silently-dropped delegate event worth a diagnostic. */
|
|
46
|
+
export declare function isUnroutableDelegateRejection(routing: WorkerDelegateRouting): boolean;
|
|
47
|
+
/**
|
|
48
|
+
* R4: record that a worker event with a valid envelope could not be routed to a coordinator.
|
|
49
|
+
* Writes a `delivery_unroutable` ledger entry (deduped within a short window) so operators see
|
|
50
|
+
* WHY a completion never arrived instead of it vanishing. No-op for non-diagnostic rejections.
|
|
51
|
+
*/
|
|
52
|
+
export declare function recordUnroutableDelegateEvent(routing: WorkerDelegateRouting, eventName: string): boolean;
|
|
53
|
+
export interface UnroutableDeliveryDiagnostic {
|
|
54
|
+
timestamp: string;
|
|
55
|
+
event: string;
|
|
56
|
+
sessionId?: string;
|
|
57
|
+
workspace?: string;
|
|
58
|
+
coordinatorDaemonId?: string;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* R4 visibility: recent delivery_unroutable diagnostics, newest first. Surfaced in mesh_status
|
|
62
|
+
* so an operator/coordinator can see that completions were dropped (envelope present, mesh
|
|
63
|
+
* unresolved) rather than the drops being invisible in a ledger nobody reads. The diagnostics
|
|
64
|
+
* live in a single shared stream because an unroutable event has no resolvable mesh.
|
|
65
|
+
*/
|
|
66
|
+
export declare function getRecentUnroutableDeliveries(opts?: {
|
|
67
|
+
sinceMs?: number;
|
|
68
|
+
limit?: number;
|
|
69
|
+
}): UnroutableDeliveryDiagnostic[];
|
|
70
|
+
export {};
|
|
@@ -17,6 +17,9 @@ export declare class MeshRuntimeStore {
|
|
|
17
17
|
hasCompletionFingerprint(fingerprint: string): boolean;
|
|
18
18
|
recordCompletionFingerprint(fingerprint: string, ttlMs: number): void;
|
|
19
19
|
sweepExpiredFingerprints(): void;
|
|
20
|
+
recordDirectDelivered(coordinatorDaemonId: string, fingerprint: string, ttlMs: number): void;
|
|
21
|
+
wasDirectDelivered(coordinatorDaemonId: string, fingerprint: string): boolean;
|
|
22
|
+
sweepExpiredDirectDelivered(): void;
|
|
20
23
|
private maybeCheckpointWal;
|
|
21
24
|
private ensureLegacyQueueMigrated;
|
|
22
25
|
getQueueEntries(meshId: string, statuses?: MeshTaskStatus[]): MeshWorkQueueEntry[];
|
package/package.json
CHANGED
|
@@ -92,6 +92,8 @@ export interface DaemonInitConfig {
|
|
|
92
92
|
dispatchMeshCommand?: (daemonId: string, command: string, args: Record<string, unknown>) => Promise<any>;
|
|
93
93
|
/** Returns selected-coordinator mesh peer telemetry for a target daemon when available. */
|
|
94
94
|
getMeshPeerConnectionStatus?: (daemonId: string) => Record<string, unknown> | null;
|
|
95
|
+
/** Cloud-only: P2P dashboard metadata sync after the core forwarder handles a mesh event. */
|
|
96
|
+
onMeshCoordinatorEventForwarded?: (payload: Record<string, unknown>) => void;
|
|
95
97
|
}
|
|
96
98
|
|
|
97
99
|
// ─── Result ───
|
|
@@ -110,6 +112,11 @@ export interface DaemonComponents {
|
|
|
110
112
|
detectedIdes: { value: IDEInfo[] };
|
|
111
113
|
refreshProviderAvailability: (providerType?: string) => Promise<void>;
|
|
112
114
|
dispatchMeshCommand?: (daemonId: string, command: string, args: Record<string, unknown>) => Promise<any>;
|
|
115
|
+
// Cloud-only hook: after the single core forwarder handles a mesh coordinator event, cloud
|
|
116
|
+
// uses this to keep its P2P dashboard view in sync (mesh-owned session metadata + flush
|
|
117
|
+
// subscriptions). Injected by daemon-cloud; absent/no-op on standalone. Replaces cloud's
|
|
118
|
+
// former separate instanceManager.onEvent listener so the event path stays single-listener.
|
|
119
|
+
onMeshCoordinatorEventForwarded?: (payload: Record<string, unknown>) => void;
|
|
113
120
|
}
|
|
114
121
|
|
|
115
122
|
export interface DaemonDevSupportOptions {
|
|
@@ -348,6 +355,7 @@ export async function initDaemonComponents(config: DaemonInitConfig): Promise<Da
|
|
|
348
355
|
detectedIdes: detectedIdesRef,
|
|
349
356
|
refreshProviderAvailability,
|
|
350
357
|
dispatchMeshCommand: config.dispatchMeshCommand,
|
|
358
|
+
onMeshCoordinatorEventForwarded: config.onMeshCoordinatorEventForwarded,
|
|
351
359
|
};
|
|
352
360
|
|
|
353
361
|
// 11. Setup Mesh Event Forwarding
|
package/src/commands/router.ts
CHANGED
|
@@ -40,6 +40,7 @@ import { createHermesManualMeshCoordinatorSetup, resolveMeshCoordinatorSetup } f
|
|
|
40
40
|
import { buildSessionEntries } from '../status/builders.js';
|
|
41
41
|
import { registerMeshCoordinator, getCoordinatorForSession } from '../mesh/coordinator-registry.js';
|
|
42
42
|
import { handleMeshForwardEvent, drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, queuePendingMeshCoordinatorEvent, type PendingMeshCoordinatorEvent } from '../mesh/mesh-events.js';
|
|
43
|
+
import { getRecentUnroutableDeliveries } from '../mesh/mesh-routing.js';
|
|
43
44
|
import { buildMeshHostRequiredFailure, normalizeMeshDaemonRole, resolveMeshHostStatus } from '../mesh/mesh-host-ownership.js';
|
|
44
45
|
import { fastForwardMeshNode } from '../mesh/mesh-fast-forward.js';
|
|
45
46
|
import { buildPreviewFreshness } from '../mesh/preview-freshness.js';
|
|
@@ -7364,6 +7365,10 @@ export class DaemonCommandRouter {
|
|
|
7364
7365
|
? args.coordinatorDaemonId.trim()
|
|
7365
7366
|
: (this.deps.statusInstanceId || undefined);
|
|
7366
7367
|
const pendingCoordinatorEvents = getPendingMeshCoordinatorEvents(meshId, callerCoordinatorDaemonId);
|
|
7368
|
+
// R4: surface recent fail-loud routing drops so a coordinator/operator can see
|
|
7369
|
+
// that a worker completion was lost (envelope present, mesh unresolved) instead
|
|
7370
|
+
// of it vanishing silently. Diagnostic-only — never cached (see omit below).
|
|
7371
|
+
const unroutableDeliveries = getRecentUnroutableDeliveries();
|
|
7367
7372
|
const previewFreshness = (() => {
|
|
7368
7373
|
const localRepoRoot = nodeStatuses
|
|
7369
7374
|
.map((node: any) => readStringValue(node?.git?.repoRoot, node?.repoRoot, node?.workspace))
|
|
@@ -7425,6 +7430,7 @@ export class DaemonCommandRouter {
|
|
|
7425
7430
|
...(asyncRefineJobs.length > 0 ? { asyncRefineJobs } : {}),
|
|
7426
7431
|
...(historicalSessions ? { historicalSessions } : {}),
|
|
7427
7432
|
...(pendingCoordinatorEvents.length > 0 ? { pendingCoordinatorEvents } : {}),
|
|
7433
|
+
...(unroutableDeliveries.length > 0 ? { unroutableDeliveries } : {}),
|
|
7428
7434
|
activeRefineJobs: Array.from(this.runningRefineJobs.values())
|
|
7429
7435
|
.filter(job => job.meshId === meshId)
|
|
7430
7436
|
.map(job => ({
|
|
@@ -7436,11 +7442,13 @@ export class DaemonCommandRouter {
|
|
|
7436
7442
|
targetCoordinatorDaemonId: job.targetCoordinatorDaemonId,
|
|
7437
7443
|
})),
|
|
7438
7444
|
};
|
|
7439
|
-
const { pendingCoordinatorEvents: _pendingCoordinatorEvents, ...cacheableStatusResult } = statusResult as any;
|
|
7445
|
+
const { pendingCoordinatorEvents: _pendingCoordinatorEvents, unroutableDeliveries: _unroutableDeliveries, ...cacheableStatusResult } = statusResult as any;
|
|
7440
7446
|
const rememberedStatus = this.rememberAggregateMeshStatus(meshId, cacheableStatusResult, refreshReason);
|
|
7441
|
-
const returnedStatus =
|
|
7442
|
-
|
|
7443
|
-
:
|
|
7447
|
+
const returnedStatus = {
|
|
7448
|
+
...rememberedStatus,
|
|
7449
|
+
...(pendingCoordinatorEvents.length > 0 ? { pendingCoordinatorEvents } : {}),
|
|
7450
|
+
...(unroutableDeliveries.length > 0 ? { unroutableDeliveries } : {}),
|
|
7451
|
+
};
|
|
7444
7452
|
logRepoMeshStatusDebug('return_live', {
|
|
7445
7453
|
meshId,
|
|
7446
7454
|
command: 'mesh_status',
|
|
@@ -10,8 +10,9 @@ import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask,
|
|
|
10
10
|
import { fastForwardMeshNode } from './mesh-fast-forward.js';
|
|
11
11
|
import { createSessionDelivery, markSessionDeliveriesTerminal, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
|
|
12
12
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
13
|
-
import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
13
|
+
import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents, markMeshCoordinatorEventDirectDelivered } from './mesh-events-pending.js';
|
|
14
14
|
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
15
|
+
import { resolveWorkerDelegateRouting, recordUnroutableDelegateEvent } from './mesh-routing.js';
|
|
15
16
|
import {
|
|
16
17
|
findRecentTerminalLedgerEvidence,
|
|
17
18
|
hasDispatchAfterTerminal,
|
|
@@ -25,6 +26,7 @@ import {
|
|
|
25
26
|
resolveEventSessionId,
|
|
26
27
|
readRefineJobId,
|
|
27
28
|
readWorkerResultMetadata,
|
|
29
|
+
sameDaemonId,
|
|
28
30
|
} from './mesh-events-utils.js';
|
|
29
31
|
|
|
30
32
|
// ---------------------------------------------------------------------------
|
|
@@ -53,6 +55,10 @@ export function __resetIdleAutoFastForwardForTests(): void {
|
|
|
53
55
|
idleAutoFastForwardLastAttempt.clear();
|
|
54
56
|
}
|
|
55
57
|
|
|
58
|
+
export function __resetMeshWorkspaceCacheForTests(): void {
|
|
59
|
+
meshByWorkspaceCache.clear();
|
|
60
|
+
}
|
|
61
|
+
|
|
56
62
|
function sweepExpiredRemoteIdleSessions(): void {
|
|
57
63
|
try {
|
|
58
64
|
MeshRuntimeStore.getInstance().pruneExpiredRemoteIdleSessions();
|
|
@@ -980,6 +986,22 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
980
986
|
(sourceSession?.getState()?.settings as Record<string, unknown>)?.meshCoordinatorDaemonId,
|
|
981
987
|
);
|
|
982
988
|
const localDaemonId = readNonEmptyString(loadConfig().machineId);
|
|
989
|
+
|
|
990
|
+
// R2: cloud P2P dashboard metadata sync. The cloud daemon used to do this from its own
|
|
991
|
+
// relay listener; now the single core forwarder invokes the injected hook (no-op on
|
|
992
|
+
// standalone) so the event path stays single-listener and the local code path is identical
|
|
993
|
+
// across standalone and cloud.
|
|
994
|
+
if (components.onMeshCoordinatorEventForwarded) {
|
|
995
|
+
try {
|
|
996
|
+
components.onMeshCoordinatorEventForwarded({
|
|
997
|
+
event: args.event,
|
|
998
|
+
meshId: args.meshId,
|
|
999
|
+
nodeId: eventNodeId || undefined,
|
|
1000
|
+
...args.metadataEvent,
|
|
1001
|
+
});
|
|
1002
|
+
} catch { /* dashboard metadata sync is best-effort */ }
|
|
1003
|
+
}
|
|
1004
|
+
|
|
983
1005
|
const intentionalCleanupStop = shouldSuppressIntentionalCleanupStop({
|
|
984
1006
|
event: args.event,
|
|
985
1007
|
meshId: args.meshId,
|
|
@@ -1349,11 +1371,52 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1349
1371
|
const instState = inst.getState();
|
|
1350
1372
|
if (instState.settings?.meshCoordinatorFor !== args.meshId) return false;
|
|
1351
1373
|
if (args.sourceInstanceId && instState.instanceId === args.sourceInstanceId) return false;
|
|
1352
|
-
|
|
1374
|
+
// canonicalize: the worker stamps the prefixed daemon id (standalone_/daemon_) while
|
|
1375
|
+
// localDaemonId is the raw machineId — a literal !== wrongly excludes the local coordinator.
|
|
1376
|
+
if (workerCoordinatorDaemonId && localDaemonId && !sameDaemonId(workerCoordinatorDaemonId, localDaemonId)) return false;
|
|
1353
1377
|
return true;
|
|
1354
1378
|
});
|
|
1355
1379
|
|
|
1356
1380
|
if (coordinatorInstances.length === 0) {
|
|
1381
|
+
// R2: the coordinator is not a live instance on THIS daemon. If it lives on a remote
|
|
1382
|
+
// daemon, forward over the injected transport (dispatchMeshCommand, supplied by cloud as
|
|
1383
|
+
// P2P; absent/no-op on standalone). This replaces cloud's separate
|
|
1384
|
+
// relayRemoteMeshCoordinatorEvent listener — remote delivery is now part of the single
|
|
1385
|
+
// core forwarder path, so local-vs-remote is one code path and standalone == cloud for
|
|
1386
|
+
// the local case. The pending queue remains the fallback when there is no remote target
|
|
1387
|
+
// or the transport send fails (e.g. P2P down), so MCP coordinators can still backfill.
|
|
1388
|
+
const remoteCoordinatorDaemonId = workerCoordinatorDaemonId && localDaemonId && !sameDaemonId(workerCoordinatorDaemonId, localDaemonId)
|
|
1389
|
+
? workerCoordinatorDaemonId
|
|
1390
|
+
: '';
|
|
1391
|
+
if (remoteCoordinatorDaemonId && components.dispatchMeshCommand) {
|
|
1392
|
+
const forwardPayload: Record<string, unknown> = {
|
|
1393
|
+
event: args.event,
|
|
1394
|
+
meshId: args.meshId,
|
|
1395
|
+
nodeId: args.nodeId || undefined,
|
|
1396
|
+
workspace: readNonEmptyString(args.metadataEvent.workspace),
|
|
1397
|
+
...args.metadataEvent,
|
|
1398
|
+
...(recoveryContext ? { recoveryContext } : {}),
|
|
1399
|
+
};
|
|
1400
|
+
components.dispatchMeshCommand(remoteCoordinatorDaemonId, 'mesh_forward_event', forwardPayload)
|
|
1401
|
+
.then(() => {
|
|
1402
|
+
LOG.info('MeshEvents', `Forwarded ${args.event} for mesh ${args.meshId} to remote coordinator daemon ${remoteCoordinatorDaemonId.slice(0, 12)}…`);
|
|
1403
|
+
})
|
|
1404
|
+
.catch((error: any) => {
|
|
1405
|
+
LOG.warn('MeshEvents', `Remote forward of ${args.event} failed (${error?.message || error}); queuing for backfill`);
|
|
1406
|
+
queuePendingMeshCoordinatorEvent({
|
|
1407
|
+
event: args.event,
|
|
1408
|
+
meshId: args.meshId,
|
|
1409
|
+
nodeLabel: args.nodeLabel,
|
|
1410
|
+
nodeId: args.nodeId || undefined,
|
|
1411
|
+
workspace: readNonEmptyString(args.metadataEvent.workspace),
|
|
1412
|
+
metadataEvent: { ...args.metadataEvent, ...(recoveryContext ? { recoveryContext } : {}) },
|
|
1413
|
+
coordinatorMessage: messageText,
|
|
1414
|
+
queuedAt: Date.now(),
|
|
1415
|
+
targetCoordinatorDaemonId: remoteCoordinatorDaemonId,
|
|
1416
|
+
});
|
|
1417
|
+
});
|
|
1418
|
+
return { success: true, forwarded: 0, remoteForwarded: true };
|
|
1419
|
+
}
|
|
1357
1420
|
if (queuePendingMeshCoordinatorEvent({
|
|
1358
1421
|
event: args.event,
|
|
1359
1422
|
meshId: args.meshId,
|
|
@@ -1380,23 +1443,35 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1380
1443
|
// Previously, terminal events skipped the queue and were only direct-injected
|
|
1381
1444
|
// into live CLI coordinators via send_message, which silently dropped them
|
|
1382
1445
|
// when the coordinator was in a generating state.
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1446
|
+
const pendingEvent = {
|
|
1447
|
+
event: args.event,
|
|
1448
|
+
meshId: args.meshId,
|
|
1449
|
+
nodeLabel: args.nodeLabel,
|
|
1450
|
+
nodeId: args.nodeId || undefined,
|
|
1451
|
+
workspace: readNonEmptyString(args.metadataEvent.workspace),
|
|
1452
|
+
metadataEvent: {
|
|
1453
|
+
...args.metadataEvent,
|
|
1454
|
+
...(recoveryContext ? { recoveryContext } : {}),
|
|
1455
|
+
},
|
|
1456
|
+
coordinatorMessage: messageText,
|
|
1457
|
+
queuedAt: Date.now(),
|
|
1458
|
+
...(workerCoordinatorDaemonId ? { targetCoordinatorDaemonId: workerCoordinatorDaemonId } : {}),
|
|
1459
|
+
};
|
|
1460
|
+
if (queuePendingMeshCoordinatorEvent(pendingEvent)) {
|
|
1397
1461
|
LOG.info('MeshEvents', `Queued ${args.event} for MCP coordinator (mesh ${args.meshId})`);
|
|
1398
1462
|
}
|
|
1399
1463
|
|
|
1464
|
+
// R3: the live coordinators below receive this event directly in their PTY. They (or their
|
|
1465
|
+
// MCP client) also poll get_pending_mesh_events with coordinatorDaemonId = this daemon, which
|
|
1466
|
+
// would re-deliver the queued copy → the user saw the same completion twice. Mark the event
|
|
1467
|
+
// direct-delivered to THIS daemon so that coordinator's own drain skips it. The queue entry
|
|
1468
|
+
// stays for every OTHER consumer (idle / MCP-only / remote) that did not get the direct inject.
|
|
1469
|
+
// markMeshCoordinatorEventDirectDelivered canonicalizes the id internally so the raw
|
|
1470
|
+
// machineId here matches the prefixed instanceId (standalone_mach_X) the coordinator drains with.
|
|
1471
|
+
if (localDaemonId) {
|
|
1472
|
+
markMeshCoordinatorEventDirectDelivered(localDaemonId, pendingEvent);
|
|
1473
|
+
}
|
|
1474
|
+
|
|
1400
1475
|
const forceInject = shouldForceInjectMeshEvent(args.event);
|
|
1401
1476
|
for (const coord of coordinatorInstances) {
|
|
1402
1477
|
const coordState = coord.getState();
|
|
@@ -1520,48 +1595,29 @@ export function setupMeshEventForwarding(components: DaemonComponents) {
|
|
|
1520
1595
|
const instanceId = readNonEmptyString(event.instanceId);
|
|
1521
1596
|
if (!instanceId) return;
|
|
1522
1597
|
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
const
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
if (
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
if (!meshIdFromDirectDispatch) return;
|
|
1598
|
+
// R1: all session→node→mesh→coordinator interpretation is folded into the single
|
|
1599
|
+
// resolveWorkerDelegateRouting() resolver. No stamp (meshNodeFor / meshNodeId /
|
|
1600
|
+
// meshCoordinatorDaemonId / meshCoordinatorNodeId / launchedByCoordinator) is read
|
|
1601
|
+
// here to make a routing decision — the resolver is the one authority, and the
|
|
1602
|
+
// forwarder consumes its typed result only.
|
|
1603
|
+
const routing = resolveWorkerDelegateRouting(components, instanceId, {
|
|
1604
|
+
getMeshById: (meshId) => getMeshWithCache(components, meshId),
|
|
1605
|
+
getMeshByWorkspace: (workspace) => getCachedMeshByWorkspace(workspace),
|
|
1606
|
+
});
|
|
1607
|
+
if (!routing.isDelegate) {
|
|
1608
|
+
// R4: a worker that presented a valid envelope but resolved to no mesh used to be
|
|
1609
|
+
// dropped silently. Leave a fail-loud diagnostic so the missing completion is
|
|
1610
|
+
// traceable. Benign non-delegate rejections (not_cli / no_workspace / etc.) are
|
|
1611
|
+
// no-ops inside recordUnroutableDelegateEvent.
|
|
1612
|
+
recordUnroutableDelegateEvent(routing, event.event);
|
|
1613
|
+
return;
|
|
1540
1614
|
}
|
|
1541
1615
|
|
|
1542
|
-
const meshIdFromRuntime = readNonEmptyString(settings.meshNodeFor) || meshIdFromDirectDispatch;
|
|
1543
|
-
|
|
1544
|
-
const isMeshDelegate = Boolean(meshIdFromRuntime || settings.launchedByCoordinator);
|
|
1545
|
-
if (!isMeshDelegate) return;
|
|
1546
|
-
|
|
1547
|
-
const mesh = meshIdFromRuntime ? getMeshWithCache(components, meshIdFromRuntime) : getCachedMeshByWorkspace(workspace);
|
|
1548
|
-
const meshId = meshIdFromRuntime || readNonEmptyString(mesh?.id);
|
|
1549
|
-
if (!meshId) return;
|
|
1550
|
-
|
|
1551
|
-
const targetNode = mesh?.nodes?.find((n: any) => n.workspace === workspace);
|
|
1552
|
-
const runtimeNodeId = readNonEmptyString(settings.meshNodeId);
|
|
1553
|
-
const resolvedNodeId = targetNode?.id || runtimeNodeId;
|
|
1554
|
-
const nodeLabel = targetNode
|
|
1555
|
-
? `Node '${targetNode.id}'`
|
|
1556
|
-
: runtimeNodeId
|
|
1557
|
-
? `Node '${runtimeNodeId}'`
|
|
1558
|
-
: `Agent at ${workspace}`;
|
|
1559
|
-
|
|
1560
1616
|
injectMeshSystemMessage(components, {
|
|
1561
|
-
meshId,
|
|
1617
|
+
meshId: routing.meshId,
|
|
1562
1618
|
sourceInstanceId: instanceId,
|
|
1563
|
-
nodeId:
|
|
1564
|
-
nodeLabel,
|
|
1619
|
+
nodeId: routing.nodeId,
|
|
1620
|
+
nodeLabel: routing.nodeLabel,
|
|
1565
1621
|
event: event.event,
|
|
1566
1622
|
metadataEvent: event,
|
|
1567
1623
|
});
|
|
@@ -4,7 +4,7 @@ import { randomUUID } from 'crypto';
|
|
|
4
4
|
import { LOG } from '../logging/logger.js';
|
|
5
5
|
import { getLedgerDir, readLedgerEntries } from './mesh-ledger.js';
|
|
6
6
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
7
|
-
import { buildMeshSystemMessage, readNonEmptyString, readRecord, resolveEventSessionId } from './mesh-events-utils.js';
|
|
7
|
+
import { buildMeshSystemMessage, canonicalDaemonId, readNonEmptyString, readRecord, resolveEventSessionId } from './mesh-events-utils.js';
|
|
8
8
|
|
|
9
9
|
// ---------------------------------------------------------------------------
|
|
10
10
|
// MCP coordinator pending-event queue — FILE-BASED PERSISTENCE
|
|
@@ -83,6 +83,47 @@ export function buildPendingEventFingerprint(event: PendingMeshCoordinatorEvent)
|
|
|
83
83
|
].join('::');
|
|
84
84
|
}
|
|
85
85
|
|
|
86
|
+
// R3: TTL for the direct-delivered marker. A coordinator polls get_pending_mesh_events
|
|
87
|
+
// well within this window after a terminal event; after it expires the marker is swept and
|
|
88
|
+
// a late/duplicate drain would (harmlessly) re-surface — but by then the event is long gone
|
|
89
|
+
// from the queue too. 10 minutes mirrors the completion-fingerprint TTL.
|
|
90
|
+
const DIRECT_DELIVERED_TTL_MS = 10 * 60 * 1000;
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* R3: record that an event was direct-injected into a live coordinator on `coordinatorDaemonId`.
|
|
94
|
+
* That coordinator's own drain (get_pending_mesh_events with the same coordinatorDaemonId) will
|
|
95
|
+
* skip the queued copy, so it receives the event exactly once instead of twice (PTY + poll).
|
|
96
|
+
* Other consumers (unscoped drainers, other daemons) are unaffected — they did not get the inject.
|
|
97
|
+
*/
|
|
98
|
+
export function markMeshCoordinatorEventDirectDelivered(
|
|
99
|
+
coordinatorDaemonId: string,
|
|
100
|
+
event: PendingMeshCoordinatorEvent,
|
|
101
|
+
): void {
|
|
102
|
+
// Canonicalize so the raw machineId used at inject time and the prefixed instanceId the
|
|
103
|
+
// coordinator drains with resolve to the same key.
|
|
104
|
+
const canonical = canonicalDaemonId(coordinatorDaemonId);
|
|
105
|
+
if (!canonical) return;
|
|
106
|
+
const fingerprint = buildPendingEventFingerprint(event);
|
|
107
|
+
if (!fingerprint.trim()) return;
|
|
108
|
+
try {
|
|
109
|
+
const store = MeshRuntimeStore.getInstance();
|
|
110
|
+
store.recordDirectDelivered(canonical, fingerprint, DIRECT_DELIVERED_TTL_MS);
|
|
111
|
+
store.sweepExpiredDirectDelivered();
|
|
112
|
+
} catch { /* best-effort — a duplicate is preferable to a crash */ }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function wasDirectDeliveredToCoordinator(coordinatorDaemonId: string, event: PendingMeshCoordinatorEvent): boolean {
|
|
116
|
+
const canonical = canonicalDaemonId(coordinatorDaemonId);
|
|
117
|
+
if (!canonical) return false;
|
|
118
|
+
const fingerprint = buildPendingEventFingerprint(event);
|
|
119
|
+
if (!fingerprint.trim()) return false;
|
|
120
|
+
try {
|
|
121
|
+
return MeshRuntimeStore.getInstance().wasDirectDelivered(canonical, fingerprint);
|
|
122
|
+
} catch {
|
|
123
|
+
return false;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
86
127
|
export function hasPendingCoordinatorEventDuplicate(event: PendingMeshCoordinatorEvent): boolean {
|
|
87
128
|
const fingerprint = buildPendingEventFingerprint(event);
|
|
88
129
|
if (!fingerprint.trim()) return false;
|
|
@@ -321,7 +362,14 @@ export function drainPendingMeshCoordinatorEvents(meshId?: string, coordinatorDa
|
|
|
321
362
|
for (const event of filtered) pushUnique(event);
|
|
322
363
|
}
|
|
323
364
|
if (merged.length === 0) return [];
|
|
324
|
-
|
|
365
|
+
// R3: when this drain is scoped to a coordinator daemon, exclude events that were already
|
|
366
|
+
// direct-injected into that coordinator's live CLI session. Unscoped drains (no daemon id)
|
|
367
|
+
// keep everything — they belong to consumers that never received the direct inject.
|
|
368
|
+
const deliverable = coordinatorDaemonId
|
|
369
|
+
? merged.filter(event => !wasDirectDeliveredToCoordinator(coordinatorDaemonId, event))
|
|
370
|
+
: merged;
|
|
371
|
+
if (deliverable.length === 0) return [];
|
|
372
|
+
return reconcilePendingMeshCoordinatorEvents(meshId, deliverable);
|
|
325
373
|
}
|
|
326
374
|
|
|
327
375
|
/** Peek at pending coordinator events without draining (non-destructive). */
|
|
@@ -356,7 +404,13 @@ export function getPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaem
|
|
|
356
404
|
pushUnique(event);
|
|
357
405
|
}
|
|
358
406
|
|
|
359
|
-
|
|
407
|
+
// R3: hide events already direct-delivered to this coordinator from its status peek, so
|
|
408
|
+
// mesh_status doesn't report a "pending" event the coordinator has in fact already received.
|
|
409
|
+
const deliverable = coordinatorDaemonId
|
|
410
|
+
? merged.filter(event => !wasDirectDeliveredToCoordinator(coordinatorDaemonId, event))
|
|
411
|
+
: merged;
|
|
412
|
+
|
|
413
|
+
return reconcilePendingMeshCoordinatorEvents(meshId, deliverable);
|
|
360
414
|
}
|
|
361
415
|
|
|
362
416
|
/**
|
|
@@ -10,6 +10,26 @@ export function readRecord(value: unknown): Record<string, unknown> | undefined
|
|
|
10
10
|
: undefined;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
+
// A single daemon is identified three ways across the system: the raw machineId
|
|
14
|
+
// (`mach_X`, what loadConfig().machineId returns and what the core forwarder uses as
|
|
15
|
+
// localDaemonId), the cloud daemon id (`daemon_mach_X`), and the standalone daemon id
|
|
16
|
+
// (`standalone_mach_X`, the runtime instanceId the MCP coordinator reports as
|
|
17
|
+
// ctx.localDaemonId). Worker envelope stamps (meshCoordinatorDaemonId) and coordinator
|
|
18
|
+
// drains use the prefixed forms, while the forwarder compares against the raw id — so a
|
|
19
|
+
// naive `===` treats the same daemon as remote and breaks both coordinator matching and
|
|
20
|
+
// the R3 direct-delivered dedup. Canonicalize to the raw id before comparing.
|
|
21
|
+
export function canonicalDaemonId(value: unknown): string {
|
|
22
|
+
const id = readNonEmptyString(value);
|
|
23
|
+
if (!id) return '';
|
|
24
|
+
return id.replace(/^(?:daemon|standalone)_/, '');
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function sameDaemonId(a: unknown, b: unknown): boolean {
|
|
28
|
+
const ca = canonicalDaemonId(a);
|
|
29
|
+
const cb = canonicalDaemonId(b);
|
|
30
|
+
return ca !== '' && ca === cb;
|
|
31
|
+
}
|
|
32
|
+
|
|
13
33
|
export function resolveEventSessionId(event: Record<string, unknown>, fallback?: unknown): string {
|
|
14
34
|
return readNonEmptyString(event.targetSessionId)
|
|
15
35
|
|| readNonEmptyString(event.sessionId)
|