@adhdev/daemon-core 0.9.82-rc.342 → 0.9.82-rc.344
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/router.d.ts +20 -4
- package/dist/index.js +654 -332
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +674 -352
- package/dist/index.mjs.map +1 -1
- package/dist/mesh/mesh-events-pending.d.ts +10 -0
- package/dist/mesh/mesh-events-utils.d.ts +10 -0
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/mesh/mesh-runtime-store.d.ts +1 -0
- package/dist/mesh/mesh-work-queue.d.ts +10 -0
- package/dist/providers/cli-provider-instance.d.ts +1 -0
- package/dist/providers/provider-instance-manager.d.ts +1 -0
- package/dist/providers/provider-instance.d.ts +2 -0
- package/dist/providers/spec/fsm-driver.d.ts +16 -0
- package/package.json +2 -2
- package/src/commands/cli-manager.ts +18 -0
- package/src/commands/router.ts +86 -8
- package/src/mesh/mesh-events-coordinator.ts +53 -3
- package/src/mesh/mesh-events-pending.ts +54 -4
- package/src/mesh/mesh-events-utils.ts +61 -9
- package/src/mesh/mesh-ledger.ts +1 -0
- package/src/mesh/mesh-reconcile-loop.ts +170 -3
- package/src/mesh/mesh-runtime-store.ts +38 -4
- package/src/mesh/mesh-work-queue.ts +13 -0
- package/src/providers/cli-provider-instance.ts +4 -1
- package/src/providers/provider-instance-manager.ts +1 -1
- package/src/providers/provider-instance.ts +1 -1
- package/src/providers/spec/fsm-driver.ts +64 -24
|
@@ -42,6 +42,12 @@ export interface MeshWorkerRelayStamp {
|
|
|
42
42
|
meshNodeFor?: string;
|
|
43
43
|
meshNodeId?: string;
|
|
44
44
|
meshCoordinatorDaemonId?: string;
|
|
45
|
+
// The ORIGINATING coordinator SESSION (not just its daemon). Carried so a worker's
|
|
46
|
+
// completion event can be routed back to the exact coordinator session that
|
|
47
|
+
// dispatched the work, even when several coordinator sessions share one daemon
|
|
48
|
+
// (the multi-coordinator misroute). Optional + absent on legacy dispatches, in
|
|
49
|
+
// which case routing falls back to the daemon-level anchor (current behaviour).
|
|
50
|
+
meshCoordinatorSessionId?: string;
|
|
45
51
|
launchedByCoordinator?: boolean;
|
|
46
52
|
}
|
|
47
53
|
|
|
@@ -63,6 +69,7 @@ export function buildMeshWorkerRelayStamp(
|
|
|
63
69
|
meshId?: unknown;
|
|
64
70
|
nodeId?: unknown;
|
|
65
71
|
coordinatorDaemonId?: unknown;
|
|
72
|
+
coordinatorSessionId?: unknown;
|
|
66
73
|
} | undefined,
|
|
67
74
|
): MeshWorkerRelayStamp | undefined {
|
|
68
75
|
if (!meshContext) return undefined;
|
|
@@ -80,9 +87,17 @@ export function buildMeshWorkerRelayStamp(
|
|
|
80
87
|
stamp.meshCoordinatorDaemonId = coordinatorDaemonId;
|
|
81
88
|
}
|
|
82
89
|
|
|
90
|
+
// Session-level anchor (multi-coordinator routing): stamp the originating
|
|
91
|
+
// coordinator session id so the completion event can target the exact session.
|
|
92
|
+
// Carried over P2P to remote workers so a remote worker's echo returns it.
|
|
93
|
+
const coordinatorSessionId = readNonEmptyString(meshContext.coordinatorSessionId);
|
|
94
|
+
if (coordinatorSessionId && !readNonEmptyString(settings.meshCoordinatorSessionId)) {
|
|
95
|
+
stamp.meshCoordinatorSessionId = coordinatorSessionId;
|
|
96
|
+
}
|
|
97
|
+
|
|
83
98
|
// A dispatch from a coordinator is itself proof of delegation; stamp it when the
|
|
84
99
|
// session is being given any mesh routing context but has no marker yet.
|
|
85
|
-
if ((meshId || nodeId || coordinatorDaemonId) && settings.launchedByCoordinator !== true) {
|
|
100
|
+
if ((meshId || nodeId || coordinatorDaemonId || coordinatorSessionId) && settings.launchedByCoordinator !== true) {
|
|
86
101
|
stamp.launchedByCoordinator = true;
|
|
87
102
|
}
|
|
88
103
|
|
|
@@ -109,6 +124,30 @@ export function readWorkerResultMetadata(event: Record<string, unknown>): Record
|
|
|
109
124
|
|
|
110
125
|
const MESH_SURFACED_PREVIEW_MAX_CHARS = 512;
|
|
111
126
|
|
|
127
|
+
// Cap for the worker final summary surfaced INLINE into the coordinator's chat
|
|
128
|
+
// (buildMeshSystemMessage). Larger than the mirror preview cap because this is the
|
|
129
|
+
// coordinator-facing payload that replaces a "go call mesh_read_chat" instruction —
|
|
130
|
+
// it should carry enough of the worker's result to act on without a second round-trip,
|
|
131
|
+
// while still bounding what is written into the coordinator PTY.
|
|
132
|
+
const MESH_COMPLETION_SURFACE_MAX_CHARS = 4000;
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* The worker's final assistant text carried on a completion event — read from
|
|
136
|
+
* `finalSummary` (and the `workerResult.summary` / `result.summary` fallbacks some
|
|
137
|
+
* paths use). Returns '' when the event carries no assistant text (lifecycle events
|
|
138
|
+
* without a summary). Shared by the coordinator chat surface (buildMeshSystemMessage)
|
|
139
|
+
* and the held-event ledger audit record so both read the summary the same way.
|
|
140
|
+
*/
|
|
141
|
+
export function readMeshCompletionSummary(metadataEvent: Record<string, unknown>): string {
|
|
142
|
+
const workerResult = readWorkerResultMetadata(metadataEvent);
|
|
143
|
+
const resultRecord = readRecord(metadataEvent.result);
|
|
144
|
+
return readNonEmptyString(metadataEvent.finalSummary)
|
|
145
|
+
|| readNonEmptyString(workerResult?.summary)
|
|
146
|
+
|| readNonEmptyString(workerResult?.finalSummary)
|
|
147
|
+
|| readNonEmptyString(resultRecord?.summary)
|
|
148
|
+
|| readNonEmptyString(resultRecord?.finalSummary);
|
|
149
|
+
}
|
|
150
|
+
|
|
112
151
|
/**
|
|
113
152
|
* A coordinator that surfaces a REMOTE worker's mesh session has no local instance for
|
|
114
153
|
* it, so the status snapshot's getLastDisplayMessage has nothing to read and the only
|
|
@@ -126,13 +165,7 @@ const MESH_SURFACED_PREVIEW_MAX_CHARS = 512;
|
|
|
126
165
|
export function resolveMeshSurfacedSessionPreview(
|
|
127
166
|
metadataEvent: Record<string, unknown>,
|
|
128
167
|
): { preview: string; role: 'assistant'; receivedAt: number } | undefined {
|
|
129
|
-
const
|
|
130
|
-
const resultRecord = readRecord(metadataEvent.result);
|
|
131
|
-
const summaryText = readNonEmptyString(metadataEvent.finalSummary)
|
|
132
|
-
|| readNonEmptyString(workerResult?.summary)
|
|
133
|
-
|| readNonEmptyString(workerResult?.finalSummary)
|
|
134
|
-
|| readNonEmptyString(resultRecord?.summary)
|
|
135
|
-
|| readNonEmptyString(resultRecord?.finalSummary);
|
|
168
|
+
const summaryText = readMeshCompletionSummary(metadataEvent);
|
|
136
169
|
if (!summaryText) return undefined;
|
|
137
170
|
const truncationSuffix = '...[truncated]';
|
|
138
171
|
const preview = summaryText.length > MESH_SURFACED_PREVIEW_MAX_CHARS
|
|
@@ -186,7 +219,26 @@ export function buildMeshSystemMessage(args: {
|
|
|
186
219
|
if (args.metadataEvent.source === 'no_progress_reconciliation') {
|
|
187
220
|
return `[System] ${args.nodeLabel} already has completion evidence${metadata}. The no-progress monitor reconciled the terminal handoff and marked the session complete; wait for the queued completion event/status refresh before doing any manual transcript check.`;
|
|
188
221
|
}
|
|
189
|
-
const
|
|
222
|
+
const reviewRecommended = args.metadataEvent.reviewRecommended === true;
|
|
223
|
+
// Auto-surface the worker's final summary directly into the coordinator chat so it
|
|
224
|
+
// does not have to call mesh_read_chat just to see the result. The summary IS the
|
|
225
|
+
// worker's final assistant message; embedding it here replaces the previous
|
|
226
|
+
// "go call mesh_read_chat" instruction with the answer itself. This rides the
|
|
227
|
+
// existing (non-modal) coordinator delivery channel — it does not write into a
|
|
228
|
+
// parked harness modal. Falls back to the read_chat instruction only when the event
|
|
229
|
+
// genuinely carries no summary (so behaviour is unchanged for summary-less events).
|
|
230
|
+
const completionSummary = readMeshCompletionSummary(args.metadataEvent);
|
|
231
|
+
if (completionSummary) {
|
|
232
|
+
const truncationSuffix = '\n…[truncated — call mesh_read_chat once for the full transcript]';
|
|
233
|
+
const surfaced = completionSummary.length > MESH_COMPLETION_SURFACE_MAX_CHARS
|
|
234
|
+
? `${completionSummary.slice(0, MESH_COMPLETION_SURFACE_MAX_CHARS - truncationSuffix.length)}${truncationSuffix}`
|
|
235
|
+
: completionSummary;
|
|
236
|
+
const verifyNote = reviewRecommended
|
|
237
|
+
? ' Completion evidence is insufficient — verify via git status or provider_session_id before assuming the task is done.'
|
|
238
|
+
: '';
|
|
239
|
+
return `[System] ${args.nodeLabel} has completed its task and is now idle${metadata}. Its final summary is included below — read it directly and only call mesh_read_chat if you need the full transcript.${verifyNote}\n\n--- ${args.nodeLabel} final summary ---\n${surfaced}`;
|
|
240
|
+
}
|
|
241
|
+
const reviewNote = reviewRecommended
|
|
190
242
|
? ' Completion evidence is insufficient — verify via git status or provider_session_id before assuming the task is done. Use mesh_read_chat once if needed, but do not poll repeatedly.'
|
|
191
243
|
: ' Use mesh_read_chat once to review its final progress, but do not poll repeatedly.';
|
|
192
244
|
return `[System] ${args.nodeLabel} has completed its task and is now idle${metadata}. This completion came from the agent status event path;${reviewNote}`;
|
package/src/mesh/mesh-ledger.ts
CHANGED
|
@@ -45,8 +45,9 @@ import type { LocalMeshEntry } from '../repo-mesh-types.js';
|
|
|
45
45
|
import { loadConfig } from '../config/config.js';
|
|
46
46
|
import { listMeshes } from '../config/mesh-config.js';
|
|
47
47
|
import { LOG } from '../logging/logger.js';
|
|
48
|
-
import { drainPendingMeshCoordinatorEvents } from './mesh-events-pending.js';
|
|
48
|
+
import { drainPendingMeshCoordinatorEvents, getPendingMeshCoordinatorEvents, buildPendingEventFingerprint, queuePendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
49
49
|
import type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
|
|
50
|
+
import { appendLedgerEntry } from './mesh-ledger.js';
|
|
50
51
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
51
52
|
import { handleMeshForwardEvent, shouldForceInjectMeshEvent, MESH_FORCE_INJECT_EVENTS, triggerMeshQueue } from './mesh-events-coordinator.js';
|
|
52
53
|
import {
|
|
@@ -54,7 +55,7 @@ import {
|
|
|
54
55
|
ackUnresolvedDelegateForward,
|
|
55
56
|
expireStaleUnresolvedDelegateForwards,
|
|
56
57
|
} from './mesh-unresolved-forward-outbox.js';
|
|
57
|
-
import { readNonEmptyString } from './mesh-events-utils.js';
|
|
58
|
+
import { readNonEmptyString, readMeshCompletionSummary } from './mesh-events-utils.js';
|
|
58
59
|
import { getActiveDirectDispatches, getQueue } from './mesh-work-queue.js';
|
|
59
60
|
import { readLedgerEntries } from './mesh-ledger.js';
|
|
60
61
|
import { pruneStaleDirectDispatches } from './mesh-active-work.js';
|
|
@@ -97,6 +98,11 @@ function resolveReconcileIntervalMs(): number {
|
|
|
97
98
|
interface LiveCoordinator {
|
|
98
99
|
meshId: string;
|
|
99
100
|
instance: ReturnType<DaemonComponents['instanceManager']['getInstance']>;
|
|
101
|
+
// Runtime session id of this coordinator instance (getState().instanceId). PHASE 2
|
|
102
|
+
// strict-matches an event's targetCoordinatorSessionId against this so a completion
|
|
103
|
+
// routes back to the exact originating coordinator session, not a sibling on the same
|
|
104
|
+
// daemon (the multi-coordinator misroute).
|
|
105
|
+
sessionId: string;
|
|
100
106
|
idle: boolean;
|
|
101
107
|
// True when the coordinator session is parked on a harness modal awaiting a
|
|
102
108
|
// human answer — claude-cli AskUserQuestion (waiting_choice) or a tool-consent
|
|
@@ -205,7 +211,8 @@ function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
|
|
|
205
211
|
// Lowercase literal compare — the SessionStatus enum is forked across modules
|
|
206
212
|
// and waiting_choice is absent from some of them (see cli-provider-instance).
|
|
207
213
|
const modalParked = status === 'waiting_choice' || status === 'waiting_approval';
|
|
208
|
-
|
|
214
|
+
const sessionId = readNonEmptyString(state.instanceId);
|
|
215
|
+
out.push({ meshId, instance: inst, sessionId, idle: status === 'idle', modalParked });
|
|
209
216
|
}
|
|
210
217
|
return out;
|
|
211
218
|
}
|
|
@@ -225,6 +232,71 @@ function injectPendingIntoCoordinator(
|
|
|
225
232
|
});
|
|
226
233
|
}
|
|
227
234
|
|
|
235
|
+
// Held-event ledger dedup: fingerprints of held terminal events already written as an
|
|
236
|
+
// `event_held` ledger audit record in THIS process. Prevents the 4s reconcile tick from
|
|
237
|
+
// re-logging the same held event every interval while a coordinator stays modal-parked.
|
|
238
|
+
// Per-process only (not persisted) — if the daemon restarts while an event is still held
|
|
239
|
+
// it is re-logged once, which is desirable: it re-confirms the event is still undelivered.
|
|
240
|
+
const heldEventLedgerRecorded = new Set<string>();
|
|
241
|
+
|
|
242
|
+
// C1 (data safety): when a terminal completion/approval/bootstrap event cannot be
|
|
243
|
+
// delivered because the only coordinators are modal-parked, the event is held at
|
|
244
|
+
// drained=0 in the pending queue (SQLite + JSONL) for a later tick. That queue is
|
|
245
|
+
// disk-persisted but carries no operator-visible audit trail and can be silently
|
|
246
|
+
// dropped by the pending-file trim (100 KB / 50-entry cap). To guarantee a held
|
|
247
|
+
// completion's worker summary is never silently lost, mirror each held terminal event
|
|
248
|
+
// into the coordinator's mesh ledger as an `event_held` entry — auditable and
|
|
249
|
+
// recoverable (the finalSummary survives even if the pending copy is later trimmed or
|
|
250
|
+
// the coordinator session is force-resolved before re-drain). Idempotent per process
|
|
251
|
+
// via heldEventLedgerRecorded so a long modal park does not spam the ledger.
|
|
252
|
+
function recordHeldTerminalEventsToLedger(
|
|
253
|
+
meshId: string,
|
|
254
|
+
drainDaemonIds: string[],
|
|
255
|
+
reason: string,
|
|
256
|
+
heldForCoordinatorCount: number,
|
|
257
|
+
): void {
|
|
258
|
+
let pending: readonly PendingMeshCoordinatorEvent[];
|
|
259
|
+
try {
|
|
260
|
+
pending = getPendingMeshCoordinatorEvents(meshId, drainDaemonIds.length > 0 ? drainDaemonIds : undefined);
|
|
261
|
+
} catch {
|
|
262
|
+
return; // best-effort audit — never let a peek failure break the tick
|
|
263
|
+
}
|
|
264
|
+
for (const event of pending) {
|
|
265
|
+
// Only audit terminal/force-inject events (completion / approval / stop / refine·
|
|
266
|
+
// bootstrap). Silent lifecycle events (agent:ready / generating_started) carry no
|
|
267
|
+
// worker output to preserve and re-drain harmlessly, so they need no audit trail.
|
|
268
|
+
if (!shouldForceInjectMeshEvent(event.event)) continue;
|
|
269
|
+
const fingerprint = buildPendingEventFingerprint(event);
|
|
270
|
+
const key = `${meshId}::${fingerprint || `${event.event}::${event.nodeId || ''}::${event.queuedAt}`}`;
|
|
271
|
+
if (heldEventLedgerRecorded.has(key)) continue;
|
|
272
|
+
heldEventLedgerRecorded.add(key);
|
|
273
|
+
const finalSummary = readMeshCompletionSummary(event.metadataEvent);
|
|
274
|
+
try {
|
|
275
|
+
appendLedgerEntry(meshId, {
|
|
276
|
+
kind: 'event_held',
|
|
277
|
+
...(event.nodeId ? { nodeId: event.nodeId } : {}),
|
|
278
|
+
payload: {
|
|
279
|
+
event: event.event,
|
|
280
|
+
reason,
|
|
281
|
+
recoverable: true,
|
|
282
|
+
heldForCoordinators: heldForCoordinatorCount,
|
|
283
|
+
nodeLabel: event.nodeLabel,
|
|
284
|
+
...(event.workspace ? { workspace: event.workspace } : {}),
|
|
285
|
+
targetCoordinatorDaemonId: event.targetCoordinatorDaemonId ?? null,
|
|
286
|
+
queuedAt: event.queuedAt,
|
|
287
|
+
...(fingerprint ? { fingerprint } : {}),
|
|
288
|
+
...(finalSummary ? { finalSummary } : {}),
|
|
289
|
+
},
|
|
290
|
+
});
|
|
291
|
+
LOG.info('MeshReconcile', `Ledger-recorded held ${event.event} for mesh ${meshId} (reason ${reason}) — recoverable from ledger`);
|
|
292
|
+
} catch (e: any) {
|
|
293
|
+
// Failed to persist — drop the dedup marker so the next tick retries.
|
|
294
|
+
heldEventLedgerRecorded.delete(key);
|
|
295
|
+
LOG.warn('MeshReconcile', `Failed to ledger-record held ${event.event} for mesh ${meshId}: ${e?.message || e}`);
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
|
|
228
300
|
// One reconcile tick. Two independent phases:
|
|
229
301
|
//
|
|
230
302
|
// PHASE 1 — Remote queue pull (the fix for remote worktree completions never
|
|
@@ -426,6 +498,24 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
426
498
|
if (targetCoordinators.length === 0) {
|
|
427
499
|
if (modalParkedCoordinators.length > 0) {
|
|
428
500
|
LOG.info('MeshReconcile', `Reconcile skip → modal-parked: holding pending event(s) for mesh ${meshId} (${modalParkedCoordinators.length} coordinator(s) awaiting a modal answer; events left queued)`);
|
|
501
|
+
// C1: mirror held terminal events into the ledger so a held completion's
|
|
502
|
+
// worker summary is auditable/recoverable even if the modal is never
|
|
503
|
+
// resolved, the coordinator restarts, or the pending file is later trimmed.
|
|
504
|
+
// The events stay queued (drained=0) for re-drain on a later tick; this only
|
|
505
|
+
// adds the durable audit copy. Idempotent per process — only newly-held
|
|
506
|
+
// events are logged. O(1)-gated: skip the peek when the queue is empty.
|
|
507
|
+
let hasPending = true;
|
|
508
|
+
if (store) {
|
|
509
|
+
try { hasPending = store.pendingEventCount(meshId) > 0; } catch { /* peek below */ }
|
|
510
|
+
}
|
|
511
|
+
if (hasPending) {
|
|
512
|
+
recordHeldTerminalEventsToLedger(
|
|
513
|
+
meshId,
|
|
514
|
+
drainDaemonIds.length > 0 ? drainDaemonIds : (localDaemonId ? [localDaemonId] : []),
|
|
515
|
+
'modal_parked',
|
|
516
|
+
modalParkedCoordinators.length,
|
|
517
|
+
);
|
|
518
|
+
}
|
|
429
519
|
}
|
|
430
520
|
continue;
|
|
431
521
|
}
|
|
@@ -453,6 +543,26 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
453
543
|
const mode = forceOnly ? 'force-drain → generating' : 'inject → idle';
|
|
454
544
|
LOG.info('MeshReconcile', `Reconcile ${mode}: ${pendingEvents.length} pending event(s) → ${targetCoordinators.length} coordinator(s) for mesh ${meshId}`);
|
|
455
545
|
for (const pending of pendingEvents) {
|
|
546
|
+
// Strict session routing (multi-coordinator): when the event names an
|
|
547
|
+
// originating coordinator session, deliver ONLY to the live coordinator whose
|
|
548
|
+
// session id matches — a sibling coordinator on the same daemon must NOT receive
|
|
549
|
+
// another coordinator's completion. When the event carries no session id (legacy /
|
|
550
|
+
// version-skewed / single-coordinator), fall back to the daemon-level set
|
|
551
|
+
// (unchanged behaviour — regression-0 for the common case).
|
|
552
|
+
const wantSession = readNonEmptyString(pending.targetCoordinatorSessionId);
|
|
553
|
+
if (wantSession) {
|
|
554
|
+
const matched = targetCoordinators.filter(c => c.sessionId === wantSession);
|
|
555
|
+
if (matched.length === 0) {
|
|
556
|
+
// The originating coordinator session is not deliverable on this daemon
|
|
557
|
+
// right now (gone, or modal-parked and excluded from targets). Strict mode
|
|
558
|
+
// does NOT broadcast to siblings — hold the event for a later tick, and
|
|
559
|
+
// ledger-expire it past a TTL so it can never wedge forever.
|
|
560
|
+
holdOrExpireStrictUnmatchedEvent(pending, wantSession, meshId);
|
|
561
|
+
continue;
|
|
562
|
+
}
|
|
563
|
+
for (const c of matched) injectPendingIntoCoordinator(c.instance, pending);
|
|
564
|
+
continue;
|
|
565
|
+
}
|
|
456
566
|
for (const c of targetCoordinators) {
|
|
457
567
|
injectPendingIntoCoordinator(c.instance, pending);
|
|
458
568
|
}
|
|
@@ -460,6 +570,56 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
460
570
|
}
|
|
461
571
|
}
|
|
462
572
|
|
|
573
|
+
// Strict-routing TTL: how long a drained completion whose originating coordinator session
|
|
574
|
+
// is not currently deliverable is held (re-queued for re-drain) before it is ledger-
|
|
575
|
+
// expired. Bounded so a coordinator session that never returns cannot wedge the event
|
|
576
|
+
// forever; broad enough to ride out a transient modal-park / brief restart.
|
|
577
|
+
const STRICT_SESSION_MATCH_TTL_MS = 60_000;
|
|
578
|
+
|
|
579
|
+
// Re-queue (hold) a strict-routed event whose coordinator session is not live, or — once it
|
|
580
|
+
// has aged past STRICT_SESSION_MATCH_TTL_MS — ledger-expire it (recoverable) and drop it.
|
|
581
|
+
// We deliberately do NOT broadcast an aged-out event to sibling coordinators: that is the
|
|
582
|
+
// very misroute strict routing exists to prevent. The drain already marked the row drained=1,
|
|
583
|
+
// so re-queuing re-persists a fresh undrained copy (dedup keys on drained=0 only); queuedAt is
|
|
584
|
+
// preserved so the TTL measures the event's true age across re-queues.
|
|
585
|
+
function holdOrExpireStrictUnmatchedEvent(
|
|
586
|
+
pending: PendingMeshCoordinatorEvent,
|
|
587
|
+
wantSession: string,
|
|
588
|
+
meshId: string,
|
|
589
|
+
): void {
|
|
590
|
+
const queuedAt = typeof pending.queuedAt === 'number' ? pending.queuedAt : Date.now();
|
|
591
|
+
if (Date.now() - queuedAt <= STRICT_SESSION_MATCH_TTL_MS) {
|
|
592
|
+
try {
|
|
593
|
+
queuePendingMeshCoordinatorEvent(pending); // preserves queuedAt → true age retained
|
|
594
|
+
LOG.info('MeshReconcile', `Strict route hold: coordinator session ${wantSession} not live on mesh ${meshId} — re-queued (${pending.event})`);
|
|
595
|
+
} catch (e: any) {
|
|
596
|
+
LOG.warn('MeshReconcile', `Strict route re-queue failed for ${pending.event} on mesh ${meshId}: ${e?.message || e}`);
|
|
597
|
+
}
|
|
598
|
+
return;
|
|
599
|
+
}
|
|
600
|
+
const finalSummary = readMeshCompletionSummary(pending.metadataEvent || {});
|
|
601
|
+
try {
|
|
602
|
+
appendLedgerEntry(meshId, {
|
|
603
|
+
kind: 'event_held',
|
|
604
|
+
...(pending.nodeId ? { nodeId: pending.nodeId } : {}),
|
|
605
|
+
payload: {
|
|
606
|
+
event: pending.event,
|
|
607
|
+
reason: 'strict_route_expired',
|
|
608
|
+
recoverable: true,
|
|
609
|
+
targetCoordinatorSessionId: wantSession,
|
|
610
|
+
targetCoordinatorDaemonId: pending.targetCoordinatorDaemonId ?? null,
|
|
611
|
+
nodeLabel: pending.nodeLabel,
|
|
612
|
+
...(pending.workspace ? { workspace: pending.workspace } : {}),
|
|
613
|
+
queuedAt,
|
|
614
|
+
...(finalSummary ? { finalSummary } : {}),
|
|
615
|
+
},
|
|
616
|
+
});
|
|
617
|
+
LOG.warn('MeshReconcile', `Strict route expire: coordinator session ${wantSession} never returned for mesh ${meshId} — recorded to ledger (recoverable), dropped (${pending.event})`);
|
|
618
|
+
} catch (e: any) {
|
|
619
|
+
LOG.warn('MeshReconcile', `Failed to ledger-expire strict-unmatched ${pending.event} for mesh ${meshId}: ${e?.message || e}`);
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
463
623
|
// Cloud-only: retry the worker-side unresolved-delegate forward outbox. For each
|
|
464
624
|
// durably-queued entry, push it to its coordinator daemon over P2P (mesh_forward_event)
|
|
465
625
|
// and ack (mark drained) ONLY on a successful, non-rejected response. A failed or
|
|
@@ -791,6 +951,13 @@ function buildForwardPayloadFromPending(event: any): Record<string, unknown> {
|
|
|
791
951
|
meshId: readNonEmptyString(event?.meshId),
|
|
792
952
|
nodeId: readNonEmptyString(event?.nodeId) || readNonEmptyString(metadata.meshNodeId),
|
|
793
953
|
workspace: readNonEmptyString(event?.workspace) || readNonEmptyString(metadata.workspace),
|
|
954
|
+
// Preserve the originating coordinator session id across the relay. It is normally
|
|
955
|
+
// carried inside metadataEvent.meshCoordinatorSessionId (spread below), but pass the
|
|
956
|
+
// top-level field through explicitly too so the handleMeshForwardEvent whitelist
|
|
957
|
+
// recovers it regardless of which carrier the producing daemon used.
|
|
958
|
+
...(readNonEmptyString(event?.targetCoordinatorSessionId)
|
|
959
|
+
? { targetCoordinatorSessionId: readNonEmptyString(event.targetCoordinatorSessionId) }
|
|
960
|
+
: {}),
|
|
794
961
|
...metadata,
|
|
795
962
|
};
|
|
796
963
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, renameSync, statSync } from 'fs';
|
|
2
2
|
import { dirname, join } from 'path';
|
|
3
|
+
import { LOG } from '../logging/logger.js';
|
|
3
4
|
import { loadBetterSqlite3 } from '../system/load-better-sqlite3.js';
|
|
4
5
|
import { getLedgerDir } from './mesh-ledger.js';
|
|
5
6
|
import { nodeSatisfiesRequiredTags } from './mesh-work-queue.js';
|
|
@@ -23,6 +24,8 @@ function legacyQueuePath(meshId: string): string {
|
|
|
23
24
|
return join(getLedgerDir(), `${safeMeshId(meshId)}.queue.json`);
|
|
24
25
|
}
|
|
25
26
|
|
|
27
|
+
let loggedMigrationFailure = false;
|
|
28
|
+
|
|
26
29
|
function meshRuntimeStorePath(): string {
|
|
27
30
|
const dir = getLedgerDir();
|
|
28
31
|
const nextPath = join(dir, 'mesh-runtime.db');
|
|
@@ -39,9 +42,23 @@ function meshRuntimeStorePath(): string {
|
|
|
39
42
|
renameSync(legacyCompanion, `${nextPath}${suffix}`);
|
|
40
43
|
}
|
|
41
44
|
}
|
|
42
|
-
} catch {
|
|
43
|
-
//
|
|
44
|
-
//
|
|
45
|
+
} catch (err: any) {
|
|
46
|
+
// Migration failed — most commonly win32 EPERM when a handle to the
|
|
47
|
+
// legacy DB is still open. Do NOT fall through to opening `nextPath`:
|
|
48
|
+
// that would create a fresh EMPTY store while the existing data stays
|
|
49
|
+
// stranded in the legacy file (split-brain / silent data loss). Instead
|
|
50
|
+
// keep using whichever file actually holds the data in-place — the next
|
|
51
|
+
// boot retries the rename. If the main rename already landed (only a
|
|
52
|
+
// companion file failed), the data is at nextPath; otherwise it is still
|
|
53
|
+
// at legacyPath.
|
|
54
|
+
if (!loggedMigrationFailure) {
|
|
55
|
+
loggedMigrationFailure = true;
|
|
56
|
+
LOG.warn(
|
|
57
|
+
'MeshRuntimeStore',
|
|
58
|
+
`Legacy beads.db→mesh-runtime.db migration failed; using existing DB in-place to avoid data loss: ${err?.message || err}`,
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
return existsSync(nextPath) ? nextPath : legacyPath;
|
|
45
62
|
}
|
|
46
63
|
return nextPath;
|
|
47
64
|
}
|
|
@@ -69,9 +86,26 @@ export class MeshRuntimeStore {
|
|
|
69
86
|
this.migrate();
|
|
70
87
|
}
|
|
71
88
|
|
|
89
|
+
private static loggedGetInstanceFailure = false;
|
|
90
|
+
|
|
72
91
|
static getInstance(): MeshRuntimeStore {
|
|
73
92
|
if (!this.instance) {
|
|
74
|
-
|
|
93
|
+
try {
|
|
94
|
+
this.instance = new MeshRuntimeStore(meshRuntimeStorePath());
|
|
95
|
+
} catch (err: any) {
|
|
96
|
+
// SQLite store could not be opened (e.g. better-sqlite3 native
|
|
97
|
+
// load failure, locked/corrupt DB). Callers wrap getInstance in
|
|
98
|
+
// try/catch and silently degrade to JSONL-only — surface ONE warn
|
|
99
|
+
// so that degraded mode is diagnosable, then re-throw unchanged.
|
|
100
|
+
if (!MeshRuntimeStore.loggedGetInstanceFailure) {
|
|
101
|
+
MeshRuntimeStore.loggedGetInstanceFailure = true;
|
|
102
|
+
LOG.warn(
|
|
103
|
+
'MeshRuntimeStore',
|
|
104
|
+
`getInstance failed; callers will degrade to JSONL-only: ${err?.message || err}`,
|
|
105
|
+
);
|
|
106
|
+
}
|
|
107
|
+
throw err;
|
|
108
|
+
}
|
|
75
109
|
}
|
|
76
110
|
return this.instance;
|
|
77
111
|
}
|
|
@@ -350,6 +350,14 @@ export interface MeshWorkQueueEntry {
|
|
|
350
350
|
};
|
|
351
351
|
/** ISO timestamp when the task was dispatched (assigned) to a node/session. Used for precise matching on completion. */
|
|
352
352
|
dispatchTimestamp?: string;
|
|
353
|
+
/**
|
|
354
|
+
* (3) The ORIGINATING coordinator session that enqueued this task. Stamped onto the
|
|
355
|
+
* worker at dispatch (meshCoordinatorSessionId) so the task's completion routes back to
|
|
356
|
+
* the exact coordinator session — even when several coordinator sessions share one
|
|
357
|
+
* daemon. Rides in the queue payload JSON (no column migration); absent on legacy rows
|
|
358
|
+
* → daemon-level routing fallback (backward + version-skew safe).
|
|
359
|
+
*/
|
|
360
|
+
sourceCoordinatorSessionId?: string;
|
|
353
361
|
createdAt: string;
|
|
354
362
|
updatedAt: string;
|
|
355
363
|
}
|
|
@@ -566,6 +574,8 @@ export function enqueueTask(
|
|
|
566
574
|
missionId?: string;
|
|
567
575
|
/** Explicit task id for batch/template flows (M5). Random UUID when omitted. */
|
|
568
576
|
id?: string;
|
|
577
|
+
/** (3) Originating coordinator session id (for session-anchored completion routing). */
|
|
578
|
+
sourceCoordinatorSessionId?: string;
|
|
569
579
|
} & MeshQueueMutationOptions,
|
|
570
580
|
): MeshWorkQueueEntry {
|
|
571
581
|
requireMeshHostQueueOwner(opts);
|
|
@@ -603,6 +613,9 @@ export function enqueueTask(
|
|
|
603
613
|
requiredTags: resolvedRequiredTags,
|
|
604
614
|
...(dependsOn.length > 0 ? { dependsOn } : {}),
|
|
605
615
|
...(typeof opts?.missionId === 'string' && opts.missionId.trim() ? { missionId: opts.missionId.trim() } : {}),
|
|
616
|
+
...(typeof opts?.sourceCoordinatorSessionId === 'string' && opts.sourceCoordinatorSessionId.trim()
|
|
617
|
+
? { sourceCoordinatorSessionId: opts.sourceCoordinatorSessionId.trim() }
|
|
618
|
+
: {}),
|
|
606
619
|
createdAt: new Date().toISOString(),
|
|
607
620
|
updatedAt: new Date().toISOString(),
|
|
608
621
|
};
|
|
@@ -879,7 +879,7 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
879
879
|
* completion events silently drop because the forwarder has nothing to
|
|
880
880
|
* match against.
|
|
881
881
|
*/
|
|
882
|
-
attachMeshAssignment(assignment: { meshId: string; nodeId?: string; taskId?: string; coordinatorDaemonId?: string }): void {
|
|
882
|
+
attachMeshAssignment(assignment: { meshId: string; nodeId?: string; taskId?: string; coordinatorDaemonId?: string; coordinatorSessionId?: string }): void {
|
|
883
883
|
if (!assignment?.meshId) return;
|
|
884
884
|
this.settings = {
|
|
885
885
|
...this.settings,
|
|
@@ -887,6 +887,9 @@ export class CliProviderInstance implements ProviderInstance {
|
|
|
887
887
|
...(assignment.nodeId ? { meshNodeId: assignment.nodeId } : {}),
|
|
888
888
|
...(assignment.taskId ? { meshActiveTaskId: assignment.taskId } : {}),
|
|
889
889
|
...(assignment.coordinatorDaemonId ? { meshCoordinatorDaemonId: assignment.coordinatorDaemonId } : {}),
|
|
890
|
+
// Session-level routing anchor: the originating coordinator session, so this
|
|
891
|
+
// worker's completion events route back to the exact session that dispatched it.
|
|
892
|
+
...(assignment.coordinatorSessionId ? { meshCoordinatorSessionId: assignment.coordinatorSessionId } : {}),
|
|
890
893
|
};
|
|
891
894
|
this.adapter.updateRuntimeSettings?.(this.settings);
|
|
892
895
|
}
|
|
@@ -310,7 +310,7 @@ export class ProviderInstanceManager {
|
|
|
310
310
|
* --direct so the worker's completion event has a coordinator routing
|
|
311
311
|
* marker in state.settings). Returns true if the instance existed and
|
|
312
312
|
* the stamp was applied. */
|
|
313
|
-
attachMeshAssignmentToInstance(instanceId: string, assignment: { meshId: string; nodeId?: string; taskId?: string; coordinatorDaemonId?: string }): boolean {
|
|
313
|
+
attachMeshAssignmentToInstance(instanceId: string, assignment: { meshId: string; nodeId?: string; taskId?: string; coordinatorDaemonId?: string; coordinatorSessionId?: string }): boolean {
|
|
314
314
|
const inst = this.instances.get(instanceId);
|
|
315
315
|
if (!inst || typeof inst.attachMeshAssignment !== 'function') {
|
|
316
316
|
try {
|
|
@@ -220,7 +220,7 @@ export interface ProviderInstance {
|
|
|
220
220
|
/** Stamp a direct-dispatch mesh task assignment so generating_completed
|
|
221
221
|
* events route back to the originating coordinator. Cleared by
|
|
222
222
|
* detachMeshAssignment when the task reaches a terminal state. */
|
|
223
|
-
attachMeshAssignment?(assignment: { meshId: string; nodeId?: string; taskId?: string }): void;
|
|
223
|
+
attachMeshAssignment?(assignment: { meshId: string; nodeId?: string; taskId?: string; coordinatorDaemonId?: string; coordinatorSessionId?: string }): void;
|
|
224
224
|
detachMeshAssignment?(): void;
|
|
225
225
|
|
|
226
226
|
/** Refresh static provider definition/scripts without restarting the live runtime. */
|
|
@@ -145,13 +145,23 @@ function countNewlines(s: string): number {
|
|
|
145
145
|
|
|
146
146
|
const SUBMIT_DELAY_FLOOR_MS = 200;
|
|
147
147
|
|
|
148
|
-
// win32 ConPTY submit reliability
|
|
149
|
-
//
|
|
150
|
-
//
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
-
//
|
|
154
|
-
|
|
148
|
+
// win32 ConPTY submit reliability. A MULTILINE message creates an Ink
|
|
149
|
+
// paste/newline-accumulation window during which a lone CR is absorbed as a
|
|
150
|
+
// literal newline instead of submitting; the window's length is
|
|
151
|
+
// nondeterministic (observed 0–~2s, driven by ConPTY byte timing), so neither a
|
|
152
|
+
// fixed delay nor a fixed CR count reliably submits. (A/B PTY testing on win32
|
|
153
|
+
// ConPTY: single-line submits on the FIRST CR; multiline needs a *variable*
|
|
154
|
+
// number of CRs as the window expires — a fixed double-CR fails outright, and
|
|
155
|
+
// bracketed-paste wrapping does NOT help.) So we VERIFY instead of guessing:
|
|
156
|
+
// write the text, then resend the submit key on a fixed cadence until the FSM
|
|
157
|
+
// observes the agent has actually left the idle composer (status flips away from
|
|
158
|
+
// 'idle' — i.e. it submitted and is generating / showing a modal), bounded by a
|
|
159
|
+
// retry budget. Once submission is observed we stop so we don't spam Enter into
|
|
160
|
+
// the next turn. Single-line messages satisfy the check after the first CR, so
|
|
161
|
+
// their behaviour is unchanged. CRs absorbed as newlines during the window are
|
|
162
|
+
// trimmed by the TUI on submit.
|
|
163
|
+
const WIN32_SUBMIT_RESEND_GAP_MS = 350;
|
|
164
|
+
const WIN32_SUBMIT_MAX_RESENDS = 14;
|
|
155
165
|
|
|
156
166
|
export function resolveSubmitDelayMs(specBeforeSubmit: number | undefined, text: string): number {
|
|
157
167
|
const lines = countNewlines(text);
|
|
@@ -217,6 +227,10 @@ export class FsmDriver implements ISpecDriver {
|
|
|
217
227
|
* after a re-prime we don't re-inject until the screen changes (which
|
|
218
228
|
* resets the stall reference) or another full stall window lapses. */
|
|
219
229
|
private lastRefocusAt = 0;
|
|
230
|
+
/** Timer driving the win32 verification-based submit resend loop (see
|
|
231
|
+
* WIN32_SUBMIT_* and scheduleWin32Submit). Re-arms itself until the FSM
|
|
232
|
+
* leaves idle (submitted) or the resend budget is spent. */
|
|
233
|
+
private win32SubmitTimer: ReturnType<typeof setTimeout> | null = null;
|
|
220
234
|
|
|
221
235
|
private currentEval: CurrentEval | null = null;
|
|
222
236
|
private stateHistory: HistoryEntry[] = [];
|
|
@@ -334,6 +348,7 @@ export class FsmDriver implements ISpecDriver {
|
|
|
334
348
|
this.delegateTimers.clear();
|
|
335
349
|
if (this.wakeTimer) { clearTimeout(this.wakeTimer); this.wakeTimer = null; }
|
|
336
350
|
if (this.stallTimer) { clearTimeout(this.stallTimer); this.stallTimer = null; }
|
|
351
|
+
if (this.win32SubmitTimer) { clearTimeout(this.win32SubmitTimer); this.win32SubmitTimer = null; }
|
|
337
352
|
this.specWatcher?.close();
|
|
338
353
|
this.adapter.kill();
|
|
339
354
|
}
|
|
@@ -841,25 +856,19 @@ export class FsmDriver implements ISpecDriver {
|
|
|
841
856
|
const beforeSubmit = resolveSubmitDelayMs(sm.delay_ms_before_submit, text);
|
|
842
857
|
|
|
843
858
|
// win32 ConPTY submit: the text and the submit key (CR) must NOT be
|
|
844
|
-
// combined into one PTY write
|
|
845
|
-
// write that carries text + a trailing CR as a bracketed/multi-line
|
|
846
|
-
// and absorb the CR as a literal newline
|
|
847
|
-
//
|
|
848
|
-
//
|
|
849
|
-
//
|
|
850
|
-
// paste-accumulation
|
|
851
|
-
//
|
|
852
|
-
//
|
|
853
|
-
//
|
|
854
|
-
// correctness of submission wins over the typing visual there.
|
|
859
|
+
// combined into one PTY write — Ink-based TUIs (claude-cli) treat a
|
|
860
|
+
// single write that carries text + a trailing CR as a bracketed/multi-line
|
|
861
|
+
// paste and absorb the CR as a literal newline. So we write the text on
|
|
862
|
+
// its own, then resend the submit key on a fixed cadence, VERIFYING after
|
|
863
|
+
// each that the agent actually left the idle composer (status flipped away
|
|
864
|
+
// from 'idle'). This handles the nondeterministic multiline
|
|
865
|
+
// paste-accumulation window where a variable number of CRs is needed; a
|
|
866
|
+
// fixed double-CR fails for multiline (see WIN32_SUBMIT_* above). perChar
|
|
867
|
+
// typing simulation is skipped on win32; correctness of submission wins
|
|
868
|
+
// over the typing visual there.
|
|
855
869
|
if (process.platform === 'win32') {
|
|
856
|
-
const submitTwice = (): void => {
|
|
857
|
-
this.adapter.send_keys(sm.submit_key);
|
|
858
|
-
setTimeout(() => this.adapter.send_keys(sm.submit_key), WIN32_SUBMIT_REPEAT_GAP_MS);
|
|
859
|
-
};
|
|
860
870
|
this.adapter.send_keys(text);
|
|
861
|
-
|
|
862
|
-
else submitTwice();
|
|
871
|
+
this.scheduleWin32Submit(sm.submit_key, beforeSubmit);
|
|
863
872
|
return;
|
|
864
873
|
}
|
|
865
874
|
|
|
@@ -881,6 +890,37 @@ export class FsmDriver implements ISpecDriver {
|
|
|
881
890
|
}, perChar);
|
|
882
891
|
}
|
|
883
892
|
|
|
893
|
+
/** The agent's current coarse status, derived from the FSM node we're in. */
|
|
894
|
+
private currentStatus(): 'idle' | 'generating' | 'approval' {
|
|
895
|
+
const st = stateById(this.spec, this.currentStateId);
|
|
896
|
+
return st ? statusForState(st) : 'idle';
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
/**
|
|
900
|
+
* win32 verification-based submit. Sends the submit key, waits a gap, and if
|
|
901
|
+
* the FSM is still 'idle' (the prompt did not submit — the CR was absorbed as
|
|
902
|
+
* a multiline-paste newline) resends, up to WIN32_SUBMIT_MAX_RESENDS. The
|
|
903
|
+
* first CR always fires (so a stale/edge status never suppresses the submit);
|
|
904
|
+
* subsequent resends are gated on still being idle, and stop the instant the
|
|
905
|
+
* agent leaves idle (submitted → generating / approval). This converges the
|
|
906
|
+
* nondeterministic multiline window without spamming Enter into the next turn.
|
|
907
|
+
*/
|
|
908
|
+
private scheduleWin32Submit(submitKey: string, initialDelayMs: number): void {
|
|
909
|
+
if (this.win32SubmitTimer) { clearTimeout(this.win32SubmitTimer); this.win32SubmitTimer = null; }
|
|
910
|
+
const fire = (attempt: number): void => {
|
|
911
|
+
this.win32SubmitTimer = null;
|
|
912
|
+
this.adapter.send_keys(submitKey);
|
|
913
|
+
if (attempt + 1 >= WIN32_SUBMIT_MAX_RESENDS) return;
|
|
914
|
+
this.win32SubmitTimer = setTimeout(() => {
|
|
915
|
+
// Left the idle composer → it submitted; stop resending.
|
|
916
|
+
if (this.currentStatus() !== 'idle') { this.win32SubmitTimer = null; return; }
|
|
917
|
+
fire(attempt + 1);
|
|
918
|
+
}, WIN32_SUBMIT_RESEND_GAP_MS);
|
|
919
|
+
};
|
|
920
|
+
if (initialDelayMs > 0) this.win32SubmitTimer = setTimeout(() => fire(0), initialDelayMs);
|
|
921
|
+
else fire(0);
|
|
922
|
+
}
|
|
923
|
+
|
|
884
924
|
private handleClickControl(controlId: string, payload?: unknown): void {
|
|
885
925
|
const ctl = (this.spec.control_bar ?? []).find(c => c.id === controlId);
|
|
886
926
|
if (!ctl) return;
|