@adhdev/daemon-core 0.9.82-rc.350 → 0.9.82-rc.352
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/upgrade-helper.d.ts +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +528 -216
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +526 -216
- package/dist/index.mjs.map +1 -1
- package/dist/logging/logger.d.ts +1 -1
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/mesh/mesh-runtime-store.d.ts +10 -0
- package/dist/mesh/mesh-work-queue.d.ts +28 -0
- package/dist/providers/approval-utils.d.ts +7 -0
- package/dist/providers/cli-provider-instance.d.ts +15 -0
- package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +1 -0
- package/package.json +2 -2
- package/src/commands/upgrade-helper.ts +57 -14
- package/src/index.ts +5 -0
- package/src/logging/command-log.ts +7 -5
- package/src/logging/logger.ts +12 -6
- package/src/mesh/mesh-active-work.ts +64 -20
- package/src/mesh/mesh-events-coordinator.ts +249 -89
- package/src/mesh/mesh-events-pending.ts +27 -19
- package/src/mesh/mesh-events-stale.ts +17 -0
- package/src/mesh/mesh-ledger.ts +1 -0
- package/src/mesh/mesh-reconcile-loop.ts +67 -7
- package/src/mesh/mesh-runtime-store.ts +22 -1
- package/src/mesh/mesh-task-stats.ts +8 -1
- package/src/mesh/mesh-work-queue.ts +89 -0
- package/src/providers/approval-utils.d.ts +1 -0
- package/src/providers/approval-utils.ts +10 -0
- package/src/providers/cli-provider-instance.ts +73 -19
- package/src/providers/sdk/v1/builders/cli/detect-status.ts +51 -0
|
@@ -7,6 +7,7 @@ import { LOG } from '../logging/logger.js';
|
|
|
7
7
|
import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
|
|
8
8
|
import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
|
|
9
9
|
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents } from './mesh-work-queue.js';
|
|
10
|
+
import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
|
|
10
11
|
import { fastForwardMeshNode } from './mesh-fast-forward.js';
|
|
11
12
|
import { createSessionDelivery, markSessionDeliveriesTerminal, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
|
|
12
13
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
@@ -17,7 +18,7 @@ import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUn
|
|
|
17
18
|
import { getLastDisplayMessage } from '../status/snapshot.js';
|
|
18
19
|
import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
19
20
|
import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
20
|
-
import { normalizeMeshNodeId, meshNodeIdMatches, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
21
|
+
import { normalizeMeshNodeId, meshNodeIdMatches, expandDaemonIdForms, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
21
22
|
import {
|
|
22
23
|
findRecentTerminalLedgerEvidence,
|
|
23
24
|
hasDispatchAfterTerminal,
|
|
@@ -35,16 +36,16 @@ import {
|
|
|
35
36
|
} from './mesh-events-utils.js';
|
|
36
37
|
|
|
37
38
|
// The set of coordinator-daemon ids this daemon answers to when draining the
|
|
38
|
-
// pending-events queue
|
|
39
|
-
//
|
|
40
|
-
//
|
|
39
|
+
// pending-events queue. Mirrors resolveCoordinatorDaemonIds in mesh-reconcile-loop:
|
|
40
|
+
// a unicast event may be stamped with the status id, the bare machineId, OR the
|
|
41
|
+
// config-form node daemonId (`daemon_<machineId>`) depending on which dispatch path
|
|
42
|
+
// created the worker. We expand to EVERY equivalent form so a `daemon_<machineId>`
|
|
43
|
+
// completion matches a coordinator that knows itself as bare `<machineId>` (the
|
|
44
|
+
// base-node completion-surface bug) and vice versa.
|
|
41
45
|
function resolveCoordinatorDrainDaemonIds(components: DaemonComponents): string[] {
|
|
42
|
-
const ids = new Set<string>();
|
|
43
46
|
const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
|
|
44
|
-
if (statusInstanceId) ids.add(statusInstanceId);
|
|
45
47
|
const machineId = readNonEmptyString(loadConfig().machineId);
|
|
46
|
-
|
|
47
|
-
return [...ids];
|
|
48
|
+
return expandDaemonIdForms([statusInstanceId, machineId]);
|
|
48
49
|
}
|
|
49
50
|
|
|
50
51
|
// ---------------------------------------------------------------------------
|
|
@@ -251,10 +252,148 @@ function isDuplicateRefineTerminalEvent(meshId: string, eventName: string, metad
|
|
|
251
252
|
return false;
|
|
252
253
|
}
|
|
253
254
|
|
|
255
|
+
// A worker/coordinator "false idle": the provider dropped to idle WITHOUT a confirmed
|
|
256
|
+
// final assistant message for the turn (a finalization timeout, or a "scheduled fallback"
|
|
257
|
+
// idle). This is the signal cli-provider-instance emits as
|
|
258
|
+
// completionDiagnostic.blockReason='missing_final_assistant' / finalAssistantPresent=false.
|
|
259
|
+
// Such a completion is NOT trustworthy terminal evidence: it must neither permanently
|
|
260
|
+
// terminate a direct-dispatch task nor suppress the genuine completion a later turn
|
|
261
|
+
// (commonly driven by a coordinator nudge / re-dispatch) produces.
|
|
262
|
+
function isFalseIdleCompletion(metadataEvent: Record<string, unknown>): boolean {
|
|
263
|
+
const diag = readRecord(metadataEvent.completionDiagnostic);
|
|
264
|
+
if (!diag) return false;
|
|
265
|
+
return diag.finalAssistantPresent === false || diag.blockReason === 'missing_final_assistant';
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// The genuine-completion counterpart: a real final summary / worker result is present and
|
|
269
|
+
// the completion is not flagged as a missing-final-assistant false idle. Used to decide
|
|
270
|
+
// whether a new completion may supersede a prior WEAK (false-idle) terminal.
|
|
271
|
+
function isGenuineCompletionEvidence(metadataEvent: Record<string, unknown>): boolean {
|
|
272
|
+
if (isFalseIdleCompletion(metadataEvent)) return false;
|
|
273
|
+
return !!readWorkerResultMetadata(metadataEvent) || !!readNonEmptyString(metadataEvent.finalSummary);
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
// True when a terminal ledger payload was recorded from WEAK completion evidence (a false
|
|
277
|
+
// idle): insufficient evidence level, review-recommended, or a missing-final-assistant
|
|
278
|
+
// completion diagnostic. A weak terminal is non-authoritative — a later genuine completion
|
|
279
|
+
// (live path) or a transcript reconcile (fallback path) may supersede it.
|
|
280
|
+
function isWeakTerminalLedgerPayload(payload: Record<string, unknown> | undefined): boolean {
|
|
281
|
+
if (!payload) return false;
|
|
282
|
+
if (payload.evidenceLevel === 'insufficient' || payload.reviewRecommended === true) return true;
|
|
283
|
+
const diag = readRecord(payload.completionDiagnostic);
|
|
284
|
+
return diag?.finalAssistantPresent === false || diag?.blockReason === 'missing_final_assistant';
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
// The latest still-active direct-dispatch taskId for a session, resolved BEFORE the
|
|
288
|
+
// completion flips the dispatch row terminal. Direct dispatches (mesh_send_task) have no
|
|
289
|
+
// work-queue row, so this is the only taskId available to attribute the terminal ledger
|
|
290
|
+
// entry (and thus mesh task-stats) to — without it the terminal carries no taskId and the
|
|
291
|
+
// task surfaces as status='unknown' / terminalKind=null in computeMeshTaskStats.
|
|
292
|
+
function resolveActiveDirectDispatchTaskId(meshId: string, sessionId: string): string | undefined {
|
|
293
|
+
try {
|
|
294
|
+
const matches = getActiveDirectDispatches(meshId).filter(d => d.sessionId === sessionId);
|
|
295
|
+
if (!matches.length) return undefined;
|
|
296
|
+
// getActiveDirectDispatches returns rows ordered by dispatched_at ASC; the last is
|
|
297
|
+
// the most recent dispatch (the re-dispatch / nudge whose completion this is).
|
|
298
|
+
return readNonEmptyString(matches[matches.length - 1].taskId) || undefined;
|
|
299
|
+
} catch {
|
|
300
|
+
return undefined;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
|
|
254
304
|
// ---------------------------------------------------------------------------
|
|
255
305
|
// Queue assignment
|
|
256
306
|
// ---------------------------------------------------------------------------
|
|
257
307
|
|
|
308
|
+
// Per-dispatch confirmation timeout (Bug B). A dispatch promise that never settles —
|
|
309
|
+
// a saturated remote P2P relay that hangs, or a transport that resolves only after
|
|
310
|
+
// the worker acks — would otherwise leave the just-claimed queue row 'assigned' with
|
|
311
|
+
// its delivery stuck 'delivering' forever: the .catch that requeues never fires, and
|
|
312
|
+
// PHASE 3 reconcile skips the row (it counts 0 pending). Racing the dispatch against
|
|
313
|
+
// this timeout guarantees a hung dispatch deterministically returns the task to
|
|
314
|
+
// 'pending' for re-dispatch. Generous so a merely-slow-but-live dispatch (a cold
|
|
315
|
+
// remote relay) is never reclaimed early; the reconcile assigned-stranded watchdog is
|
|
316
|
+
// the durable cross-restart backstop for a timer lost to a daemon restart.
|
|
317
|
+
const DISPATCH_CONFIRM_TIMEOUT_MS = 120_000;
|
|
318
|
+
|
|
319
|
+
interface DeliverTaskContext {
|
|
320
|
+
meshId: string;
|
|
321
|
+
nodeId: string;
|
|
322
|
+
sessionId: string;
|
|
323
|
+
providerType: string;
|
|
324
|
+
task: MeshWorkQueueEntry;
|
|
325
|
+
transport: 'remote' | 'local';
|
|
326
|
+
sourceCoordinatorSessionId?: string;
|
|
327
|
+
sourceCoordinatorDaemonId?: string;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// CONS scope 3: the SINGLE source of truth for dispatching a claimed task to its
|
|
331
|
+
// session. The remote (P2P dispatchMeshCommand) and local (cliManager.handleCliCommand)
|
|
332
|
+
// branches differ ONLY in the transport call — the delivery record, the delivered/failed
|
|
333
|
+
// transitions, the pending-requeue-on-failure, the dispatch_failed ledger entry, AND the
|
|
334
|
+
// Bug B hang timeout are identical and live here once so a future change to the dispatch
|
|
335
|
+
// lifecycle cannot drift between the two paths. The caller passes a `dispatchThunk` that
|
|
336
|
+
// performs only the transport-specific send and returns its promise.
|
|
337
|
+
function deliverTaskToSession(dispatchThunk: () => Promise<unknown>, ctx: DeliverTaskContext): void {
|
|
338
|
+
const delivery = createSessionDelivery({
|
|
339
|
+
meshId: ctx.meshId,
|
|
340
|
+
nodeId: ctx.nodeId,
|
|
341
|
+
sessionId: ctx.sessionId,
|
|
342
|
+
providerType: ctx.providerType,
|
|
343
|
+
taskId: ctx.task.id,
|
|
344
|
+
kind: 'task',
|
|
345
|
+
message: ctx.task.message,
|
|
346
|
+
status: 'delivering',
|
|
347
|
+
...(ctx.sourceCoordinatorSessionId ? { sourceCoordinatorSessionId: ctx.sourceCoordinatorSessionId } : {}),
|
|
348
|
+
...(ctx.sourceCoordinatorDaemonId ? { sourceCoordinatorDaemonId: ctx.sourceCoordinatorDaemonId } : {}),
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
// Invoke the transport synchronously (preserves the prior fire-and-forget timing,
|
|
352
|
+
// and lets a synchronous throw fall into the same failure path as a rejection).
|
|
353
|
+
let dispatchPromise: Promise<unknown>;
|
|
354
|
+
try {
|
|
355
|
+
dispatchPromise = Promise.resolve(dispatchThunk());
|
|
356
|
+
} catch (e) {
|
|
357
|
+
dispatchPromise = Promise.reject(e);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
361
|
+
const guarded = Promise.race([
|
|
362
|
+
dispatchPromise,
|
|
363
|
+
new Promise<never>((_, reject) => {
|
|
364
|
+
timer = setTimeout(
|
|
365
|
+
() => reject(new Error(`dispatch_confirm_timeout after ${DISPATCH_CONFIRM_TIMEOUT_MS}ms`)),
|
|
366
|
+
DISPATCH_CONFIRM_TIMEOUT_MS,
|
|
367
|
+
);
|
|
368
|
+
// Never keep the process alive solely for this confirm-timeout timer.
|
|
369
|
+
if (typeof (timer as { unref?: () => void })?.unref === 'function') (timer as { unref: () => void }).unref();
|
|
370
|
+
}),
|
|
371
|
+
]);
|
|
372
|
+
|
|
373
|
+
guarded.then(() => {
|
|
374
|
+
if (timer) clearTimeout(timer);
|
|
375
|
+
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
376
|
+
}).catch((e: any) => {
|
|
377
|
+
if (timer) clearTimeout(timer);
|
|
378
|
+
// A dispatch failure (transport reject OR hang timeout) is most often transient —
|
|
379
|
+
// a busy/refusing adapter, or a relay that never acked — not a permanent task
|
|
380
|
+
// failure. Marking the task terminal here would permanently kill tasks a later
|
|
381
|
+
// tick delivers fine. Return it to 'pending' and record a retryable dispatch_failed
|
|
382
|
+
// ledger entry so the reconcile loop re-dispatches it. Identical for both transports.
|
|
383
|
+
LOG.error('MeshQueue', `Failed to dispatch task via ${ctx.transport} to node ${ctx.nodeId}: ${e?.message}`);
|
|
384
|
+
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
385
|
+
updateTaskStatus(ctx.meshId, ctx.task.id, 'pending');
|
|
386
|
+
try {
|
|
387
|
+
appendLedgerEntry(ctx.meshId, {
|
|
388
|
+
kind: 'dispatch_failed' as any,
|
|
389
|
+
nodeId: ctx.nodeId,
|
|
390
|
+
sessionId: ctx.sessionId,
|
|
391
|
+
payload: { taskId: ctx.task.id, deliveryId: delivery.id, error: e?.message, retryable: true, transport: ctx.transport },
|
|
392
|
+
});
|
|
393
|
+
} catch { /* ledger write is best-effort */ }
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
|
|
258
397
|
export function tryAssignQueueTask(
|
|
259
398
|
components: DaemonComponents,
|
|
260
399
|
meshId: string,
|
|
@@ -288,45 +427,36 @@ export function tryAssignQueueTask(
|
|
|
288
427
|
// completion back to that exact session (multi-coordinator). Carried over P2P
|
|
289
428
|
// to the remote worker, which echoes it on its completion event.
|
|
290
429
|
const sourceCoordinatorSessionId = readNonEmptyString(task.sourceCoordinatorSessionId) || undefined;
|
|
291
|
-
const
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
430
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
431
|
+
const remoteDaemonId = node.daemonId;
|
|
432
|
+
// CONS3: only the transport call differs — everything else (delivery record,
|
|
433
|
+
// status transitions, requeue-on-failure, ledger, Bug B hang timeout) is in
|
|
434
|
+
// the shared deliverTaskToSession helper.
|
|
435
|
+
deliverTaskToSession(
|
|
436
|
+
() => dispatchMeshCommand(remoteDaemonId, 'agent_command', {
|
|
437
|
+
targetSessionId: sessionId,
|
|
438
|
+
cliType: providerType,
|
|
439
|
+
action: 'send_chat',
|
|
440
|
+
message: task.message,
|
|
441
|
+
meshContext: {
|
|
442
|
+
meshId,
|
|
443
|
+
nodeId,
|
|
444
|
+
taskId: task.id,
|
|
445
|
+
...(localDaemonIdForDispatch ? { coordinatorDaemonId: localDaemonIdForDispatch } : {}),
|
|
446
|
+
...(sourceCoordinatorSessionId ? { coordinatorSessionId: sourceCoordinatorSessionId } : {}),
|
|
447
|
+
},
|
|
448
|
+
}),
|
|
449
|
+
{
|
|
309
450
|
meshId,
|
|
310
451
|
nodeId,
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
452
|
+
sessionId,
|
|
453
|
+
providerType,
|
|
454
|
+
task,
|
|
455
|
+
transport: 'remote',
|
|
456
|
+
...(sourceCoordinatorSessionId ? { sourceCoordinatorSessionId } : {}),
|
|
457
|
+
...(localDaemonIdForDispatch ? { sourceCoordinatorDaemonId: localDaemonIdForDispatch } : {}),
|
|
314
458
|
},
|
|
315
|
-
|
|
316
|
-
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
317
|
-
}).catch((e: any) => {
|
|
318
|
-
LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
|
|
319
|
-
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
320
|
-
updateTaskStatus(meshId, task.id, 'pending');
|
|
321
|
-
try {
|
|
322
|
-
appendLedgerEntry(meshId, {
|
|
323
|
-
kind: 'dispatch_failed' as any,
|
|
324
|
-
nodeId,
|
|
325
|
-
sessionId,
|
|
326
|
-
payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
|
|
327
|
-
});
|
|
328
|
-
} catch { /* ledger write is best-effort */ }
|
|
329
|
-
});
|
|
459
|
+
);
|
|
330
460
|
return true;
|
|
331
461
|
}
|
|
332
462
|
}
|
|
@@ -363,44 +493,26 @@ export function tryAssignQueueTask(
|
|
|
363
493
|
}
|
|
364
494
|
} catch { /* best-effort — dispatch still proceeds */ }
|
|
365
495
|
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
// Mirror the remote-dispatch catch above: a local dispatch failure is most often a
|
|
387
|
-
// transient busy/refusal (e.g. the adapter rejected send_chat while mid-generation),
|
|
388
|
-
// not a permanent task failure. Marking the task terminal 'failed' here with no ledger
|
|
389
|
-
// and no retry permanently killed tasks that a later tick would have delivered fine.
|
|
390
|
-
// Return the task to 'pending' and record a retryable dispatch_failed ledger entry so
|
|
391
|
-
// the reconcile loop re-dispatches it, exactly as the remote branch does.
|
|
392
|
-
LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
|
|
393
|
-
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
394
|
-
updateTaskStatus(meshId, task.id, 'pending');
|
|
395
|
-
try {
|
|
396
|
-
appendLedgerEntry(meshId, {
|
|
397
|
-
kind: 'dispatch_failed' as any,
|
|
398
|
-
nodeId,
|
|
399
|
-
sessionId,
|
|
400
|
-
payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
|
|
401
|
-
});
|
|
402
|
-
} catch { /* ledger write is best-effort */ }
|
|
403
|
-
});
|
|
496
|
+
// CONS3: same shared dispatch lifecycle as the remote branch — only the transport
|
|
497
|
+
// (cliManager.handleCliCommand) differs.
|
|
498
|
+
deliverTaskToSession(
|
|
499
|
+
() => components.cliManager.handleCliCommand('agent_command', {
|
|
500
|
+
targetSessionId: sessionId,
|
|
501
|
+
cliType: providerType,
|
|
502
|
+
action: 'send_chat',
|
|
503
|
+
message: task.message,
|
|
504
|
+
}),
|
|
505
|
+
{
|
|
506
|
+
meshId,
|
|
507
|
+
nodeId,
|
|
508
|
+
sessionId,
|
|
509
|
+
providerType,
|
|
510
|
+
task,
|
|
511
|
+
transport: 'local',
|
|
512
|
+
...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { sourceCoordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
|
|
513
|
+
...(readNonEmptyString(loadConfig().machineId) ? { sourceCoordinatorDaemonId: readNonEmptyString(loadConfig().machineId) } : {}),
|
|
514
|
+
},
|
|
515
|
+
);
|
|
404
516
|
|
|
405
517
|
return true;
|
|
406
518
|
}
|
|
@@ -880,7 +992,12 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
880
992
|
|
|
881
993
|
const candidateNodes = Array.isArray(mesh?.nodes)
|
|
882
994
|
? mesh.nodes.filter((node: any) => {
|
|
883
|
-
|
|
995
|
+
// Bug A: match the target pin with the shared 3-form (id / nodeId / node_id)
|
|
996
|
+
// normalizer, mirroring the remote-idle drain (meshNodeIdMatches at the
|
|
997
|
+
// getRemoteIdleSessions filter). A strict `readMeshNodeId(node) !== targetNodeId`
|
|
998
|
+
// dropped a target node whose identity arrived under a different form (a freshly
|
|
999
|
+
// mesh_clone_node'd worktree), emptying candidateNodes and mislabelling the skip.
|
|
1000
|
+
if (task.targetNodeId && !meshNodeIdMatches(node, task.targetNodeId)) return false;
|
|
884
1001
|
// Skip nodes that can never satisfy requiredTags regardless of which provider
|
|
885
1002
|
// from providerPriority is selected. A node satisfies tags if at least one
|
|
886
1003
|
// provider in its priority list would produce matching capability tags.
|
|
@@ -895,7 +1012,20 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
895
1012
|
})
|
|
896
1013
|
: [];
|
|
897
1014
|
if (!candidateNodes.length) {
|
|
898
|
-
|
|
1015
|
+
// Bug A: distinguish the two ways the candidate set empties. A task pinned to a
|
|
1016
|
+
// targetNodeId whose node is absent from the mesh (or whose id arrived under a
|
|
1017
|
+
// different form) is a ROUTING miss — report it as `target_node_id_unmatched`, not
|
|
1018
|
+
// the hard-coded `no_node_satisfies_required_tags`, which mislabelled a 3-form
|
|
1019
|
+
// node-id mismatch as a capability failure and sent diagnosis down the wrong path.
|
|
1020
|
+
// Only fall back to the tag reason when no target pin is in play, or the pin DID
|
|
1021
|
+
// match a node but its tags excluded it (a genuine capability miss).
|
|
1022
|
+
const targetPinUnmatched = !!task.targetNodeId
|
|
1023
|
+
&& !(Array.isArray(mesh?.nodes) && mesh.nodes.some((n: any) => meshNodeIdMatches(n, task.targetNodeId)));
|
|
1024
|
+
markAutoLaunch(meshId, task.id, {
|
|
1025
|
+
status: 'skipped',
|
|
1026
|
+
reason: targetPinUnmatched ? 'target_node_id_unmatched' : 'no_node_satisfies_required_tags',
|
|
1027
|
+
nodeId: task.targetNodeId,
|
|
1028
|
+
});
|
|
899
1029
|
continue;
|
|
900
1030
|
}
|
|
901
1031
|
|
|
@@ -1555,7 +1685,17 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1555
1685
|
});
|
|
1556
1686
|
if (terminal?.kind === 'task_completed' && !sessionHasActiveAssignment(args.meshId, eventSessionId)) {
|
|
1557
1687
|
const newDispatchAfterTerminal = hasDispatchAfterTerminal(args.meshId, eventSessionId, terminal.id);
|
|
1558
|
-
|
|
1688
|
+
// Fix B (re-dispatch 2nd-completion routing): a prior terminal recorded from a FALSE
|
|
1689
|
+
// idle (weak evidence / no confirmed final assistant) must NOT permanently suppress a
|
|
1690
|
+
// later GENUINE completion of the same session. providerSessionId is stable across a
|
|
1691
|
+
// session's turns, so the providerSessionId/finalSummary dedup below would otherwise
|
|
1692
|
+
// swallow the real 2nd-turn completion that a coordinator nudge (direct re-dispatch)
|
|
1693
|
+
// drove — exactly the missed-event bug. When the prior terminal was weak and the new
|
|
1694
|
+
// event carries genuine completion evidence, let it through so it is recorded and
|
|
1695
|
+
// re-attributed to the latest task (the normal task_completed path below).
|
|
1696
|
+
const supersedesWeakTerminal = isWeakTerminalLedgerPayload(terminal.payload)
|
|
1697
|
+
&& isGenuineCompletionEvidence(args.metadataEvent);
|
|
1698
|
+
if (!newDispatchAfterTerminal && !supersedesWeakTerminal) {
|
|
1559
1699
|
const terminalProviderSessionId = readNonEmptyString(terminal.payload.providerSessionId);
|
|
1560
1700
|
const terminalFinalSummary = readNonEmptyString(terminal.payload.finalSummary);
|
|
1561
1701
|
const eventProviderSessionId = readNonEmptyString(args.metadataEvent.providerSessionId);
|
|
@@ -1606,7 +1746,7 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1606
1746
|
}
|
|
1607
1747
|
}
|
|
1608
1748
|
|
|
1609
|
-
function markSessionTerminal(sessionId: string, outcome: 'completed' | 'failed', occurredAtMs?: number | null): { id?: string } | null {
|
|
1749
|
+
function markSessionTerminal(sessionId: string, outcome: 'completed' | 'failed', occurredAtMs?: number | null, opts?: { tentativeIfDirect?: boolean }): { id?: string } | null {
|
|
1610
1750
|
// C2: prefer an exact taskId match when the completion event carries one —
|
|
1611
1751
|
// it's immune to coordinator↔worker clock skew that can hide the assigned row.
|
|
1612
1752
|
const eventTaskId = readNonEmptyString(args.metadataEvent.taskId) || undefined;
|
|
@@ -1614,20 +1754,36 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1614
1754
|
occurredAt: occurredAtMs != null ? new Date(occurredAtMs).toISOString() : undefined,
|
|
1615
1755
|
taskId: eventTaskId,
|
|
1616
1756
|
});
|
|
1617
|
-
|
|
1757
|
+
// Fix A (early-terminal prevention): a false-idle completion (no confirmed final
|
|
1758
|
+
// assistant) for a DIRECT dispatch — i.e. no work-queue row matched — must not flip the
|
|
1759
|
+
// dispatch row terminal. Leaving it active lets the reconcile loop (PHASE 4) re-read the
|
|
1760
|
+
// transcript and record the genuine completion once the worker truly finishes (commonly
|
|
1761
|
+
// after a coordinator nudge / re-dispatch). A matched queue task, or a completion with
|
|
1762
|
+
// genuine evidence, is marked terminal as before.
|
|
1763
|
+
const leaveDirectDispatchActive = !task && opts?.tentativeIfDirect === true;
|
|
1764
|
+
if (!leaveDirectDispatchActive) {
|
|
1765
|
+
updateDirectDispatchStatus(args.meshId, sessionId, outcome);
|
|
1766
|
+
}
|
|
1618
1767
|
markSessionDeliveriesTerminal(args.meshId, sessionId, outcome);
|
|
1619
1768
|
setImmediate(() => cleanupTerminalDirectDispatches());
|
|
1620
1769
|
return task ? { id: task.id } : null;
|
|
1621
1770
|
}
|
|
1622
1771
|
|
|
1623
1772
|
let completedTaskForLedger: { id?: string } | null = null;
|
|
1773
|
+
// Fix B: direct-dispatch taskId used to attribute the terminal ledger entry when no
|
|
1774
|
+
// work-queue row matches (resolved BEFORE markSessionTerminal flips the dispatch terminal).
|
|
1775
|
+
let directDispatchTaskIdForLedger: string | undefined;
|
|
1624
1776
|
if (args.event === 'agent:generating_completed') {
|
|
1625
1777
|
const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
|
|
1626
1778
|
const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
|
|
1627
1779
|
const providerType = readNonEmptyString(args.metadataEvent.providerType);
|
|
1628
1780
|
|
|
1629
1781
|
if (sessionId) {
|
|
1630
|
-
|
|
1782
|
+
directDispatchTaskIdForLedger = resolveActiveDirectDispatchTaskId(args.meshId, sessionId);
|
|
1783
|
+
// A false-idle completion of a direct dispatch is recorded but kept tentative (the
|
|
1784
|
+
// dispatch row stays active for the reconcile fallback); a genuine completion is terminal.
|
|
1785
|
+
const isFalseIdle = isFalseIdleCompletion(args.metadataEvent);
|
|
1786
|
+
completedTaskForLedger = markSessionTerminal(sessionId, 'completed', eventTimestamp, { tentativeIfDirect: isFalseIdle });
|
|
1631
1787
|
if (nodeId && providerType) {
|
|
1632
1788
|
runIdleMaintenanceThenAssignQueue(components, { meshId: args.meshId, nodeId, sessionId, providerType });
|
|
1633
1789
|
}
|
|
@@ -1729,6 +1885,7 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1729
1885
|
} catch { /* best-effort */ }
|
|
1730
1886
|
}
|
|
1731
1887
|
if (sessionId) {
|
|
1888
|
+
directDispatchTaskIdForLedger = resolveActiveDirectDispatchTaskId(args.meshId, sessionId);
|
|
1732
1889
|
completedTaskForLedger = markSessionTerminal(sessionId, 'failed');
|
|
1733
1890
|
}
|
|
1734
1891
|
}
|
|
@@ -1761,7 +1918,10 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
|
|
|
1761
1918
|
payload: {
|
|
1762
1919
|
event: args.event,
|
|
1763
1920
|
nodeLabel: args.nodeLabel,
|
|
1764
|
-
|
|
1921
|
+
// Fix B: fall back to the direct-dispatch taskId when no work-queue row
|
|
1922
|
+
// matched, so the terminal entry is attributable in mesh task-stats
|
|
1923
|
+
// (otherwise the direct task shows status='unknown' / terminalKind=null).
|
|
1924
|
+
taskId: completedTaskForLedger?.id || directDispatchTaskIdForLedger || undefined,
|
|
1765
1925
|
providerSessionId,
|
|
1766
1926
|
finalSummary,
|
|
1767
1927
|
workerResult,
|
|
@@ -5,6 +5,7 @@ import { LOG } from '../logging/logger.js';
|
|
|
5
5
|
import { getLedgerDir, readLedgerEntries, appendLedgerEntry } from './mesh-ledger.js';
|
|
6
6
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
7
7
|
import { buildMeshSystemMessage, readNonEmptyString, readRecord, resolveEventSessionId, readMeshCompletionSummary } from './mesh-events-utils.js';
|
|
8
|
+
import { expandDaemonIdForms } from '@adhdev/mesh-shared';
|
|
8
9
|
|
|
9
10
|
// ---------------------------------------------------------------------------
|
|
10
11
|
// MCP coordinator pending-event queue — FILE-BASED PERSISTENCE
|
|
@@ -48,24 +49,22 @@ export interface PendingMeshCoordinatorEvent {
|
|
|
48
49
|
const REFINE_TERMINAL_EVENTS = new Set(['refine:completed', 'refine:failed']);
|
|
49
50
|
|
|
50
51
|
/** Normalise a coordinator-daemon-id argument (single id, list, or undefined) into a
|
|
51
|
-
* de-duplicated list of non-empty strings
|
|
52
|
-
*
|
|
52
|
+
* de-duplicated list of non-empty strings, EXPANDED to every equivalent daemon-id
|
|
53
|
+
* form (bare `mach_X` ≡ `daemon_mach_X` ≡ `standalone_mach_X`).
|
|
54
|
+
*
|
|
55
|
+
* A coordinator resolves its own id through one path (status instanceId, the config-
|
|
56
|
+
* form node daemonId, or the bare machineId) but a worker stamps a completion's
|
|
57
|
+
* `coordinator_daemon_id` through another, so the two are routinely in DIFFERENT
|
|
58
|
+
* forms of the SAME machine. The scope filter is an exact-string match, so without
|
|
59
|
+
* expansion a `daemon_mach_X`-scoped completion is silently skipped by a coordinator
|
|
60
|
+
* that only knows itself as bare `mach_X` (the base-node completion-surface bug).
|
|
61
|
+
* Expanding here fixes every drain/peek/surface caller uniformly. The first ORIGINAL
|
|
62
|
+
* id stays at [0] so per-daemon JSONL file naming keeps its primary; expansion stays
|
|
63
|
+
* within one machine core so a different coordinator's events are never claimed. */
|
|
53
64
|
function normalizeCoordinatorDaemonIds(
|
|
54
65
|
coordinatorDaemonId?: string | null | ReadonlyArray<string>,
|
|
55
66
|
): string[] {
|
|
56
|
-
|
|
57
|
-
? coordinatorDaemonId
|
|
58
|
-
: coordinatorDaemonId != null ? [coordinatorDaemonId] : [];
|
|
59
|
-
const seen = new Set<string>();
|
|
60
|
-
const out: string[] = [];
|
|
61
|
-
for (const id of raw) {
|
|
62
|
-
if (typeof id !== 'string') continue;
|
|
63
|
-
const trimmed = id.trim();
|
|
64
|
-
if (!trimmed || seen.has(trimmed)) continue;
|
|
65
|
-
seen.add(trimmed);
|
|
66
|
-
out.push(trimmed);
|
|
67
|
-
}
|
|
68
|
-
return out;
|
|
67
|
+
return expandDaemonIdForms(coordinatorDaemonId);
|
|
69
68
|
}
|
|
70
69
|
|
|
71
70
|
export function readRefineJobId(event: { metadataEvent?: Record<string, unknown> } | Record<string, unknown>): string {
|
|
@@ -303,6 +302,7 @@ export function queuePendingMeshCoordinatorEvent(event: PendingMeshCoordinatorEv
|
|
|
303
302
|
const fingerprint = buildPendingEventFingerprint(event);
|
|
304
303
|
|
|
305
304
|
// G3: Write to SQLite inbox (primary path going forward)
|
|
305
|
+
let sqliteOk = false;
|
|
306
306
|
try {
|
|
307
307
|
MeshRuntimeStore.getInstance().insertPendingEvent({
|
|
308
308
|
id: randomUUID(),
|
|
@@ -313,14 +313,22 @@ export function queuePendingMeshCoordinatorEvent(event: PendingMeshCoordinatorEv
|
|
|
313
313
|
fingerprint: fingerprint || null,
|
|
314
314
|
queuedAt: event.queuedAt,
|
|
315
315
|
});
|
|
316
|
+
sqliteOk = true;
|
|
316
317
|
} catch {
|
|
317
318
|
// SQLite write failure is non-fatal; JSONL fallback below still works.
|
|
318
319
|
}
|
|
319
320
|
|
|
320
|
-
// Also write to JSONL (retained as legacy/export artifact)
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
321
|
+
// Also write to JSONL (retained as legacy/export artifact). Best-effort once
|
|
322
|
+
// SQLite (the primary store) has the event: a JSONL append failure (disk full,
|
|
323
|
+
// permissions) must NOT report the whole persist as failed when SQLite holds it.
|
|
324
|
+
try {
|
|
325
|
+
const path = getPendingEventsPath(event.meshId, event.targetCoordinatorDaemonId);
|
|
326
|
+
trimPendingEventsIfNeeded(path);
|
|
327
|
+
appendFileSync(path, JSON.stringify(event) + '\n', 'utf-8');
|
|
328
|
+
} catch (e: any) {
|
|
329
|
+
if (!sqliteOk) throw e; // neither store has it — surface as a real failure
|
|
330
|
+
LOG.warn('MeshEvents', `JSONL append failed for mesh ${event.meshId}; SQLite holds the event: ${e?.message || e}`);
|
|
331
|
+
}
|
|
324
332
|
return true;
|
|
325
333
|
} catch (e: any) {
|
|
326
334
|
LOG.warn('MeshEvents', `Failed to persist pending coordinator event: ${e?.message || e}`);
|
|
@@ -74,6 +74,17 @@ export function hasUnterminalDirectDispatchLedgerEntry(meshId: string, sessionId
|
|
|
74
74
|
return false;
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
// True when a terminal ledger payload was recorded from WEAK completion evidence (a false
|
|
78
|
+
// idle): insufficient evidence level, review-recommended, or a missing-final-assistant
|
|
79
|
+
// completion diagnostic. Mirrors isWeakTerminalLedgerPayload in mesh-events-coordinator —
|
|
80
|
+
// a weak terminal is non-authoritative and may be superseded by a genuine completion.
|
|
81
|
+
function isWeakCompletionLedgerPayload(payload: Record<string, unknown> | undefined): boolean {
|
|
82
|
+
if (!payload) return false;
|
|
83
|
+
if (payload.evidenceLevel === 'insufficient' || payload.reviewRecommended === true) return true;
|
|
84
|
+
const diag = readRecord(payload.completionDiagnostic);
|
|
85
|
+
return diag?.finalAssistantPresent === false || diag?.blockReason === 'missing_final_assistant';
|
|
86
|
+
}
|
|
87
|
+
|
|
77
88
|
function findDirectDispatchLedgerEntry(args: {
|
|
78
89
|
meshId: string;
|
|
79
90
|
taskId: string;
|
|
@@ -121,6 +132,12 @@ function hasTerminalLedgerAfterDispatch(args: {
|
|
|
121
132
|
if (!afterDispatch) continue;
|
|
122
133
|
}
|
|
123
134
|
if (entry.kind !== 'task_completed' && entry.kind !== 'task_failed' && entry.kind !== 'task_stalled') continue;
|
|
135
|
+
// Fix C (reconcile fallback expansion): a task_completed recorded from a FALSE idle
|
|
136
|
+
// (weak evidence / no confirmed final assistant) is NOT authoritative terminal
|
|
137
|
+
// evidence. Skip it so the transcript reconcile can still synthesize the GENUINE
|
|
138
|
+
// completion for a re-dispatched / prematurely-terminated direct task instead of
|
|
139
|
+
// bailing with alreadyTerminal.
|
|
140
|
+
if (entry.kind === 'task_completed' && isWeakCompletionLedgerPayload(entry.payload)) continue;
|
|
124
141
|
const terminalTaskId = readNonEmptyString(entry.payload?.taskId);
|
|
125
142
|
if (terminalTaskId && terminalTaskId === args.taskId) return true;
|
|
126
143
|
if (terminalTaskId && terminalTaskId !== args.taskId) continue;
|