@adhdev/daemon-core 0.9.82-rc.351 → 0.9.82-rc.353
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/router.d.ts +12 -0
- package/dist/commands/upgrade-helper.d.ts +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +815 -500
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +812 -500
- package/dist/index.mjs.map +1 -1
- package/dist/logging/logger.d.ts +1 -1
- package/dist/mesh/mesh-ledger.d.ts +1 -1
- package/dist/mesh/mesh-runtime-store.d.ts +9 -0
- package/dist/mesh/mesh-work-queue.d.ts +28 -0
- package/dist/providers/approval-utils.d.ts +7 -0
- package/dist/providers/cli-provider-instance.d.ts +15 -0
- package/dist/providers/sdk/v1/builders/cli/detect-status.d.ts +1 -0
- package/package.json +2 -2
- package/src/commands/cli-manager.ts +13 -4
- package/src/commands/router.ts +133 -33
- package/src/commands/upgrade-helper.ts +57 -14
- package/src/index.ts +5 -0
- package/src/logging/command-log.ts +7 -5
- package/src/logging/logger.ts +12 -6
- package/src/mesh/mesh-events-coordinator.ts +165 -84
- package/src/mesh/mesh-events-pending.ts +14 -15
- package/src/mesh/mesh-ledger.ts +1 -0
- package/src/mesh/mesh-reconcile-loop.ts +67 -7
- package/src/mesh/mesh-runtime-store.ts +17 -0
- package/src/mesh/mesh-work-queue.ts +89 -0
- package/src/providers/approval-utils.d.ts +1 -0
- package/src/providers/approval-utils.ts +10 -0
- package/src/providers/cli-provider-instance.ts +73 -19
- package/src/providers/sdk/v1/builders/cli/detect-status.ts +51 -0
|
@@ -7,6 +7,7 @@ import { LOG } from '../logging/logger.js';
|
|
|
7
7
|
import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
|
|
8
8
|
import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
|
|
9
9
|
import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents } from './mesh-work-queue.js';
|
|
10
|
+
import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
|
|
10
11
|
import { fastForwardMeshNode } from './mesh-fast-forward.js';
|
|
11
12
|
import { createSessionDelivery, markSessionDeliveriesTerminal, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
|
|
12
13
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
@@ -17,7 +18,7 @@ import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUn
|
|
|
17
18
|
import { getLastDisplayMessage } from '../status/snapshot.js';
|
|
18
19
|
import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
19
20
|
import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
|
|
20
|
-
import { normalizeMeshNodeId, meshNodeIdMatches, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
21
|
+
import { normalizeMeshNodeId, meshNodeIdMatches, expandDaemonIdForms, type MeshNodeIdentified } from '@adhdev/mesh-shared';
|
|
21
22
|
import {
|
|
22
23
|
findRecentTerminalLedgerEvidence,
|
|
23
24
|
hasDispatchAfterTerminal,
|
|
@@ -35,16 +36,16 @@ import {
|
|
|
35
36
|
} from './mesh-events-utils.js';
|
|
36
37
|
|
|
37
38
|
// The set of coordinator-daemon ids this daemon answers to when draining the
|
|
38
|
-
// pending-events queue
|
|
39
|
-
//
|
|
40
|
-
//
|
|
39
|
+
// pending-events queue. Mirrors resolveCoordinatorDaemonIds in mesh-reconcile-loop:
|
|
40
|
+
// a unicast event may be stamped with the status id, the bare machineId, OR the
|
|
41
|
+
// config-form node daemonId (`daemon_<machineId>`) depending on which dispatch path
|
|
42
|
+
// created the worker. We expand to EVERY equivalent form so a `daemon_<machineId>`
|
|
43
|
+
// completion matches a coordinator that knows itself as bare `<machineId>` (the
|
|
44
|
+
// base-node completion-surface bug) and vice versa.
|
|
41
45
|
function resolveCoordinatorDrainDaemonIds(components: DaemonComponents): string[] {
|
|
42
|
-
const ids = new Set<string>();
|
|
43
46
|
const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
|
|
44
|
-
if (statusInstanceId) ids.add(statusInstanceId);
|
|
45
47
|
const machineId = readNonEmptyString(loadConfig().machineId);
|
|
46
|
-
|
|
47
|
-
return [...ids];
|
|
48
|
+
return expandDaemonIdForms([statusInstanceId, machineId]);
|
|
48
49
|
}
|
|
49
50
|
|
|
50
51
|
// ---------------------------------------------------------------------------
|
|
@@ -304,6 +305,95 @@ function resolveActiveDirectDispatchTaskId(meshId: string, sessionId: string): s
|
|
|
304
305
|
// Queue assignment
|
|
305
306
|
// ---------------------------------------------------------------------------
|
|
306
307
|
|
|
308
|
+
// Per-dispatch confirmation timeout (Bug B). A dispatch promise that never settles —
|
|
309
|
+
// a saturated remote P2P relay that hangs, or a transport that resolves only after
|
|
310
|
+
// the worker acks — would otherwise leave the just-claimed queue row 'assigned' with
|
|
311
|
+
// its delivery stuck 'delivering' forever: the .catch that requeues never fires, and
|
|
312
|
+
// PHASE 3 reconcile skips the row (it counts 0 pending). Racing the dispatch against
|
|
313
|
+
// this timeout guarantees a hung dispatch deterministically returns the task to
|
|
314
|
+
// 'pending' for re-dispatch. Generous so a merely-slow-but-live dispatch (a cold
|
|
315
|
+
// remote relay) is never reclaimed early; the reconcile assigned-stranded watchdog is
|
|
316
|
+
// the durable cross-restart backstop for a timer lost to a daemon restart.
|
|
317
|
+
const DISPATCH_CONFIRM_TIMEOUT_MS = 120_000;
|
|
318
|
+
|
|
319
|
+
interface DeliverTaskContext {
|
|
320
|
+
meshId: string;
|
|
321
|
+
nodeId: string;
|
|
322
|
+
sessionId: string;
|
|
323
|
+
providerType: string;
|
|
324
|
+
task: MeshWorkQueueEntry;
|
|
325
|
+
transport: 'remote' | 'local';
|
|
326
|
+
sourceCoordinatorSessionId?: string;
|
|
327
|
+
sourceCoordinatorDaemonId?: string;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
// CONS scope 3: the SINGLE source of truth for dispatching a claimed task to its
|
|
331
|
+
// session. The remote (P2P dispatchMeshCommand) and local (cliManager.handleCliCommand)
|
|
332
|
+
// branches differ ONLY in the transport call — the delivery record, the delivered/failed
|
|
333
|
+
// transitions, the pending-requeue-on-failure, the dispatch_failed ledger entry, AND the
|
|
334
|
+
// Bug B hang timeout are identical and live here once so a future change to the dispatch
|
|
335
|
+
// lifecycle cannot drift between the two paths. The caller passes a `dispatchThunk` that
|
|
336
|
+
// performs only the transport-specific send and returns its promise.
|
|
337
|
+
function deliverTaskToSession(dispatchThunk: () => Promise<unknown>, ctx: DeliverTaskContext): void {
|
|
338
|
+
const delivery = createSessionDelivery({
|
|
339
|
+
meshId: ctx.meshId,
|
|
340
|
+
nodeId: ctx.nodeId,
|
|
341
|
+
sessionId: ctx.sessionId,
|
|
342
|
+
providerType: ctx.providerType,
|
|
343
|
+
taskId: ctx.task.id,
|
|
344
|
+
kind: 'task',
|
|
345
|
+
message: ctx.task.message,
|
|
346
|
+
status: 'delivering',
|
|
347
|
+
...(ctx.sourceCoordinatorSessionId ? { sourceCoordinatorSessionId: ctx.sourceCoordinatorSessionId } : {}),
|
|
348
|
+
...(ctx.sourceCoordinatorDaemonId ? { sourceCoordinatorDaemonId: ctx.sourceCoordinatorDaemonId } : {}),
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
// Invoke the transport synchronously (preserves the prior fire-and-forget timing,
|
|
352
|
+
// and lets a synchronous throw fall into the same failure path as a rejection).
|
|
353
|
+
let dispatchPromise: Promise<unknown>;
|
|
354
|
+
try {
|
|
355
|
+
dispatchPromise = Promise.resolve(dispatchThunk());
|
|
356
|
+
} catch (e) {
|
|
357
|
+
dispatchPromise = Promise.reject(e);
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
361
|
+
const guarded = Promise.race([
|
|
362
|
+
dispatchPromise,
|
|
363
|
+
new Promise<never>((_, reject) => {
|
|
364
|
+
timer = setTimeout(
|
|
365
|
+
() => reject(new Error(`dispatch_confirm_timeout after ${DISPATCH_CONFIRM_TIMEOUT_MS}ms`)),
|
|
366
|
+
DISPATCH_CONFIRM_TIMEOUT_MS,
|
|
367
|
+
);
|
|
368
|
+
// Never keep the process alive solely for this confirm-timeout timer.
|
|
369
|
+
if (typeof (timer as { unref?: () => void })?.unref === 'function') (timer as { unref: () => void }).unref();
|
|
370
|
+
}),
|
|
371
|
+
]);
|
|
372
|
+
|
|
373
|
+
guarded.then(() => {
|
|
374
|
+
if (timer) clearTimeout(timer);
|
|
375
|
+
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
376
|
+
}).catch((e: any) => {
|
|
377
|
+
if (timer) clearTimeout(timer);
|
|
378
|
+
// A dispatch failure (transport reject OR hang timeout) is most often transient —
|
|
379
|
+
// a busy/refusing adapter, or a relay that never acked — not a permanent task
|
|
380
|
+
// failure. Marking the task terminal here would permanently kill tasks a later
|
|
381
|
+
// tick delivers fine. Return it to 'pending' and record a retryable dispatch_failed
|
|
382
|
+
// ledger entry so the reconcile loop re-dispatches it. Identical for both transports.
|
|
383
|
+
LOG.error('MeshQueue', `Failed to dispatch task via ${ctx.transport} to node ${ctx.nodeId}: ${e?.message}`);
|
|
384
|
+
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
385
|
+
updateTaskStatus(ctx.meshId, ctx.task.id, 'pending');
|
|
386
|
+
try {
|
|
387
|
+
appendLedgerEntry(ctx.meshId, {
|
|
388
|
+
kind: 'dispatch_failed' as any,
|
|
389
|
+
nodeId: ctx.nodeId,
|
|
390
|
+
sessionId: ctx.sessionId,
|
|
391
|
+
payload: { taskId: ctx.task.id, deliveryId: delivery.id, error: e?.message, retryable: true, transport: ctx.transport },
|
|
392
|
+
});
|
|
393
|
+
} catch { /* ledger write is best-effort */ }
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
|
|
307
397
|
export function tryAssignQueueTask(
|
|
308
398
|
components: DaemonComponents,
|
|
309
399
|
meshId: string,
|
|
@@ -337,45 +427,36 @@ export function tryAssignQueueTask(
|
|
|
337
427
|
// completion back to that exact session (multi-coordinator). Carried over P2P
|
|
338
428
|
// to the remote worker, which echoes it on its completion event.
|
|
339
429
|
const sourceCoordinatorSessionId = readNonEmptyString(task.sourceCoordinatorSessionId) || undefined;
|
|
340
|
-
const
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
430
|
+
const dispatchMeshCommand = components.dispatchMeshCommand;
|
|
431
|
+
const remoteDaemonId = node.daemonId;
|
|
432
|
+
// CONS3: only the transport call differs — everything else (delivery record,
|
|
433
|
+
// status transitions, requeue-on-failure, ledger, Bug B hang timeout) is in
|
|
434
|
+
// the shared deliverTaskToSession helper.
|
|
435
|
+
deliverTaskToSession(
|
|
436
|
+
() => dispatchMeshCommand(remoteDaemonId, 'agent_command', {
|
|
437
|
+
targetSessionId: sessionId,
|
|
438
|
+
cliType: providerType,
|
|
439
|
+
action: 'send_chat',
|
|
440
|
+
message: task.message,
|
|
441
|
+
meshContext: {
|
|
442
|
+
meshId,
|
|
443
|
+
nodeId,
|
|
444
|
+
taskId: task.id,
|
|
445
|
+
...(localDaemonIdForDispatch ? { coordinatorDaemonId: localDaemonIdForDispatch } : {}),
|
|
446
|
+
...(sourceCoordinatorSessionId ? { coordinatorSessionId: sourceCoordinatorSessionId } : {}),
|
|
447
|
+
},
|
|
448
|
+
}),
|
|
449
|
+
{
|
|
358
450
|
meshId,
|
|
359
451
|
nodeId,
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
452
|
+
sessionId,
|
|
453
|
+
providerType,
|
|
454
|
+
task,
|
|
455
|
+
transport: 'remote',
|
|
456
|
+
...(sourceCoordinatorSessionId ? { sourceCoordinatorSessionId } : {}),
|
|
457
|
+
...(localDaemonIdForDispatch ? { sourceCoordinatorDaemonId: localDaemonIdForDispatch } : {}),
|
|
363
458
|
},
|
|
364
|
-
|
|
365
|
-
updateSessionDeliveryStatus(delivery.id, 'delivered');
|
|
366
|
-
}).catch((e: any) => {
|
|
367
|
-
LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
|
|
368
|
-
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
369
|
-
updateTaskStatus(meshId, task.id, 'pending');
|
|
370
|
-
try {
|
|
371
|
-
appendLedgerEntry(meshId, {
|
|
372
|
-
kind: 'dispatch_failed' as any,
|
|
373
|
-
nodeId,
|
|
374
|
-
sessionId,
|
|
375
|
-
payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
|
|
376
|
-
});
|
|
377
|
-
} catch { /* ledger write is best-effort */ }
|
|
378
|
-
});
|
|
459
|
+
);
|
|
379
460
|
return true;
|
|
380
461
|
}
|
|
381
462
|
}
|
|
@@ -412,44 +493,26 @@ export function tryAssignQueueTask(
|
|
|
412
493
|
}
|
|
413
494
|
} catch { /* best-effort — dispatch still proceeds */ }
|
|
414
495
|
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
// Mirror the remote-dispatch catch above: a local dispatch failure is most often a
|
|
436
|
-
// transient busy/refusal (e.g. the adapter rejected send_chat while mid-generation),
|
|
437
|
-
// not a permanent task failure. Marking the task terminal 'failed' here with no ledger
|
|
438
|
-
// and no retry permanently killed tasks that a later tick would have delivered fine.
|
|
439
|
-
// Return the task to 'pending' and record a retryable dispatch_failed ledger entry so
|
|
440
|
-
// the reconcile loop re-dispatches it, exactly as the remote branch does.
|
|
441
|
-
LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
|
|
442
|
-
updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
|
|
443
|
-
updateTaskStatus(meshId, task.id, 'pending');
|
|
444
|
-
try {
|
|
445
|
-
appendLedgerEntry(meshId, {
|
|
446
|
-
kind: 'dispatch_failed' as any,
|
|
447
|
-
nodeId,
|
|
448
|
-
sessionId,
|
|
449
|
-
payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
|
|
450
|
-
});
|
|
451
|
-
} catch { /* ledger write is best-effort */ }
|
|
452
|
-
});
|
|
496
|
+
// CONS3: same shared dispatch lifecycle as the remote branch — only the transport
|
|
497
|
+
// (cliManager.handleCliCommand) differs.
|
|
498
|
+
deliverTaskToSession(
|
|
499
|
+
() => components.cliManager.handleCliCommand('agent_command', {
|
|
500
|
+
targetSessionId: sessionId,
|
|
501
|
+
cliType: providerType,
|
|
502
|
+
action: 'send_chat',
|
|
503
|
+
message: task.message,
|
|
504
|
+
}),
|
|
505
|
+
{
|
|
506
|
+
meshId,
|
|
507
|
+
nodeId,
|
|
508
|
+
sessionId,
|
|
509
|
+
providerType,
|
|
510
|
+
task,
|
|
511
|
+
transport: 'local',
|
|
512
|
+
...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { sourceCoordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
|
|
513
|
+
...(readNonEmptyString(loadConfig().machineId) ? { sourceCoordinatorDaemonId: readNonEmptyString(loadConfig().machineId) } : {}),
|
|
514
|
+
},
|
|
515
|
+
);
|
|
453
516
|
|
|
454
517
|
return true;
|
|
455
518
|
}
|
|
@@ -929,7 +992,12 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
929
992
|
|
|
930
993
|
const candidateNodes = Array.isArray(mesh?.nodes)
|
|
931
994
|
? mesh.nodes.filter((node: any) => {
|
|
932
|
-
|
|
995
|
+
// Bug A: match the target pin with the shared 3-form (id / nodeId / node_id)
|
|
996
|
+
// normalizer, mirroring the remote-idle drain (meshNodeIdMatches at the
|
|
997
|
+
// getRemoteIdleSessions filter). A strict `readMeshNodeId(node) !== targetNodeId`
|
|
998
|
+
// dropped a target node whose identity arrived under a different form (a freshly
|
|
999
|
+
// mesh_clone_node'd worktree), emptying candidateNodes and mislabelling the skip.
|
|
1000
|
+
if (task.targetNodeId && !meshNodeIdMatches(node, task.targetNodeId)) return false;
|
|
933
1001
|
// Skip nodes that can never satisfy requiredTags regardless of which provider
|
|
934
1002
|
// from providerPriority is selected. A node satisfies tags if at least one
|
|
935
1003
|
// provider in its priority list would produce matching capability tags.
|
|
@@ -944,7 +1012,20 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
|
|
|
944
1012
|
})
|
|
945
1013
|
: [];
|
|
946
1014
|
if (!candidateNodes.length) {
|
|
947
|
-
|
|
1015
|
+
// Bug A: distinguish the two ways the candidate set empties. A task pinned to a
|
|
1016
|
+
// targetNodeId whose node is absent from the mesh (or whose id arrived under a
|
|
1017
|
+
// different form) is a ROUTING miss — report it as `target_node_id_unmatched`, not
|
|
1018
|
+
// the hard-coded `no_node_satisfies_required_tags`, which mislabelled a 3-form
|
|
1019
|
+
// node-id mismatch as a capability failure and sent diagnosis down the wrong path.
|
|
1020
|
+
// Only fall back to the tag reason when no target pin is in play, or the pin DID
|
|
1021
|
+
// match a node but its tags excluded it (a genuine capability miss).
|
|
1022
|
+
const targetPinUnmatched = !!task.targetNodeId
|
|
1023
|
+
&& !(Array.isArray(mesh?.nodes) && mesh.nodes.some((n: any) => meshNodeIdMatches(n, task.targetNodeId)));
|
|
1024
|
+
markAutoLaunch(meshId, task.id, {
|
|
1025
|
+
status: 'skipped',
|
|
1026
|
+
reason: targetPinUnmatched ? 'target_node_id_unmatched' : 'no_node_satisfies_required_tags',
|
|
1027
|
+
nodeId: task.targetNodeId,
|
|
1028
|
+
});
|
|
948
1029
|
continue;
|
|
949
1030
|
}
|
|
950
1031
|
|
|
@@ -5,6 +5,7 @@ import { LOG } from '../logging/logger.js';
|
|
|
5
5
|
import { getLedgerDir, readLedgerEntries, appendLedgerEntry } from './mesh-ledger.js';
|
|
6
6
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
7
7
|
import { buildMeshSystemMessage, readNonEmptyString, readRecord, resolveEventSessionId, readMeshCompletionSummary } from './mesh-events-utils.js';
|
|
8
|
+
import { expandDaemonIdForms } from '@adhdev/mesh-shared';
|
|
8
9
|
|
|
9
10
|
// ---------------------------------------------------------------------------
|
|
10
11
|
// MCP coordinator pending-event queue — FILE-BASED PERSISTENCE
|
|
@@ -48,24 +49,22 @@ export interface PendingMeshCoordinatorEvent {
|
|
|
48
49
|
const REFINE_TERMINAL_EVENTS = new Set(['refine:completed', 'refine:failed']);
|
|
49
50
|
|
|
50
51
|
/** Normalise a coordinator-daemon-id argument (single id, list, or undefined) into a
|
|
51
|
-
* de-duplicated list of non-empty strings
|
|
52
|
-
*
|
|
52
|
+
* de-duplicated list of non-empty strings, EXPANDED to every equivalent daemon-id
|
|
53
|
+
* form (bare `mach_X` ≡ `daemon_mach_X` ≡ `standalone_mach_X`).
|
|
54
|
+
*
|
|
55
|
+
* A coordinator resolves its own id through one path (status instanceId, the config-
|
|
56
|
+
* form node daemonId, or the bare machineId) but a worker stamps a completion's
|
|
57
|
+
* `coordinator_daemon_id` through another, so the two are routinely in DIFFERENT
|
|
58
|
+
* forms of the SAME machine. The scope filter is an exact-string match, so without
|
|
59
|
+
* expansion a `daemon_mach_X`-scoped completion is silently skipped by a coordinator
|
|
60
|
+
* that only knows itself as bare `mach_X` (the base-node completion-surface bug).
|
|
61
|
+
* Expanding here fixes every drain/peek/surface caller uniformly. The first ORIGINAL
|
|
62
|
+
* id stays at [0] so per-daemon JSONL file naming keeps its primary; expansion stays
|
|
63
|
+
* within one machine core so a different coordinator's events are never claimed. */
|
|
53
64
|
function normalizeCoordinatorDaemonIds(
|
|
54
65
|
coordinatorDaemonId?: string | null | ReadonlyArray<string>,
|
|
55
66
|
): string[] {
|
|
56
|
-
|
|
57
|
-
? coordinatorDaemonId
|
|
58
|
-
: coordinatorDaemonId != null ? [coordinatorDaemonId] : [];
|
|
59
|
-
const seen = new Set<string>();
|
|
60
|
-
const out: string[] = [];
|
|
61
|
-
for (const id of raw) {
|
|
62
|
-
if (typeof id !== 'string') continue;
|
|
63
|
-
const trimmed = id.trim();
|
|
64
|
-
if (!trimmed || seen.has(trimmed)) continue;
|
|
65
|
-
seen.add(trimmed);
|
|
66
|
-
out.push(trimmed);
|
|
67
|
-
}
|
|
68
|
-
return out;
|
|
67
|
+
return expandDaemonIdForms(coordinatorDaemonId);
|
|
69
68
|
}
|
|
70
69
|
|
|
71
70
|
export function readRefineJobId(event: { metadataEvent?: Record<string, unknown> } | Record<string, unknown>): string {
|
package/src/mesh/mesh-ledger.ts
CHANGED
|
@@ -56,7 +56,8 @@ import {
|
|
|
56
56
|
expireStaleUnresolvedDelegateForwards,
|
|
57
57
|
} from './mesh-unresolved-forward-outbox.js';
|
|
58
58
|
import { readNonEmptyString, readMeshCompletionSummary } from './mesh-events-utils.js';
|
|
59
|
-
import {
|
|
59
|
+
import { expandDaemonIdForms } from '@adhdev/mesh-shared';
|
|
60
|
+
import { getActiveDirectDispatches, getQueue, reclaimStrandedAssignedTask } from './mesh-work-queue.js';
|
|
60
61
|
import { readLedgerEntries } from './mesh-ledger.js';
|
|
61
62
|
import { pruneStaleDirectDispatches } from './mesh-active-work.js';
|
|
62
63
|
import { reconcileDirectDispatchCompletionFromTranscript } from './mesh-events-stale.js';
|
|
@@ -119,16 +120,19 @@ interface LiveCoordinator {
|
|
|
119
120
|
// - the daemon's canonical status id (`standalone_<machineId>` / `daemon_<machineId>`),
|
|
120
121
|
// stamped by the MCP layer via ctx.localDaemonId (= getStatus().status.instanceId), or
|
|
121
122
|
// - the bare machineId, stamped by the local queue-assignment path (loadConfig().machineId).
|
|
123
|
+
// - the config-form node daemonId (`daemon_<machineId>`), which the MCP layer's
|
|
124
|
+
// resolveCoordinatorDaemonId prefers and stamps onto direct-dispatch workers.
|
|
122
125
|
// Draining with only one of these silently misses events stamped with the other —
|
|
123
|
-
// the exact reason a generating coordinator never self-received local completions
|
|
124
|
-
//
|
|
126
|
+
// the exact reason a generating coordinator never self-received local completions,
|
|
127
|
+
// and the base-node completion-surface bug (base completions land full-form
|
|
128
|
+
// `daemon_<machineId>` while a coordinator that only knows itself as bare
|
|
129
|
+
// `<machineId>` never matches them). We expand to EVERY equivalent form so the
|
|
130
|
+
// scope match (host gate, self-node detection, and the drain IN-filter downstream)
|
|
131
|
+
// succeeds regardless of which path stamped the event.
|
|
125
132
|
function resolveCoordinatorDaemonIds(components: DaemonComponents): string[] {
|
|
126
|
-
const ids = new Set<string>();
|
|
127
133
|
const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
|
|
128
|
-
if (statusInstanceId) ids.add(statusInstanceId);
|
|
129
134
|
const machineId = readNonEmptyString(loadConfig().machineId);
|
|
130
|
-
|
|
131
|
-
return [...ids];
|
|
135
|
+
return expandDaemonIdForms([statusInstanceId, machineId]);
|
|
132
136
|
}
|
|
133
137
|
|
|
134
138
|
// Whether THIS daemon is the coordinator/host for a mesh — i.e. the daemon that
|
|
@@ -316,6 +320,47 @@ function recordHeldTerminalEventsToLedger(
|
|
|
316
320
|
// PHASE 2 — Live CLI inject. For each mesh that has a live CLI coordinator on
|
|
317
321
|
// THIS daemon, drain the local queue and inject pending events into the PTY.
|
|
318
322
|
// Unchanged from before.
|
|
323
|
+
// Bug B: how long a row may sit 'assigned' with an unconfirmed dispatch before the
|
|
324
|
+
// watchdog reclaims it. Must be comfortably larger than the per-dispatch confirm
|
|
325
|
+
// timeout (DISPATCH_CONFIRM_TIMEOUT_MS in mesh-events-coordinator) so a slow-but-live
|
|
326
|
+
// dispatch still inside its normal confirm window is never reclaimed early — this is
|
|
327
|
+
// the durable backstop for the case the in-process confirm timer can't cover (a timer
|
|
328
|
+
// lost to a daemon restart between claim and confirm).
|
|
329
|
+
const ASSIGNED_STRANDED_DEADLINE_MS = 5 * 60_000;
|
|
330
|
+
|
|
331
|
+
// PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
|
|
332
|
+
// flips a row to 'assigned' BEFORE the fire-and-forget dispatch runs. If that dispatch
|
|
333
|
+
// neither rejects (→ no .catch requeue) nor is confirmed delivered — a relay that hangs
|
|
334
|
+
// without acking, or a confirm timer lost across a restart — the row stays 'assigned'
|
|
335
|
+
// forever: it contributes 0 pending, so PHASE 3 (gated on pendingQueueTaskCount>0) never
|
|
336
|
+
// re-examines it, and nothing but a manual requeue clears it. This is that missing net.
|
|
337
|
+
//
|
|
338
|
+
// Regression guard: a row whose delivery IS confirmed (delivered/acked/completed) is a
|
|
339
|
+
// genuinely in-flight (or completion-lost) task — left to PHASE 4's completion reconcile,
|
|
340
|
+
// never reclaimed here. And the deadline is generous so a slow-but-live dispatch still in
|
|
341
|
+
// its normal confirm window is never reclaimed early. Reclaimed rows return to 'pending'
|
|
342
|
+
// with ownership cleared, so the PHASE 3 trigger below re-dispatches them this same tick.
|
|
343
|
+
function recoverStrandedAssignedDispatches(meshId: string, store: MeshRuntimeStore): void {
|
|
344
|
+
const assigned = getQueue(meshId, { status: ['assigned'] });
|
|
345
|
+
if (!assigned.length) return;
|
|
346
|
+
const nowMs = Date.now();
|
|
347
|
+
for (const row of assigned) {
|
|
348
|
+
const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
|
|
349
|
+
if (!Number.isFinite(dispatchedAtMs)) continue; // no dispatch ts → can't age it
|
|
350
|
+
if (nowMs - dispatchedAtMs < ASSIGNED_STRANDED_DEADLINE_MS) continue; // still in confirm window
|
|
351
|
+
if (store.taskHasConfirmedDelivery(meshId, row.id)) continue; // dispatched → PHASE 4's job
|
|
352
|
+
const reclaimed = reclaimStrandedAssignedTask(meshId, row.id, {
|
|
353
|
+
reason: 'assigned_stranded_dispatch_unconfirmed',
|
|
354
|
+
ageMs: nowMs - dispatchedAtMs,
|
|
355
|
+
});
|
|
356
|
+
if (reclaimed) {
|
|
357
|
+
LOG.warn('MeshReconcile', `Reclaimed stranded assigned task ${row.id} on mesh ${meshId} `
|
|
358
|
+
+ `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, dispatched `
|
|
359
|
+
+ `${Math.round((nowMs - dispatchedAtMs) / 1000)}s ago, never confirmed delivered → ${reclaimed.status})`);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
|
|
319
364
|
export async function runMeshReconcileTick(components: DaemonComponents): Promise<void> {
|
|
320
365
|
const localDaemonId = readNonEmptyString(loadConfig().machineId) || undefined;
|
|
321
366
|
// The id-set used to scope the local queue drain (status id + machineId). See
|
|
@@ -360,6 +405,21 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
|
|
|
360
405
|
}
|
|
361
406
|
}
|
|
362
407
|
|
|
408
|
+
// ── PHASE 2.5: assigned-stranded dispatch watchdog (Bug B) ─────────────────
|
|
409
|
+
// Runs before PHASE 3 so any row it returns to 'pending' is re-dispatched by the
|
|
410
|
+
// PHASE 3 trigger in this same tick. See recoverStrandedAssignedDispatches.
|
|
411
|
+
if (store) {
|
|
412
|
+
for (const mesh of listMeshes()) {
|
|
413
|
+
const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
|
|
414
|
+
if (!daemonHostsMesh(mesh, selfIds)) continue;
|
|
415
|
+
try {
|
|
416
|
+
recoverStrandedAssignedDispatches(mesh.id, store);
|
|
417
|
+
} catch (e: any) {
|
|
418
|
+
LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
|
|
363
423
|
// ── PHASE 3: recover pending queue claims for newly-idle sessions ──────────
|
|
364
424
|
// The event-driven claim paths (agent:ready / agent:generating_completed in
|
|
365
425
|
// mesh-events-coordinator) re-claim the queue the moment a session goes idle,
|
|
@@ -1075,6 +1075,23 @@ export class MeshRuntimeStore {
|
|
|
1075
1075
|
}));
|
|
1076
1076
|
}
|
|
1077
1077
|
|
|
1078
|
+
/**
|
|
1079
|
+
* Bug B watchdog support: true when at least one delivery record for the task has
|
|
1080
|
+
* reached a confirmed-handed-off status (delivered / acked / completed). The
|
|
1081
|
+
* assigned-stranded watchdog uses this to distinguish a dispatch that was never
|
|
1082
|
+
* confirmed (reclaimable) from one that WAS handed to the worker (a genuinely
|
|
1083
|
+
* in-flight or completion-lost task, which is PHASE 4's responsibility, not this
|
|
1084
|
+
* watchdog's). Indexed by (mesh_id, task_id).
|
|
1085
|
+
*/
|
|
1086
|
+
taskHasConfirmedDelivery(meshId: string, taskId: string): boolean {
|
|
1087
|
+
const row = this.db.prepare(`
|
|
1088
|
+
SELECT 1 FROM mesh_session_delivery
|
|
1089
|
+
WHERE mesh_id = ? AND task_id = ? AND status IN ('delivered','acked','completed')
|
|
1090
|
+
LIMIT 1
|
|
1091
|
+
`).get(meshId, taskId) as { 1: number } | undefined;
|
|
1092
|
+
return !!row;
|
|
1093
|
+
}
|
|
1094
|
+
|
|
1078
1095
|
expireStaleSessionDeliveries(meshId: string): void {
|
|
1079
1096
|
const now = new Date().toISOString();
|
|
1080
1097
|
this.db.prepare(`
|
|
@@ -5,6 +5,8 @@ import { MESH_CONVERGE_REFINE_TAG, resolveAutoConvergeCodeChange } from '../repo
|
|
|
5
5
|
import { MeshRuntimeStore } from './mesh-runtime-store.js';
|
|
6
6
|
import { getMesh } from '../config/mesh-config.js';
|
|
7
7
|
import { LOG } from '../logging/logger.js';
|
|
8
|
+
import { appendLedgerEntry } from './mesh-ledger.js';
|
|
9
|
+
import type { MeshLedgerKind } from './mesh-ledger.js';
|
|
8
10
|
|
|
9
11
|
export type MeshTaskStatus = 'pending' | 'assigned' | 'completed' | 'failed' | 'cancelled';
|
|
10
12
|
export type MeshActiveTaskStatus = Extract<MeshTaskStatus, 'pending' | 'assigned'>;
|
|
@@ -339,6 +341,14 @@ export interface MeshWorkQueueEntry {
|
|
|
339
341
|
requeueCount?: number;
|
|
340
342
|
/** Max automatic requeue attempts. When requeueCount reaches this, task is auto-failed. */
|
|
341
343
|
maxRetries?: number;
|
|
344
|
+
/**
|
|
345
|
+
* Bug B: number of times the reconcile assigned-stranded watchdog has reclaimed this
|
|
346
|
+
* row from 'assigned' back to 'pending' because its dispatch was never confirmed
|
|
347
|
+
* delivered. Separate from requeueCount (operator/execution retries) and bounded by
|
|
348
|
+
* MAX_STRANDED_RECLAIMS so a permanently-undeliverable target auto-fails rather than
|
|
349
|
+
* cycling reclaim→re-dispatch→strand forever.
|
|
350
|
+
*/
|
|
351
|
+
strandedReclaimCount?: number;
|
|
342
352
|
/** Last automatic queue session spin-up attempt, for mesh_view_queue/debug visibility. */
|
|
343
353
|
autoLaunch?: {
|
|
344
354
|
status: 'skipped' | 'started' | 'failed' | 'completed';
|
|
@@ -887,6 +897,85 @@ export function requeueTask(
|
|
|
887
897
|
});
|
|
888
898
|
}
|
|
889
899
|
|
|
900
|
+
/**
|
|
901
|
+
* Max times the assigned-stranded watchdog will reclaim a single task before giving
|
|
902
|
+
* up and failing it. Bounds the reclaim→re-dispatch→strand cycle so a permanently
|
|
903
|
+
* undeliverable target (e.g. a node whose transport is wedged) eventually fails and
|
|
904
|
+
* unblocks its dependents instead of looping every reconcile tick.
|
|
905
|
+
*/
|
|
906
|
+
const MAX_STRANDED_RECLAIMS = 3;
|
|
907
|
+
|
|
908
|
+
/**
|
|
909
|
+
* Bug B: reclaim a task stuck in 'assigned' because its dispatch was never confirmed.
|
|
910
|
+
*
|
|
911
|
+
* claimNextTask atomically marks a row 'assigned' BEFORE the fire-and-forget dispatch
|
|
912
|
+
* runs. If that dispatch neither rejects (→ no .catch requeue) nor is confirmed
|
|
913
|
+
* delivered — a relay that hangs without acking, or a confirm timer lost across a
|
|
914
|
+
* daemon restart — the row stays 'assigned' forever, contributing 0 pending so PHASE 3
|
|
915
|
+
* reconcile never re-examines it. This returns such a row to 'pending' and clears its
|
|
916
|
+
* dead assignment ownership (node / session / provider / dispatchTimestamp) — the same
|
|
917
|
+
* ownership-clear requeueTask applies — so PHASE 3 can re-dispatch it onto a fresh idle
|
|
918
|
+
* session.
|
|
919
|
+
*
|
|
920
|
+
* Guarded to 'assigned' rows only (a completion/cancel that already moved the row off
|
|
921
|
+
* 'assigned' must never be resurrected) and bounded by MAX_STRANDED_RECLAIMS (beyond
|
|
922
|
+
* which the task is failed so dependents unblock).
|
|
923
|
+
*/
|
|
924
|
+
export function reclaimStrandedAssignedTask(
|
|
925
|
+
meshId: string,
|
|
926
|
+
taskId: string,
|
|
927
|
+
opts?: { reason?: string; ageMs?: number } & MeshQueueMutationOptions,
|
|
928
|
+
): MeshWorkQueueEntry | null {
|
|
929
|
+
requireMeshHostQueueOwner(opts);
|
|
930
|
+
return withQueueLock(meshId, () => {
|
|
931
|
+
const entry = MeshRuntimeStore.getInstance().findQueueEntryById(meshId, taskId);
|
|
932
|
+
if (!entry) return null;
|
|
933
|
+
// Only a still-assigned row is stranded. If a completion/cancel already moved it
|
|
934
|
+
// off 'assigned', there is nothing to reclaim — never resurrect a terminal row.
|
|
935
|
+
if (entry.status !== 'assigned') return null;
|
|
936
|
+
const now = new Date().toISOString();
|
|
937
|
+
const reason = opts?.reason || 'assigned_stranded_dispatch_unconfirmed';
|
|
938
|
+
const reclaims = (entry.strandedReclaimCount || 0) + 1;
|
|
939
|
+
const prevNode = entry.assignedNodeId;
|
|
940
|
+
const prevSession = entry.assignedSessionId;
|
|
941
|
+
// Always clear the dead assignment ownership so a re-claim starts clean and the
|
|
942
|
+
// assigned-counters (which filter status==='assigned') stop counting this row.
|
|
943
|
+
delete entry.assignedNodeId;
|
|
944
|
+
delete entry.assignedSessionId;
|
|
945
|
+
delete entry.assignedProviderType;
|
|
946
|
+
delete entry.dispatchTimestamp;
|
|
947
|
+
entry.strandedReclaimCount = reclaims;
|
|
948
|
+
entry.updatedAt = now;
|
|
949
|
+
if (reclaims > MAX_STRANDED_RECLAIMS) {
|
|
950
|
+
// Repeatedly undeliverable — stop cycling and fail it so dependents unblock.
|
|
951
|
+
entry.status = 'failed';
|
|
952
|
+
entry.cancelReason = `stranded_dispatch_unrecovered: reclaimed ${reclaims - 1} time(s) without a confirmed dispatch`;
|
|
953
|
+
MeshRuntimeStore.getInstance().updateQueueEntry(entry);
|
|
954
|
+
propagateDependencyFailure(meshId, taskId);
|
|
955
|
+
} else {
|
|
956
|
+
entry.status = 'pending';
|
|
957
|
+
entry.requeuedAt = now;
|
|
958
|
+
entry.requeueReason = reason;
|
|
959
|
+
MeshRuntimeStore.getInstance().updateQueueEntry(entry);
|
|
960
|
+
}
|
|
961
|
+
try {
|
|
962
|
+
appendLedgerEntry(meshId, {
|
|
963
|
+
kind: 'task_reclaimed' as MeshLedgerKind,
|
|
964
|
+
nodeId: prevNode,
|
|
965
|
+
sessionId: prevSession,
|
|
966
|
+
payload: {
|
|
967
|
+
taskId,
|
|
968
|
+
reason,
|
|
969
|
+
...(typeof opts?.ageMs === 'number' ? { ageMs: opts.ageMs } : {}),
|
|
970
|
+
reclaimCount: reclaims,
|
|
971
|
+
outcome: entry.status,
|
|
972
|
+
},
|
|
973
|
+
});
|
|
974
|
+
} catch { /* ledger write is best-effort */ }
|
|
975
|
+
return entry;
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
|
|
890
979
|
/**
|
|
891
980
|
* Update the status of the task currently assigned to a specific session.
|
|
892
981
|
*/
|
|
@@ -8,4 +8,5 @@ export declare function pickAutoApprovalButton(buttons: string[] | null | undefi
|
|
|
8
8
|
index: number;
|
|
9
9
|
label: string;
|
|
10
10
|
};
|
|
11
|
+
export declare function hasNegativeApprovalOption(buttons: string[] | null | undefined): boolean;
|
|
11
12
|
export declare function formatAutoApprovalMessage(modalMessage?: string, buttonLabel?: string): string;
|
|
@@ -31,6 +31,16 @@ function isNegativeApprovalLabel(value: string): boolean {
|
|
|
31
31
|
|| /\bdo not\b/.test(label);
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
/**
|
|
35
|
+
* True when any of the given button labels reads as a decline/negative option
|
|
36
|
+
* (No / Deny / Cancel / Skip / …). Used as the second half of an approval-modal
|
|
37
|
+
* structural anchor: a real approval modal offers BOTH an affirmative and a
|
|
38
|
+
* decline, which distinguishes it from a generic numbered menu or prose list.
|
|
39
|
+
*/
|
|
40
|
+
export function hasNegativeApprovalOption(buttons: string[] | null | undefined): boolean {
|
|
41
|
+
return (buttons || []).some((button) => isNegativeApprovalLabel(String(button || '')));
|
|
42
|
+
}
|
|
43
|
+
|
|
34
44
|
export function getApprovalPositiveHints(provider?: Pick<ProviderModule, 'approvalPositiveHints'> | null): string[] {
|
|
35
45
|
const customHints = Array.isArray(provider?.approvalPositiveHints)
|
|
36
46
|
? provider.approvalPositiveHints
|