@adhdev/daemon-core 0.9.82-rc.351 → 0.9.82-rc.353

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -7,6 +7,7 @@ import { LOG } from '../logging/logger.js';
7
7
  import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
8
8
  import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
9
9
  import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents } from './mesh-work-queue.js';
10
+ import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
10
11
  import { fastForwardMeshNode } from './mesh-fast-forward.js';
11
12
  import { createSessionDelivery, markSessionDeliveriesTerminal, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
12
13
  import { MeshRuntimeStore } from './mesh-runtime-store.js';
@@ -17,7 +18,7 @@ import { enqueueUnresolvedDelegateForward, peekUnresolvedDelegateForwards, ackUn
17
18
  import { getLastDisplayMessage } from '../status/snapshot.js';
18
19
  import { resolveDelegatedWorkerAutoApprove, resolveProviderMaxParallel, resolveNodeSchedulingPriority, normalizeMeshSchedulingStrategy } from '../repo-mesh-types.js';
19
20
  import type { RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
20
- import { normalizeMeshNodeId, meshNodeIdMatches, type MeshNodeIdentified } from '@adhdev/mesh-shared';
21
+ import { normalizeMeshNodeId, meshNodeIdMatches, expandDaemonIdForms, type MeshNodeIdentified } from '@adhdev/mesh-shared';
21
22
  import {
22
23
  findRecentTerminalLedgerEvidence,
23
24
  hasDispatchAfterTerminal,
@@ -35,16 +36,16 @@ import {
35
36
  } from './mesh-events-utils.js';
36
37
 
37
38
  // The set of coordinator-daemon ids this daemon answers to when draining the
38
- // pending-events queue (canonical status id + bare machineId). Mirrors
39
- // resolveCoordinatorDaemonIds in mesh-reconcile-loop — a unicast event may be
40
- // stamped with either id depending on which dispatch path created the worker.
39
+ // pending-events queue. Mirrors resolveCoordinatorDaemonIds in mesh-reconcile-loop:
40
+ // a unicast event may be stamped with the status id, the bare machineId, OR the
41
+ // config-form node daemonId (`daemon_<machineId>`) depending on which dispatch path
42
+ // created the worker. We expand to EVERY equivalent form so a `daemon_<machineId>`
43
+ // completion matches a coordinator that knows itself as bare `<machineId>` (the
44
+ // base-node completion-surface bug) and vice versa.
41
45
  function resolveCoordinatorDrainDaemonIds(components: DaemonComponents): string[] {
42
- const ids = new Set<string>();
43
46
  const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
44
- if (statusInstanceId) ids.add(statusInstanceId);
45
47
  const machineId = readNonEmptyString(loadConfig().machineId);
46
- if (machineId) ids.add(machineId);
47
- return [...ids];
48
+ return expandDaemonIdForms([statusInstanceId, machineId]);
48
49
  }
49
50
 
50
51
  // ---------------------------------------------------------------------------
@@ -304,6 +305,95 @@ function resolveActiveDirectDispatchTaskId(meshId: string, sessionId: string): s
304
305
  // Queue assignment
305
306
  // ---------------------------------------------------------------------------
306
307
 
308
+ // Per-dispatch confirmation timeout (Bug B). A dispatch promise that never settles —
309
+ // a saturated remote P2P relay that hangs, or a transport that resolves only after
310
+ // the worker acks — would otherwise leave the just-claimed queue row 'assigned' with
311
+ // its delivery stuck 'delivering' forever: the .catch that requeues never fires, and
312
+ // PHASE 3 reconcile skips the row (it counts 0 pending). Racing the dispatch against
313
+ // this timeout guarantees a hung dispatch deterministically returns the task to
314
+ // 'pending' for re-dispatch. Generous so a merely-slow-but-live dispatch (a cold
315
+ // remote relay) is never reclaimed early; the reconcile assigned-stranded watchdog is
316
+ // the durable cross-restart backstop for a timer lost to a daemon restart.
317
+ const DISPATCH_CONFIRM_TIMEOUT_MS = 120_000;
318
+
319
+ interface DeliverTaskContext {
320
+ meshId: string;
321
+ nodeId: string;
322
+ sessionId: string;
323
+ providerType: string;
324
+ task: MeshWorkQueueEntry;
325
+ transport: 'remote' | 'local';
326
+ sourceCoordinatorSessionId?: string;
327
+ sourceCoordinatorDaemonId?: string;
328
+ }
329
+
330
+ // CONS scope 3: the SINGLE source of truth for dispatching a claimed task to its
331
+ // session. The remote (P2P dispatchMeshCommand) and local (cliManager.handleCliCommand)
332
+ // branches differ ONLY in the transport call — the delivery record, the delivered/failed
333
+ // transitions, the pending-requeue-on-failure, the dispatch_failed ledger entry, AND the
334
+ // Bug B hang timeout are identical and live here once so a future change to the dispatch
335
+ // lifecycle cannot drift between the two paths. The caller passes a `dispatchThunk` that
336
+ // performs only the transport-specific send and returns its promise.
337
+ function deliverTaskToSession(dispatchThunk: () => Promise<unknown>, ctx: DeliverTaskContext): void {
338
+ const delivery = createSessionDelivery({
339
+ meshId: ctx.meshId,
340
+ nodeId: ctx.nodeId,
341
+ sessionId: ctx.sessionId,
342
+ providerType: ctx.providerType,
343
+ taskId: ctx.task.id,
344
+ kind: 'task',
345
+ message: ctx.task.message,
346
+ status: 'delivering',
347
+ ...(ctx.sourceCoordinatorSessionId ? { sourceCoordinatorSessionId: ctx.sourceCoordinatorSessionId } : {}),
348
+ ...(ctx.sourceCoordinatorDaemonId ? { sourceCoordinatorDaemonId: ctx.sourceCoordinatorDaemonId } : {}),
349
+ });
350
+
351
+ // Invoke the transport synchronously (preserves the prior fire-and-forget timing,
352
+ // and lets a synchronous throw fall into the same failure path as a rejection).
353
+ let dispatchPromise: Promise<unknown>;
354
+ try {
355
+ dispatchPromise = Promise.resolve(dispatchThunk());
356
+ } catch (e) {
357
+ dispatchPromise = Promise.reject(e);
358
+ }
359
+
360
+ let timer: ReturnType<typeof setTimeout> | undefined;
361
+ const guarded = Promise.race([
362
+ dispatchPromise,
363
+ new Promise<never>((_, reject) => {
364
+ timer = setTimeout(
365
+ () => reject(new Error(`dispatch_confirm_timeout after ${DISPATCH_CONFIRM_TIMEOUT_MS}ms`)),
366
+ DISPATCH_CONFIRM_TIMEOUT_MS,
367
+ );
368
+ // Never keep the process alive solely for this confirm-timeout timer.
369
+ if (typeof (timer as { unref?: () => void })?.unref === 'function') (timer as { unref: () => void }).unref();
370
+ }),
371
+ ]);
372
+
373
+ guarded.then(() => {
374
+ if (timer) clearTimeout(timer);
375
+ updateSessionDeliveryStatus(delivery.id, 'delivered');
376
+ }).catch((e: any) => {
377
+ if (timer) clearTimeout(timer);
378
+ // A dispatch failure (transport reject OR hang timeout) is most often transient —
379
+ // a busy/refusing adapter, or a relay that never acked — not a permanent task
380
+ // failure. Marking the task terminal here would permanently kill tasks a later
381
+ // tick delivers fine. Return it to 'pending' and record a retryable dispatch_failed
382
+ // ledger entry so the reconcile loop re-dispatches it. Identical for both transports.
383
+ LOG.error('MeshQueue', `Failed to dispatch task via ${ctx.transport} to node ${ctx.nodeId}: ${e?.message}`);
384
+ updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
385
+ updateTaskStatus(ctx.meshId, ctx.task.id, 'pending');
386
+ try {
387
+ appendLedgerEntry(ctx.meshId, {
388
+ kind: 'dispatch_failed' as any,
389
+ nodeId: ctx.nodeId,
390
+ sessionId: ctx.sessionId,
391
+ payload: { taskId: ctx.task.id, deliveryId: delivery.id, error: e?.message, retryable: true, transport: ctx.transport },
392
+ });
393
+ } catch { /* ledger write is best-effort */ }
394
+ });
395
+ }
396
+
307
397
  export function tryAssignQueueTask(
308
398
  components: DaemonComponents,
309
399
  meshId: string,
@@ -337,45 +427,36 @@ export function tryAssignQueueTask(
337
427
  // completion back to that exact session (multi-coordinator). Carried over P2P
338
428
  // to the remote worker, which echoes it on its completion event.
339
429
  const sourceCoordinatorSessionId = readNonEmptyString(task.sourceCoordinatorSessionId) || undefined;
340
- const delivery = createSessionDelivery({
341
- meshId,
342
- nodeId,
343
- sessionId,
344
- providerType,
345
- taskId: task.id,
346
- kind: 'task',
347
- message: task.message,
348
- status: 'delivering',
349
- ...(sourceCoordinatorSessionId ? { sourceCoordinatorSessionId } : {}),
350
- ...(localDaemonIdForDispatch ? { sourceCoordinatorDaemonId: localDaemonIdForDispatch } : {}),
351
- });
352
- components.dispatchMeshCommand(node.daemonId, 'agent_command', {
353
- targetSessionId: sessionId,
354
- cliType: providerType,
355
- action: 'send_chat',
356
- message: task.message,
357
- meshContext: {
430
+ const dispatchMeshCommand = components.dispatchMeshCommand;
431
+ const remoteDaemonId = node.daemonId;
432
+ // CONS3: only the transport call differs — everything else (delivery record,
433
+ // status transitions, requeue-on-failure, ledger, Bug B hang timeout) is in
434
+ // the shared deliverTaskToSession helper.
435
+ deliverTaskToSession(
436
+ () => dispatchMeshCommand(remoteDaemonId, 'agent_command', {
437
+ targetSessionId: sessionId,
438
+ cliType: providerType,
439
+ action: 'send_chat',
440
+ message: task.message,
441
+ meshContext: {
442
+ meshId,
443
+ nodeId,
444
+ taskId: task.id,
445
+ ...(localDaemonIdForDispatch ? { coordinatorDaemonId: localDaemonIdForDispatch } : {}),
446
+ ...(sourceCoordinatorSessionId ? { coordinatorSessionId: sourceCoordinatorSessionId } : {}),
447
+ },
448
+ }),
449
+ {
358
450
  meshId,
359
451
  nodeId,
360
- taskId: task.id,
361
- ...(localDaemonIdForDispatch ? { coordinatorDaemonId: localDaemonIdForDispatch } : {}),
362
- ...(sourceCoordinatorSessionId ? { coordinatorSessionId: sourceCoordinatorSessionId } : {}),
452
+ sessionId,
453
+ providerType,
454
+ task,
455
+ transport: 'remote',
456
+ ...(sourceCoordinatorSessionId ? { sourceCoordinatorSessionId } : {}),
457
+ ...(localDaemonIdForDispatch ? { sourceCoordinatorDaemonId: localDaemonIdForDispatch } : {}),
363
458
  },
364
- }).then(() => {
365
- updateSessionDeliveryStatus(delivery.id, 'delivered');
366
- }).catch((e: any) => {
367
- LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
368
- updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
369
- updateTaskStatus(meshId, task.id, 'pending');
370
- try {
371
- appendLedgerEntry(meshId, {
372
- kind: 'dispatch_failed' as any,
373
- nodeId,
374
- sessionId,
375
- payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
376
- });
377
- } catch { /* ledger write is best-effort */ }
378
- });
459
+ );
379
460
  return true;
380
461
  }
381
462
  }
@@ -412,44 +493,26 @@ export function tryAssignQueueTask(
412
493
  }
413
494
  } catch { /* best-effort — dispatch still proceeds */ }
414
495
 
415
- const delivery = createSessionDelivery({
416
- meshId,
417
- nodeId,
418
- sessionId,
419
- providerType,
420
- taskId: task.id,
421
- kind: 'task',
422
- message: task.message,
423
- status: 'delivering',
424
- ...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { sourceCoordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
425
- ...(readNonEmptyString(loadConfig().machineId) ? { sourceCoordinatorDaemonId: readNonEmptyString(loadConfig().machineId) } : {}),
426
- });
427
- components.cliManager.handleCliCommand('agent_command', {
428
- targetSessionId: sessionId,
429
- cliType: providerType,
430
- action: 'send_chat',
431
- message: task.message,
432
- }).then(() => {
433
- updateSessionDeliveryStatus(delivery.id, 'delivered');
434
- }).catch((e: any) => {
435
- // Mirror the remote-dispatch catch above: a local dispatch failure is most often a
436
- // transient busy/refusal (e.g. the adapter rejected send_chat while mid-generation),
437
- // not a permanent task failure. Marking the task terminal 'failed' here with no ledger
438
- // and no retry permanently killed tasks that a later tick would have delivered fine.
439
- // Return the task to 'pending' and record a retryable dispatch_failed ledger entry so
440
- // the reconcile loop re-dispatches it, exactly as the remote branch does.
441
- LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
442
- updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
443
- updateTaskStatus(meshId, task.id, 'pending');
444
- try {
445
- appendLedgerEntry(meshId, {
446
- kind: 'dispatch_failed' as any,
447
- nodeId,
448
- sessionId,
449
- payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
450
- });
451
- } catch { /* ledger write is best-effort */ }
452
- });
496
+ // CONS3: same shared dispatch lifecycle as the remote branch — only the transport
497
+ // (cliManager.handleCliCommand) differs.
498
+ deliverTaskToSession(
499
+ () => components.cliManager.handleCliCommand('agent_command', {
500
+ targetSessionId: sessionId,
501
+ cliType: providerType,
502
+ action: 'send_chat',
503
+ message: task.message,
504
+ }),
505
+ {
506
+ meshId,
507
+ nodeId,
508
+ sessionId,
509
+ providerType,
510
+ task,
511
+ transport: 'local',
512
+ ...(readNonEmptyString(task.sourceCoordinatorSessionId) ? { sourceCoordinatorSessionId: readNonEmptyString(task.sourceCoordinatorSessionId) } : {}),
513
+ ...(readNonEmptyString(loadConfig().machineId) ? { sourceCoordinatorDaemonId: readNonEmptyString(loadConfig().machineId) } : {}),
514
+ },
515
+ );
453
516
 
454
517
  return true;
455
518
  }
@@ -929,7 +992,12 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
929
992
 
930
993
  const candidateNodes = Array.isArray(mesh?.nodes)
931
994
  ? mesh.nodes.filter((node: any) => {
932
- if (task.targetNodeId && readMeshNodeId(node) !== task.targetNodeId) return false;
995
+ // Bug A: match the target pin with the shared 3-form (id / nodeId / node_id)
996
+ // normalizer, mirroring the remote-idle drain (meshNodeIdMatches at the
997
+ // getRemoteIdleSessions filter). A strict `readMeshNodeId(node) !== targetNodeId`
998
+ // dropped a target node whose identity arrived under a different form (a freshly
999
+ // mesh_clone_node'd worktree), emptying candidateNodes and mislabelling the skip.
1000
+ if (task.targetNodeId && !meshNodeIdMatches(node, task.targetNodeId)) return false;
933
1001
  // Skip nodes that can never satisfy requiredTags regardless of which provider
934
1002
  // from providerPriority is selected. A node satisfies tags if at least one
935
1003
  // provider in its priority list would produce matching capability tags.
@@ -944,7 +1012,20 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
944
1012
  })
945
1013
  : [];
946
1014
  if (!candidateNodes.length) {
947
- markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'no_node_satisfies_required_tags', nodeId: task.targetNodeId });
1015
+ // Bug A: distinguish the two ways the candidate set empties. A task pinned to a
1016
+ // targetNodeId whose node is absent from the mesh (or whose id arrived under a
1017
+ // different form) is a ROUTING miss — report it as `target_node_id_unmatched`, not
1018
+ // the hard-coded `no_node_satisfies_required_tags`, which mislabelled a 3-form
1019
+ // node-id mismatch as a capability failure and sent diagnosis down the wrong path.
1020
+ // Only fall back to the tag reason when no target pin is in play, or the pin DID
1021
+ // match a node but its tags excluded it (a genuine capability miss).
1022
+ const targetPinUnmatched = !!task.targetNodeId
1023
+ && !(Array.isArray(mesh?.nodes) && mesh.nodes.some((n: any) => meshNodeIdMatches(n, task.targetNodeId)));
1024
+ markAutoLaunch(meshId, task.id, {
1025
+ status: 'skipped',
1026
+ reason: targetPinUnmatched ? 'target_node_id_unmatched' : 'no_node_satisfies_required_tags',
1027
+ nodeId: task.targetNodeId,
1028
+ });
948
1029
  continue;
949
1030
  }
950
1031
 
@@ -5,6 +5,7 @@ import { LOG } from '../logging/logger.js';
5
5
  import { getLedgerDir, readLedgerEntries, appendLedgerEntry } from './mesh-ledger.js';
6
6
  import { MeshRuntimeStore } from './mesh-runtime-store.js';
7
7
  import { buildMeshSystemMessage, readNonEmptyString, readRecord, resolveEventSessionId, readMeshCompletionSummary } from './mesh-events-utils.js';
8
+ import { expandDaemonIdForms } from '@adhdev/mesh-shared';
8
9
 
9
10
  // ---------------------------------------------------------------------------
10
11
  // MCP coordinator pending-event queue — FILE-BASED PERSISTENCE
@@ -48,24 +49,22 @@ export interface PendingMeshCoordinatorEvent {
48
49
  const REFINE_TERMINAL_EVENTS = new Set(['refine:completed', 'refine:failed']);
49
50
 
50
51
  /** Normalise a coordinator-daemon-id argument (single id, list, or undefined) into a
51
- * de-duplicated list of non-empty strings. The first entry is treated as primary for
52
- * per-daemon JSONL file naming; all entries are accepted by drain/peek targeting. */
52
+ * de-duplicated list of non-empty strings, EXPANDED to every equivalent daemon-id
53
+ * form (bare `mach_X` `daemon_mach_X` `standalone_mach_X`).
54
+ *
55
+ * A coordinator resolves its own id through one path (status instanceId, the config-
56
+ * form node daemonId, or the bare machineId) but a worker stamps a completion's
57
+ * `coordinator_daemon_id` through another, so the two are routinely in DIFFERENT
58
+ * forms of the SAME machine. The scope filter is an exact-string match, so without
59
+ * expansion a `daemon_mach_X`-scoped completion is silently skipped by a coordinator
60
+ * that only knows itself as bare `mach_X` (the base-node completion-surface bug).
61
+ * Expanding here fixes every drain/peek/surface caller uniformly. The first ORIGINAL
62
+ * id stays at [0] so per-daemon JSONL file naming keeps its primary; expansion stays
63
+ * within one machine core so a different coordinator's events are never claimed. */
53
64
  function normalizeCoordinatorDaemonIds(
54
65
  coordinatorDaemonId?: string | null | ReadonlyArray<string>,
55
66
  ): string[] {
56
- const raw = Array.isArray(coordinatorDaemonId)
57
- ? coordinatorDaemonId
58
- : coordinatorDaemonId != null ? [coordinatorDaemonId] : [];
59
- const seen = new Set<string>();
60
- const out: string[] = [];
61
- for (const id of raw) {
62
- if (typeof id !== 'string') continue;
63
- const trimmed = id.trim();
64
- if (!trimmed || seen.has(trimmed)) continue;
65
- seen.add(trimmed);
66
- out.push(trimmed);
67
- }
68
- return out;
67
+ return expandDaemonIdForms(coordinatorDaemonId);
69
68
  }
70
69
 
71
70
  export function readRefineJobId(event: { metadataEvent?: Record<string, unknown> } | Record<string, unknown>): string {
@@ -43,6 +43,7 @@ export type MeshLedgerKind =
43
43
  | 'delivery_unroutable'
44
44
  | 'direct_dispatch_pruned'
45
45
  | 'event_held'
46
+ | 'task_reclaimed'
46
47
  ;
47
48
 
48
49
  export interface MeshLedgerEntry {
@@ -56,7 +56,8 @@ import {
56
56
  expireStaleUnresolvedDelegateForwards,
57
57
  } from './mesh-unresolved-forward-outbox.js';
58
58
  import { readNonEmptyString, readMeshCompletionSummary } from './mesh-events-utils.js';
59
- import { getActiveDirectDispatches, getQueue } from './mesh-work-queue.js';
59
+ import { expandDaemonIdForms } from '@adhdev/mesh-shared';
60
+ import { getActiveDirectDispatches, getQueue, reclaimStrandedAssignedTask } from './mesh-work-queue.js';
60
61
  import { readLedgerEntries } from './mesh-ledger.js';
61
62
  import { pruneStaleDirectDispatches } from './mesh-active-work.js';
62
63
  import { reconcileDirectDispatchCompletionFromTranscript } from './mesh-events-stale.js';
@@ -119,16 +120,19 @@ interface LiveCoordinator {
119
120
  // - the daemon's canonical status id (`standalone_<machineId>` / `daemon_<machineId>`),
120
121
  // stamped by the MCP layer via ctx.localDaemonId (= getStatus().status.instanceId), or
121
122
  // - the bare machineId, stamped by the local queue-assignment path (loadConfig().machineId).
123
+ // - the config-form node daemonId (`daemon_<machineId>`), which the MCP layer's
124
+ // resolveCoordinatorDaemonId prefers and stamps onto direct-dispatch workers.
122
125
  // Draining with only one of these silently misses events stamped with the other —
123
- // the exact reason a generating coordinator never self-received local completions.
124
- // We accept BOTH so the drain matches regardless of which path stamped the event.
126
+ // the exact reason a generating coordinator never self-received local completions,
127
+ // and the base-node completion-surface bug (base completions land full-form
128
+ // `daemon_<machineId>` while a coordinator that only knows itself as bare
129
+ // `<machineId>` never matches them). We expand to EVERY equivalent form so the
130
+ // scope match (host gate, self-node detection, and the drain IN-filter downstream)
131
+ // succeeds regardless of which path stamped the event.
125
132
  function resolveCoordinatorDaemonIds(components: DaemonComponents): string[] {
126
- const ids = new Set<string>();
127
133
  const statusInstanceId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
128
- if (statusInstanceId) ids.add(statusInstanceId);
129
134
  const machineId = readNonEmptyString(loadConfig().machineId);
130
- if (machineId) ids.add(machineId);
131
- return [...ids];
135
+ return expandDaemonIdForms([statusInstanceId, machineId]);
132
136
  }
133
137
 
134
138
  // Whether THIS daemon is the coordinator/host for a mesh — i.e. the daemon that
@@ -316,6 +320,47 @@ function recordHeldTerminalEventsToLedger(
316
320
  // PHASE 2 — Live CLI inject. For each mesh that has a live CLI coordinator on
317
321
  // THIS daemon, drain the local queue and inject pending events into the PTY.
318
322
  // Unchanged from before.
323
+ // Bug B: how long a row may sit 'assigned' with an unconfirmed dispatch before the
324
+ // watchdog reclaims it. Must be comfortably larger than the per-dispatch confirm
325
+ // timeout (DISPATCH_CONFIRM_TIMEOUT_MS in mesh-events-coordinator) so a slow-but-live
326
+ // dispatch still inside its normal confirm window is never reclaimed early — this is
327
+ // the durable backstop for the case the in-process confirm timer can't cover (a timer
328
+ // lost to a daemon restart between claim and confirm).
329
+ const ASSIGNED_STRANDED_DEADLINE_MS = 5 * 60_000;
330
+
331
+ // PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
332
+ // flips a row to 'assigned' BEFORE the fire-and-forget dispatch runs. If that dispatch
333
+ // neither rejects (→ no .catch requeue) nor is confirmed delivered — a relay that hangs
334
+ // without acking, or a confirm timer lost across a restart — the row stays 'assigned'
335
+ // forever: it contributes 0 pending, so PHASE 3 (gated on pendingQueueTaskCount>0) never
336
+ // re-examines it, and nothing but a manual requeue clears it. This is that missing net.
337
+ //
338
+ // Regression guard: a row whose delivery IS confirmed (delivered/acked/completed) is a
339
+ // genuinely in-flight (or completion-lost) task — left to PHASE 4's completion reconcile,
340
+ // never reclaimed here. And the deadline is generous so a slow-but-live dispatch still in
341
+ // its normal confirm window is never reclaimed early. Reclaimed rows return to 'pending'
342
+ // with ownership cleared, so the PHASE 3 trigger below re-dispatches them this same tick.
343
+ function recoverStrandedAssignedDispatches(meshId: string, store: MeshRuntimeStore): void {
344
+ const assigned = getQueue(meshId, { status: ['assigned'] });
345
+ if (!assigned.length) return;
346
+ const nowMs = Date.now();
347
+ for (const row of assigned) {
348
+ const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
349
+ if (!Number.isFinite(dispatchedAtMs)) continue; // no dispatch ts → can't age it
350
+ if (nowMs - dispatchedAtMs < ASSIGNED_STRANDED_DEADLINE_MS) continue; // still in confirm window
351
+ if (store.taskHasConfirmedDelivery(meshId, row.id)) continue; // dispatched → PHASE 4's job
352
+ const reclaimed = reclaimStrandedAssignedTask(meshId, row.id, {
353
+ reason: 'assigned_stranded_dispatch_unconfirmed',
354
+ ageMs: nowMs - dispatchedAtMs,
355
+ });
356
+ if (reclaimed) {
357
+ LOG.warn('MeshReconcile', `Reclaimed stranded assigned task ${row.id} on mesh ${meshId} `
358
+ + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, dispatched `
359
+ + `${Math.round((nowMs - dispatchedAtMs) / 1000)}s ago, never confirmed delivered → ${reclaimed.status})`);
360
+ }
361
+ }
362
+ }
363
+
319
364
  export async function runMeshReconcileTick(components: DaemonComponents): Promise<void> {
320
365
  const localDaemonId = readNonEmptyString(loadConfig().machineId) || undefined;
321
366
  // The id-set used to scope the local queue drain (status id + machineId). See
@@ -360,6 +405,21 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
360
405
  }
361
406
  }
362
407
 
408
+ // ── PHASE 2.5: assigned-stranded dispatch watchdog (Bug B) ─────────────────
409
+ // Runs before PHASE 3 so any row it returns to 'pending' is re-dispatched by the
410
+ // PHASE 3 trigger in this same tick. See recoverStrandedAssignedDispatches.
411
+ if (store) {
412
+ for (const mesh of listMeshes()) {
413
+ const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
414
+ if (!daemonHostsMesh(mesh, selfIds)) continue;
415
+ try {
416
+ recoverStrandedAssignedDispatches(mesh.id, store);
417
+ } catch (e: any) {
418
+ LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
419
+ }
420
+ }
421
+ }
422
+
363
423
  // ── PHASE 3: recover pending queue claims for newly-idle sessions ──────────
364
424
  // The event-driven claim paths (agent:ready / agent:generating_completed in
365
425
  // mesh-events-coordinator) re-claim the queue the moment a session goes idle,
@@ -1075,6 +1075,23 @@ export class MeshRuntimeStore {
1075
1075
  }));
1076
1076
  }
1077
1077
 
1078
+ /**
1079
+ * Bug B watchdog support: true when at least one delivery record for the task has
1080
+ * reached a confirmed-handed-off status (delivered / acked / completed). The
1081
+ * assigned-stranded watchdog uses this to distinguish a dispatch that was never
1082
+ * confirmed (reclaimable) from one that WAS handed to the worker (a genuinely
1083
+ * in-flight or completion-lost task, which is PHASE 4's responsibility, not this
1084
+ * watchdog's). Indexed by (mesh_id, task_id).
1085
+ */
1086
+ taskHasConfirmedDelivery(meshId: string, taskId: string): boolean {
1087
+ const row = this.db.prepare(`
1088
+ SELECT 1 FROM mesh_session_delivery
1089
+ WHERE mesh_id = ? AND task_id = ? AND status IN ('delivered','acked','completed')
1090
+ LIMIT 1
1091
+ `).get(meshId, taskId) as { 1: number } | undefined;
1092
+ return !!row;
1093
+ }
1094
+
1078
1095
  expireStaleSessionDeliveries(meshId: string): void {
1079
1096
  const now = new Date().toISOString();
1080
1097
  this.db.prepare(`
@@ -5,6 +5,8 @@ import { MESH_CONVERGE_REFINE_TAG, resolveAutoConvergeCodeChange } from '../repo
5
5
  import { MeshRuntimeStore } from './mesh-runtime-store.js';
6
6
  import { getMesh } from '../config/mesh-config.js';
7
7
  import { LOG } from '../logging/logger.js';
8
+ import { appendLedgerEntry } from './mesh-ledger.js';
9
+ import type { MeshLedgerKind } from './mesh-ledger.js';
8
10
 
9
11
  export type MeshTaskStatus = 'pending' | 'assigned' | 'completed' | 'failed' | 'cancelled';
10
12
  export type MeshActiveTaskStatus = Extract<MeshTaskStatus, 'pending' | 'assigned'>;
@@ -339,6 +341,14 @@ export interface MeshWorkQueueEntry {
339
341
  requeueCount?: number;
340
342
  /** Max automatic requeue attempts. When requeueCount reaches this, task is auto-failed. */
341
343
  maxRetries?: number;
344
+ /**
345
+ * Bug B: number of times the reconcile assigned-stranded watchdog has reclaimed this
346
+ * row from 'assigned' back to 'pending' because its dispatch was never confirmed
347
+ * delivered. Separate from requeueCount (operator/execution retries) and bounded by
348
+ * MAX_STRANDED_RECLAIMS so a permanently-undeliverable target auto-fails rather than
349
+ * cycling reclaim→re-dispatch→strand forever.
350
+ */
351
+ strandedReclaimCount?: number;
342
352
  /** Last automatic queue session spin-up attempt, for mesh_view_queue/debug visibility. */
343
353
  autoLaunch?: {
344
354
  status: 'skipped' | 'started' | 'failed' | 'completed';
@@ -887,6 +897,85 @@ export function requeueTask(
887
897
  });
888
898
  }
889
899
 
900
+ /**
901
+ * Max times the assigned-stranded watchdog will reclaim a single task before giving
902
+ * up and failing it. Bounds the reclaim→re-dispatch→strand cycle so a permanently
903
+ * undeliverable target (e.g. a node whose transport is wedged) eventually fails and
904
+ * unblocks its dependents instead of looping every reconcile tick.
905
+ */
906
+ const MAX_STRANDED_RECLAIMS = 3;
907
+
908
+ /**
909
+ * Bug B: reclaim a task stuck in 'assigned' because its dispatch was never confirmed.
910
+ *
911
+ * claimNextTask atomically marks a row 'assigned' BEFORE the fire-and-forget dispatch
912
+ * runs. If that dispatch neither rejects (→ no .catch requeue) nor is confirmed
913
+ * delivered — a relay that hangs without acking, or a confirm timer lost across a
914
+ * daemon restart — the row stays 'assigned' forever, contributing 0 pending so PHASE 3
915
+ * reconcile never re-examines it. This returns such a row to 'pending' and clears its
916
+ * dead assignment ownership (node / session / provider / dispatchTimestamp) — the same
917
+ * ownership-clear requeueTask applies — so PHASE 3 can re-dispatch it onto a fresh idle
918
+ * session.
919
+ *
920
+ * Guarded to 'assigned' rows only (a completion/cancel that already moved the row off
921
+ * 'assigned' must never be resurrected) and bounded by MAX_STRANDED_RECLAIMS (beyond
922
+ * which the task is failed so dependents unblock).
923
+ */
924
+ export function reclaimStrandedAssignedTask(
925
+ meshId: string,
926
+ taskId: string,
927
+ opts?: { reason?: string; ageMs?: number } & MeshQueueMutationOptions,
928
+ ): MeshWorkQueueEntry | null {
929
+ requireMeshHostQueueOwner(opts);
930
+ return withQueueLock(meshId, () => {
931
+ const entry = MeshRuntimeStore.getInstance().findQueueEntryById(meshId, taskId);
932
+ if (!entry) return null;
933
+ // Only a still-assigned row is stranded. If a completion/cancel already moved it
934
+ // off 'assigned', there is nothing to reclaim — never resurrect a terminal row.
935
+ if (entry.status !== 'assigned') return null;
936
+ const now = new Date().toISOString();
937
+ const reason = opts?.reason || 'assigned_stranded_dispatch_unconfirmed';
938
+ const reclaims = (entry.strandedReclaimCount || 0) + 1;
939
+ const prevNode = entry.assignedNodeId;
940
+ const prevSession = entry.assignedSessionId;
941
+ // Always clear the dead assignment ownership so a re-claim starts clean and the
942
+ // assigned-counters (which filter status==='assigned') stop counting this row.
943
+ delete entry.assignedNodeId;
944
+ delete entry.assignedSessionId;
945
+ delete entry.assignedProviderType;
946
+ delete entry.dispatchTimestamp;
947
+ entry.strandedReclaimCount = reclaims;
948
+ entry.updatedAt = now;
949
+ if (reclaims > MAX_STRANDED_RECLAIMS) {
950
+ // Repeatedly undeliverable — stop cycling and fail it so dependents unblock.
951
+ entry.status = 'failed';
952
+ entry.cancelReason = `stranded_dispatch_unrecovered: reclaimed ${reclaims - 1} time(s) without a confirmed dispatch`;
953
+ MeshRuntimeStore.getInstance().updateQueueEntry(entry);
954
+ propagateDependencyFailure(meshId, taskId);
955
+ } else {
956
+ entry.status = 'pending';
957
+ entry.requeuedAt = now;
958
+ entry.requeueReason = reason;
959
+ MeshRuntimeStore.getInstance().updateQueueEntry(entry);
960
+ }
961
+ try {
962
+ appendLedgerEntry(meshId, {
963
+ kind: 'task_reclaimed' as MeshLedgerKind,
964
+ nodeId: prevNode,
965
+ sessionId: prevSession,
966
+ payload: {
967
+ taskId,
968
+ reason,
969
+ ...(typeof opts?.ageMs === 'number' ? { ageMs: opts.ageMs } : {}),
970
+ reclaimCount: reclaims,
971
+ outcome: entry.status,
972
+ },
973
+ });
974
+ } catch { /* ledger write is best-effort */ }
975
+ return entry;
976
+ });
977
+ }
978
+
890
979
  /**
891
980
  * Update the status of the task currently assigned to a specific session.
892
981
  */
@@ -8,4 +8,5 @@ export declare function pickAutoApprovalButton(buttons: string[] | null | undefi
8
8
  index: number;
9
9
  label: string;
10
10
  };
11
+ export declare function hasNegativeApprovalOption(buttons: string[] | null | undefined): boolean;
11
12
  export declare function formatAutoApprovalMessage(modalMessage?: string, buttonLabel?: string): string;
@@ -31,6 +31,16 @@ function isNegativeApprovalLabel(value: string): boolean {
31
31
  || /\bdo not\b/.test(label);
32
32
  }
33
33
 
34
+ /**
35
+ * True when any of the given button labels reads as a decline/negative option
36
+ * (No / Deny / Cancel / Skip / …). Used as the second half of an approval-modal
37
+ * structural anchor: a real approval modal offers BOTH an affirmative and a
38
+ * decline, which distinguishes it from a generic numbered menu or prose list.
39
+ */
40
+ export function hasNegativeApprovalOption(buttons: string[] | null | undefined): boolean {
41
+ return (buttons || []).some((button) => isNegativeApprovalLabel(String(button || '')));
42
+ }
43
+
34
44
  export function getApprovalPositiveHints(provider?: Pick<ProviderModule, 'approvalPositiveHints'> | null): string[] {
35
45
  const customHints = Array.isArray(provider?.approvalPositiveHints)
36
46
  ? provider.approvalPositiveHints