@adhdev/daemon-core 0.9.82-rc.537 → 0.9.82-rc.539

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,3 +2,17 @@ import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
2
2
  import type { LocalMeshEntry } from '../repo-mesh-types.js';
3
3
  export declare function reconcileUnterminatedDirectDispatches(components: DaemonComponents, mesh: LocalMeshEntry, selfIds: string[], localDaemonId: string | undefined): Promise<void>;
4
4
  export declare function autoPruneStaleDirectDispatches(components: DaemonComponents, mesh: LocalMeshEntry, selfIds: string[], localDaemonId: string | undefined, minAgeMs: number): Promise<void>;
5
+ export declare function pollAssignedTaskTerminalEvidence(components: DaemonComponents, mesh: {
6
+ id: string;
7
+ nodes?: Array<{
8
+ id: string;
9
+ daemonId?: string;
10
+ workspace?: string;
11
+ }>;
12
+ }, row: {
13
+ id: string;
14
+ assignedSessionId?: string;
15
+ assignedNodeId?: string;
16
+ assignedProviderType?: string;
17
+ dispatchTimestamp?: string;
18
+ }): Promise<'completed' | null>;
@@ -374,6 +374,28 @@ export declare function requeueTask(meshId: string, taskId: string, opts?: {
374
374
  /** Per-task retry cap override. Falls back to mesh policy maxTaskRetries (default 1). */
375
375
  maxRetries?: number;
376
376
  } & MeshQueueMutationOptions): MeshWorkQueueEntry | null;
377
+ /**
378
+ * TASK-PROMPT-REDRIVE-AFTER-COMPLETE: the reclaim reasons the assigned-stranded watchdog
379
+ * uses when it RE-DRIVES a delivered-but-not-terminal task (returns it to 'pending' so the
380
+ * SAME prompt is re-dispatched). These are distinct from `assigned_stranded_dispatch_unconfirmed`
381
+ * (a dispatch that was NEVER handed off — nothing ran, so a late completion is impossible).
382
+ *
383
+ * A re-drive assumes the worker never finished. But for an autoLaunch/worktree worker the
384
+ * turn-lifecycle events (agent:generating_started/completed) do NOT reliably reach the
385
+ * coordinator ledger, so the deadline can elapse and re-drive fire while the worker's genuine
386
+ * completion is merely LATE (observed live: it lands 0.9s–98s AFTER the reclaim). The late
387
+ * completion must then SUPERSEDE the re-drive rather than be dropped — the completion handler's
388
+ * flip-miss safety net checks a row reclaimed for one of these reasons within
389
+ * {@link REDRIVE_SUPERSEDE_WINDOW_MS} of its `requeuedAt`.
390
+ */
391
+ export declare const REDRIVE_RECLAIM_REASONS: ReadonlySet<string>;
392
+ /**
393
+ * How long after a re-drive reclaim's `requeuedAt` a late completion still supersedes the
394
+ * re-dispatch. Comfortably covers the observed 0.9s–98s completion-vs-reclaim race with margin,
395
+ * while staying far short of the time it would take a genuinely fresh re-dispatched turn to
396
+ * produce its OWN completion — so a real second turn is never mistaken for the superseded one.
397
+ */
398
+ export declare const REDRIVE_SUPERSEDE_WINDOW_MS: number;
377
399
  /**
378
400
  * Bug B: reclaim a task stuck in 'assigned' because its dispatch was never confirmed.
379
401
  *
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adhdev/daemon-core",
3
- "version": "0.9.82-rc.537",
3
+ "version": "0.9.82-rc.539",
4
4
  "description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -47,8 +47,8 @@
47
47
  "author": "vilmire",
48
48
  "license": "AGPL-3.0-or-later",
49
49
  "dependencies": {
50
- "@adhdev/mesh-shared": "0.9.82-rc.537",
51
- "@adhdev/session-host-core": "0.9.82-rc.537",
50
+ "@adhdev/mesh-shared": "0.9.82-rc.539",
51
+ "@adhdev/session-host-core": "0.9.82-rc.539",
52
52
  "@agentclientprotocol/sdk": "^0.16.1",
53
53
  "ajv": "^8.20.0",
54
54
  "ajv-formats": "^3.0.1",
Binary file
@@ -175,6 +175,7 @@ export class CliStateEngine {
175
175
  * paint and made the engine type "1" repeatedly into the prompt.
176
176
  */
177
177
  modalLostAt = 0;
178
+ private modalLostRecheckTimer: NodeJS.Timeout | null = null;
178
179
  private approvalExitTimeout: NodeJS.Timeout | null = null;
179
180
 
180
181
  // ── Response tracking ────────────────────────────
@@ -243,6 +244,12 @@ export class CliStateEngine {
243
244
  setStatus(status: CliSessionStatus['status'], trigger?: string): void {
244
245
  const prev = this.currentStatus;
245
246
  if (prev === status) return;
247
+ // Leaving waiting_approval — cancel any pending modal-lost recheck; it
248
+ // only exists to wake a quiescent PTY still pinned to waiting_approval.
249
+ if (prev === 'waiting_approval' && this.modalLostRecheckTimer) {
250
+ clearTimeout(this.modalLostRecheckTimer);
251
+ this.modalLostRecheckTimer = null;
252
+ }
246
253
  this.currentStatus = status;
247
254
  this.statusHistory.push({ status, at: Date.now(), trigger });
248
255
  if (this.statusHistory.length > 50) this.statusHistory.shift();
@@ -456,6 +463,7 @@ export class CliStateEngine {
456
463
  if (this.settleTimer) { clearTimeout(this.settleTimer); this.settleTimer = null; }
457
464
  if (this.idleTimeout) { clearTimeout(this.idleTimeout); this.idleTimeout = null; }
458
465
  if (this.approvalExitTimeout) { clearTimeout(this.approvalExitTimeout); this.approvalExitTimeout = null; }
466
+ if (this.modalLostRecheckTimer) { clearTimeout(this.modalLostRecheckTimer); this.modalLostRecheckTimer = null; }
459
467
  if (this.finishRetryTimer) { clearTimeout(this.finishRetryTimer); this.finishRetryTimer = null; }
460
468
  if (this.pendingScriptStatusTimer) { clearTimeout(this.pendingScriptStatusTimer); this.pendingScriptStatusTimer = null; }
461
469
  if (this.providerErrorRetryTimer) { clearTimeout(this.providerErrorRetryTimer); this.providerErrorRetryTimer = null; }
@@ -822,7 +830,23 @@ export class CliStateEngine {
822
830
  // signature each time, which typed the approval key
823
831
  // ("1") into the prompt repeatedly. Wait for the modal to
824
832
  // stay gone for `approvalCooldown` before clearing.
825
- if (this.currentStatus === 'waiting_approval' && this.activeModal) {
833
+ //
834
+ // (fix: kimi approve-resolve-stuck) The recovery used to require
835
+ // `this.activeModal` to be non-null. But a provider can be pinned
836
+ // to `waiting_approval` with `activeModal === null`: kimi's
837
+ // questionPattern (`run.*command`) false-positive-matches the
838
+ // user's echoed prompt ("✨ Run the shell command: …"), so
839
+ // detectStatus reports `waiting_approval` from the question cue
840
+ // alone while parseApproval extracts zero buttons (→ null modal,
841
+ // never captured). After the real approval resolves and kimi goes
842
+ // quiet, no further PTY output arrives, so this is the LAST
843
+ // settled evaluation — with the old `&& this.activeModal` guard it
844
+ // bare-returned and the FSM latched `waiting_approval` forever
845
+ // (the approval appears never to resolve). Recover whenever we are
846
+ // pinned to waiting_approval with no actionable modal, captured or
847
+ // not, and arm a re-check so a quiescent PTY still gets one more
848
+ // evaluation pass to reach the hysteresis deadline.
849
+ if (this.currentStatus === 'waiting_approval') {
826
850
  const lostAt = this.modalLostAt || Date.now();
827
851
  if (!this.modalLostAt) this.modalLostAt = lostAt;
828
852
  if (Date.now() - lostAt >= this.timeouts.approvalCooldown) {
@@ -830,6 +854,12 @@ export class CliStateEngine {
830
854
  this.modalLostAt = 0;
831
855
  this.setStatus('generating', 'approval_lost_modal');
832
856
  this.callbacks.onStatusChange();
857
+ } else {
858
+ // Not yet past the hysteresis window. The PTY may be
859
+ // quiescent (kimi emits nothing once idle), so schedule an
860
+ // explicit re-evaluation — otherwise no settle tick ever
861
+ // fires again and the recovery above is never reached.
862
+ this.armModalLostRecheck();
833
863
  }
834
864
  }
835
865
  return;
@@ -1151,6 +1181,26 @@ export class CliStateEngine {
1151
1181
 
1152
1182
  // ─── Helpers ────────────────────────────────────────────────────────────
1153
1183
 
1184
+ /**
1185
+ * Schedule one more settled evaluation while pinned to `waiting_approval`
1186
+ * with no actionable modal. The settled FSM normally only re-runs on new
1187
+ * PTY output; a provider whose modal cue lingers in a form detectStatus
1188
+ * still matches (e.g. kimi's questionPattern hitting the user echo) but
1189
+ * whose PTY has gone quiet would never get another evaluation, latching
1190
+ * `waiting_approval` forever. This timer guarantees the modal-lost recovery
1191
+ * in `applyWaitingApproval` is reached even against a silent PTY. It is a
1192
+ * no-op once the FSM leaves `waiting_approval` (the re-evaluation itself
1193
+ * takes the recovery branch and clears the state).
1194
+ */
1195
+ private armModalLostRecheck(): void {
1196
+ if (this.modalLostRecheckTimer) return;
1197
+ this.modalLostRecheckTimer = setTimeout(() => {
1198
+ this.modalLostRecheckTimer = null;
1199
+ if (this.currentStatus !== 'waiting_approval') return;
1200
+ this.evaluateSettled(this.transport.getSnapshot());
1201
+ }, this.timeouts.approvalCooldown);
1202
+ }
1203
+
1154
1204
  private armApprovalExitTimeout(): void {
1155
1205
  if (this.approvalExitTimeout) clearTimeout(this.approvalExitTimeout);
1156
1206
  this.approvalExitTimeout = setTimeout(() => {
@@ -745,8 +745,90 @@ function decideCliReadChatSource(args: {
745
745
  const supportsNative = supportsCliNativeTranscript(args.providerType, args.provider);
746
746
  const observation = buildObservationForCli(args, supportsNative);
747
747
  const sessionKey = chatSourceSessionKey(args.providerType, args.sessionId);
748
+
749
+ // STICKY-NATIVE: once a session's transcript is bound to native-history via
750
+ // a TRUSTED EXACT provider-session identity (the on-disk uuid == this
751
+ // session's own transcript file, not a recency/mtime guess), a single read
752
+ // that momentarily fails to reproduce the full native slice is a TRANSIENT
753
+ // gap, NOT evidence native regressed or vanished. Concretely, cursor-agent
754
+ // rewrites its JSONL tail mid-turn — the file is briefly empty, or exposes a
755
+ // SHRUNK slice (old assistant row dropped while the new turn is streamed) —
756
+ // so a per-read observation sees either `native_unavailable/empty` or a
757
+ // `native_regressed_shrunk`. Either one flips a NativeLocked machine to
758
+ // `pty-parser` for that read (and, after enough misses, permanently to
759
+ // PtyOnly). That is exactly the reported symptom: cursor chat provenance
760
+ // "flips back and forth between PTY and native-history".
761
+ //
762
+ // Fix: for a session that (a) resolved via a trusted EXACT identity and
763
+ // (b) is already committed to native (NativeLocked/Recovering), speculatively
764
+ // observe, and if the machine WOULD flip to pty-parser, roll the machine
765
+ // state back and HOLD native-history as authoritative for this read. Exact
766
+ // identity means the read is provably against THIS session's own transcript
767
+ // file, so a transient gap on it can only be a mid-write artifact — never a
768
+ // genuine session switch or data loss (cursor's file is cumulative). A
769
+ // workspace-heuristic (non-exact) read is never eligible, so a real session
770
+ // switch still flips normally.
771
+ const priorSnapshot = CHAT_SOURCE_REGISTRY.snapshotRecord(sessionKey);
772
+ const priorState = priorSnapshot?.state ?? CHAT_SOURCE_REGISTRY.getState(sessionKey);
773
+ const eligibleForStickyHold = args.trustedExactNativeIdentity === true
774
+ && (priorState.name === 'NativeLocked' || priorState.name === 'Recovering');
775
+
748
776
  let decision = CHAT_SOURCE_REGISTRY.observe(sessionKey, observation);
749
777
 
778
+ if (eligibleForStickyHold && decision.selected === 'pty-parser') {
779
+ // The machine flipped a native-committed, exact-identity session to PTY
780
+ // on this read. Treat it as a transient native gap: undo the observe and
781
+ // report native-history held. We surface whatever native rows this read
782
+ // DID map (empty on a true gap, a shrunk slice on a mid-write rewrite);
783
+ // the streaming/tail delivery layer already suppresses an empty tail from
784
+ // clobbering the last real tail, so holding native with a thin slice is
785
+ // strictly safer than emitting PTY under a native-authority session.
786
+ CHAT_SOURCE_REGISTRY.restoreRecord(sessionKey, priorSnapshot);
787
+ const heldNativeMessages: ChatMessage[] = observation.kind === 'native_present'
788
+ ? extractNativeMessagesFromResult(args.providerType, args.nativeHistoryResult)
789
+ : [];
790
+ const messageSource = buildCliMessageSourceProvenance({
791
+ selected: 'native-history',
792
+ provider: args.providerType,
793
+ nativeHandle: typeof args.nativeHistoryResult?.providerSessionId === 'string'
794
+ ? args.nativeHistoryResult.providerSessionId
795
+ : undefined,
796
+ sessionWorkspace: args.sessionWorkspace,
797
+ intendedWorkspace: args.intendedWorkspace,
798
+ transcriptWorkspace: undefined,
799
+ fallbackReason: 'native_history_transient_gap_held',
800
+ nativeSource: 'provider-native',
801
+ sourcePath: typeof args.nativeHistoryResult?.sourcePath === 'string' ? args.nativeHistoryResult.sourcePath : undefined,
802
+ sourceMtimeMs: typeof args.nativeHistoryResult?.sourceMtimeMs === 'number' ? args.nativeHistoryResult.sourceMtimeMs : undefined,
803
+ nativeHistoryCoverage: undefined,
804
+ partialReason: undefined,
805
+ unavailableReason: observation.kind === 'native_unavailable' ? observation.reason : undefined,
806
+ nativeMessages: heldNativeMessages,
807
+ ptyMessages: args.ptyMessages,
808
+ returnedMessages: heldNativeMessages,
809
+ safeMapping: args.safeMapping,
810
+ freshEnough: true,
811
+ ptyStatusApprovalOnly: true,
812
+ });
813
+ return {
814
+ decision: {
815
+ selected: 'native-history',
816
+ nextState: priorState,
817
+ transition: {
818
+ fromState: priorState.name,
819
+ toState: priorState.name,
820
+ event: 'NoOp',
821
+ cause: decision.transition.cause,
822
+ at: Date.now(),
823
+ },
824
+ lockState: { locked: priorState.name === 'NativeLocked' },
825
+ },
826
+ messageSource,
827
+ nativeMessages: heldNativeMessages,
828
+ nativeSelected: true,
829
+ };
830
+ }
831
+
750
832
  // A restored runtime can briefly expose different native slices while the
751
833
  // provider transcript settles (for example, startup/system rows may be
752
834
  // filtered after the first read). The source machine correctly treats a
@@ -1335,13 +1417,27 @@ function readCliProviderNativeHistory(agentStr: string, args: {
1335
1417
  const pinnedProviderSessionId = typeof args.pinnedProviderSessionId === 'string'
1336
1418
  ? args.pinnedProviderSessionId.trim()
1337
1419
  : '';
1338
- // Pin reuse (PRIMARY): a later read whose live binding is gone
1339
- // (historySessionId empty, no live spawnedAtMs) can still resolve to the
1340
- // session it was last bound to. Read THAT session directly by threading the
1341
- // pin through as historySessionIdsame code path an explicit session read
1342
- // takes — instead of fail-closing. Never overrides a caller-supplied
1343
- // historySessionId; only kicks in when there is none.
1344
- const effectiveHistorySessionId = args.historySessionId || (!canBindFromLiveSession ? pinnedProviderSessionId : '');
1420
+ // Pin reuse (PRIMARY): a read with no caller-supplied historySessionId can
1421
+ // still resolve to the session it was last bound to. Read THAT session
1422
+ // directly by threading the pin through as historySessionId same code path
1423
+ // an explicit session read takes instead of relying on the live spawn/cwd/
1424
+ // mtime heuristic. Never overrides a caller-supplied historySessionId; only
1425
+ // kicks in when there is none.
1426
+ //
1427
+ // The pin is preferred EVEN FOR A LIVE SESSION (canBindFromLiveSession).
1428
+ // The pin is keyed on this session's own mesh id (getBoundProviderSessionIdPin
1429
+ // (targetSessionId)) and holds the provider-native uuid proven in a prior
1430
+ // read, so it can only ever resolve to THIS session's own transcript — it
1431
+ // cannot alias a concurrent session sharing the cwd. Bypassing the pin while
1432
+ // a session is live (the old behaviour) forced the FIRST read of every new
1433
+ // turn back onto the spawn/mtime heuristic: cursor's native tail is briefly
1434
+ // stale (previous turn) versus the just-echoed PTY user line, so the
1435
+ // workspace-overlap safe-mapping gate fails and the read flips to PTY
1436
+ // (native_history_not_safely_mapped) before native re-locks. Preferring the
1437
+ // pin makes that first read an EXACT-identity lookup (trustedExactNativeIdentity
1438
+ // = true), which reads the correct cumulative file AND lets the STICKY-NATIVE
1439
+ // hold cover the turn boundary. This is the pin-bypass class fix.
1440
+ const effectiveHistorySessionId = args.historySessionId || pinnedProviderSessionId || '';
1345
1441
  // Last-resort workspace-latest (b): only when nothing above resolved a
1346
1442
  // session id AND no pin exists AND the caller opted in with a workspace.
1347
1443
  // Strictly behind pin reuse — pinnedProviderSessionId being set disables it.
@@ -414,3 +414,88 @@ export async function autoPruneStaleDirectDispatches(
414
414
  LOG.info('MeshReconcile', `Auto-pruned ${result.prunedCount} orphaned direct dispatch record(s) for mesh ${mesh.id}`);
415
415
  }
416
416
  }
417
+
418
+ // TASK-PROMPT-REDRIVE-AFTER-COMPLETE (Fix A-i). Single-shot transcript poll for a CLAIM-PATH
419
+ // (queue-assigned) row that PHASE 2.5's long delivered-no-turn deadline is about to RE-DRIVE.
420
+ //
421
+ // PHASE 4 (reconcileUnterminatedDirectDispatches) recovers a lost completion by re-reading the
422
+ // worker transcript, but it is bound to DIRECT-dispatch ledger rows — it never touches a
423
+ // claim-path queue row. So the F3 long-deadline reclaim reaches its 15-min deadline with an empty
424
+ // ledger for an autoLaunch/worktree worker whose generating_started/completed events never
425
+ // propagated, and re-drives a task the worker actually FINISHED (the owner's symptom). This poll
426
+ // gives that reclaim the SAME transcript evidence PHASE 4 uses, for the queue row: if the worker
427
+ // session is idle with a final assistant summary dated at/after this task's dispatch, the task is
428
+ // done — return its terminal outcome so the caller short-circuits to that status instead of
429
+ // reclaiming. Unlike PHASE 4 there is no acked-hold/grace machinery: the caller only invokes this
430
+ // AFTER the full 15-min deadline, so a single idle-with-final-assistant read is decisive.
431
+ //
432
+ // Conservative: a non-idle read, a read failure, no final summary, or a summary provably BEFORE
433
+ // dispatch all yield null (fall through to the caller's normal reclaim decision) — the poll can
434
+ // only PREVENT a wrong re-drive, never invent a completion.
435
+ export async function pollAssignedTaskTerminalEvidence(
436
+ components: DaemonComponents,
437
+ mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
438
+ row: { id: string; assignedSessionId?: string; assignedNodeId?: string; assignedProviderType?: string; dispatchTimestamp?: string },
439
+ ): Promise<'completed' | null> {
440
+ const sessionId = readNonEmptyString(row.assignedSessionId);
441
+ const nodeId = readNonEmptyString(row.assignedNodeId);
442
+ if (!sessionId || !nodeId) return null; // no worker to read
443
+
444
+ const node = (mesh.nodes ?? []).find(n => n.id === nodeId);
445
+ const nodeDaemonId = readNonEmptyString(node?.daemonId);
446
+ const localDaemonId = readNonEmptyString((components as { statusInstanceId?: string }).statusInstanceId);
447
+ const isLocalNode = !nodeDaemonId
448
+ || daemonIdsEquivalent(nodeDaemonId, localDaemonId)
449
+ || !!components.instanceManager.getInstance(sessionId);
450
+
451
+ const providerType = readNonEmptyString(row.assignedProviderType);
452
+ const readArgs: Record<string, unknown> = {
453
+ sessionId,
454
+ targetSessionId: sessionId,
455
+ tailLimit: 10,
456
+ ...(node?.workspace ? { workspace: node.workspace } : {}),
457
+ ...(providerType ? { agentType: providerType, providerType } : {}),
458
+ };
459
+
460
+ let payload: Record<string, unknown> | null = null;
461
+ try {
462
+ if (isLocalNode) {
463
+ const result = await components.commandHandler?.handle('read_chat', readArgs);
464
+ if (result && (result as { success?: boolean }).success === false) return null;
465
+ payload = unwrapReadChatPayload(result);
466
+ } else if (components.dispatchMeshCommand) {
467
+ const result = await components.dispatchMeshCommand(nodeDaemonId, 'read_chat', readArgs);
468
+ payload = unwrapReadChatPayload(result);
469
+ if (payload && (payload as { success?: boolean }).success === false) return null;
470
+ } else {
471
+ return null; // remote node, no P2P transport — can't read this tick
472
+ }
473
+ } catch {
474
+ return null; // transport error / session gone → inconclusive, let the caller decide
475
+ }
476
+ if (!payload) return null;
477
+
478
+ // Only a settled-idle session is a turn-end; a generating/waiting session is mid-turn and
479
+ // must NEVER be short-circuited to completed.
480
+ if (readChatPayloadStatus(payload) !== 'idle') return null;
481
+
482
+ const messages = Array.isArray(payload.messages) ? payload.messages as ChatMessage[] : [];
483
+ const evidence = extractFinalAssistantSummaryEvidence(messages);
484
+ if (!evidence.finalSummary) return null; // idle but no assistant result yet → not a turn-end
485
+
486
+ // Stale-summary guard (same bar as PHASE 4): a reused session's transcript tail may hold a
487
+ // PRIOR task's summary. Require the final assistant message to be dated at/after THIS task's
488
+ // dispatch. When either timestamp is unusable, do NOT short-circuit — fall through so we never
489
+ // synthesize a completion off a possibly-stale tail (the reclaim path is the safe default).
490
+ const dispatchedAtMs = Date.parse(readNonEmptyString(row.dispatchTimestamp));
491
+ const transcriptAtMs = Date.parse(evidence.transcriptMessageAt ?? '');
492
+ if (!(Number.isFinite(dispatchedAtMs) && Number.isFinite(transcriptAtMs) && transcriptAtMs >= dispatchedAtMs)) {
493
+ return null;
494
+ }
495
+
496
+ // Idle + a final assistant message dated after dispatch = the worker finished this turn. We
497
+ // cannot distinguish a self-reported failure from the plain transcript tail here (that lives
498
+ // in buildTaskCompletionEvidence's structured-result path), and the alternative — re-driving a
499
+ // finished worker — is strictly worse, so a proven turn-end short-circuits to 'completed'.
500
+ return 'completed';
501
+ }
@@ -4,7 +4,8 @@ import { getMesh, getMeshByRepo, listMeshes } from '../config/mesh-config.js';
4
4
  import { LOG } from '../logging/logger.js';
5
5
  import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
6
6
  import type { SessionRecoveryContext } from './mesh-ledger.js';
7
- import { updateSessionTaskStatus, enqueueTask, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents, getQueue } from './mesh-work-queue.js';
7
+ import { updateSessionTaskStatus, updateTaskStatus, enqueueTask, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents, getQueue, REDRIVE_RECLAIM_REASONS, REDRIVE_SUPERSEDE_WINDOW_MS } from './mesh-work-queue.js';
8
+ import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
8
9
  import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus, consumeSessionDelivery } from './mesh-delivery-policy.js';
9
10
  import { MeshRuntimeStore, pruneMeshRuntimeRetention } from './mesh-runtime-store.js';
10
11
  import { maybeInjectIdleActiveMissionReminder } from './mesh-idle-reminder.js';
@@ -810,6 +811,93 @@ function stopStaleMeshWorker(
810
811
  }
811
812
  }
812
813
 
814
+ /**
815
+ * TASK-PROMPT-REDRIVE-AFTER-COMPLETE (Fix C): a genuine completion echoed a taskId whose queue
816
+ * row is NOT 'assigned' anymore — because the assigned-stranded watchdog RE-DROVE it (long
817
+ * delivered-no-turn deadline / unknown-grace) while this very completion was still in flight.
818
+ * For an autoLaunch/worktree worker the turn-lifecycle events don't reliably reach the
819
+ * coordinator ledger, so the deadline fires before the completion propagates (observed live at
820
+ * 0.9s–98s late). The re-drive re-injects the SAME prompt into the already-finished worker (the
821
+ * owner's symptom). This late completion PROVES the original turn finished, so it SUPERSEDES the
822
+ * re-drive: flip the row terminal and stop any duplicate re-dispatch.
823
+ *
824
+ * Bounded to a row reclaimed for a RE-DRIVE reason within {@link REDRIVE_SUPERSEDE_WINDOW_MS} of
825
+ * its `requeuedAt` — outside that window a `pending` row is an unrelated retry (or a fresh
826
+ * re-dispatched turn whose own completion this is not), and is left untouched. Returns true when
827
+ * it superseded the re-drive (row flipped terminal), false to fall through to normal handling.
828
+ */
829
+ function supersedeRedriveReclaimForLateCompletion(
830
+ components: DaemonComponents,
831
+ meshId: string,
832
+ row: MeshWorkQueueEntry,
833
+ completingSessionId: string,
834
+ outcome: 'completed' | 'failed',
835
+ args: { nodeId?: string; event: string; metadataEvent: Record<string, unknown> },
836
+ ): boolean {
837
+ // Only a re-drive reclaim is superseded. A row reclaimed as never-delivered
838
+ // (assigned_stranded_dispatch_unconfirmed) never ran, so a completion for it is not a
839
+ // late race — leave it to the normal path. A row already terminal is a no-op.
840
+ if (!row.requeueReason || !REDRIVE_RECLAIM_REASONS.has(row.requeueReason)) return false;
841
+ if (row.status === 'completed' || row.status === 'failed' || row.status === 'cancelled') return false;
842
+ const requeuedAtMs = Date.parse(row.requeuedAt ?? '');
843
+ if (!Number.isFinite(requeuedAtMs)) return false;
844
+ if (Date.now() - requeuedAtMs > REDRIVE_SUPERSEDE_WINDOW_MS) return false;
845
+
846
+ // The re-drive may have already re-dispatched the SAME prompt onto a FRESH session (row is
847
+ // 'assigned' again to a different session). That duplicate is executing work the original
848
+ // worker already finished — stop it so the prompt is not run twice. (When the row is still
849
+ // 'pending' there is no duplicate yet; flipping it terminal below prevents PHASE 3 from ever
850
+ // re-dispatching it.)
851
+ const reDispatchedSessionId = row.assignedSessionId;
852
+ if (
853
+ row.status === 'assigned'
854
+ && reDispatchedSessionId
855
+ && !sessionIdsEquivalent(reDispatchedSessionId, completingSessionId)
856
+ ) {
857
+ stopStaleMeshWorker(components, {
858
+ meshId,
859
+ sessionId: reDispatchedSessionId,
860
+ nodeId: row.assignedNodeId,
861
+ providerType: row.assignedProviderType,
862
+ });
863
+ }
864
+
865
+ // Flip the row terminal by its exact id (immune to the cleared session ownership) and record
866
+ // the terminal ledger evidence, so the reconcile watchdog's terminal-ledger branch also sees
867
+ // it and no further reclaim fires.
868
+ endTaskDispatchInFlight(meshId, row.id);
869
+ updateTaskStatus(meshId, row.id, outcome === 'completed' ? 'completed' : 'failed');
870
+ if (!findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })) {
871
+ try {
872
+ appendLedgerEntry(meshId, {
873
+ kind: outcome === 'completed' ? 'task_completed' : 'task_failed',
874
+ sessionId: completingSessionId,
875
+ nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
876
+ providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
877
+ payload: {
878
+ taskId: row.id,
879
+ event: args.event,
880
+ source: 'redrive_late_completion_supersede',
881
+ reclaimReason: row.requeueReason,
882
+ reclaimAgeMs: Date.now() - requeuedAtMs,
883
+ finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
884
+ },
885
+ });
886
+ } catch { /* best-effort ledger write */ }
887
+ }
888
+ LOG.warn('MeshQueue', `Late completion superseded re-drive for task ${row.id} on mesh ${meshId} `
889
+ + `(reclaimed '${row.requeueReason}' ${Math.round((Date.now() - requeuedAtMs) / 1000)}s ago; completing session ${completingSessionId}) `
890
+ + `→ flipped ${outcome}${(row.status === 'assigned' && reDispatchedSessionId && !sessionIdsEquivalent(reDispatchedSessionId, completingSessionId)) ? `, stopped duplicate re-dispatch on ${reDispatchedSessionId}` : ''}`);
891
+ traceMeshEventDrop('redrive_late_completion_supersede', {
892
+ taskId: row.id,
893
+ sessionId: completingSessionId,
894
+ nodeId: row.assignedNodeId ?? args.nodeId,
895
+ meshId,
896
+ event: args.event,
897
+ }, `${row.requeueReason} ${Math.round((Date.now() - requeuedAtMs) / 1000)}s → ${outcome}`);
898
+ return true;
899
+ }
900
+
813
901
  function injectMeshSystemMessage(components: DaemonComponents, args: {
814
902
  meshId: string;
815
903
  sourceInstanceId?: string;
@@ -1022,6 +1110,13 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
1022
1110
  },
1023
1111
  });
1024
1112
  }
1113
+ } else if (strandedRow && supersedeRedriveReclaimForLateCompletion(components, args.meshId, strandedRow, sessionId, outcome, args)) {
1114
+ // TASK-PROMPT-REDRIVE-AFTER-COMPLETE (Fix C, late-completion supersede): the row
1115
+ // is NOT 'assigned' — it was re-driven (reclaimed → 'pending', or already
1116
+ // re-dispatched to a fresh session) because the long delivered-no-turn deadline
1117
+ // fired before this genuine completion propagated. The completion proves the
1118
+ // ORIGINAL worker finished, so it SUPERSEDES the re-drive: the helper flipped the
1119
+ // row terminal and stopped any duplicate re-dispatch (handled inside).
1025
1120
  }
1026
1121
  } catch { /* best-effort safety net — never fail the completion path */ }
1027
1122
  }
@@ -1967,10 +1967,14 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
1967
1967
  // a launch that can never claim. Mirrors claimNextQueueTask's convergence gate.
1968
1968
  if (task.taskMode === 'convergence' && node?.isLocalWorktree === true) return false;
1969
1969
  // Skip nodes that can never satisfy requiredTags regardless of which provider
1970
- // from providerPriority is selected. A node satisfies tags if at least one
1971
- // provider in its priority list would produce matching capability tags.
1970
+ // is selected. A node satisfies tags if at least one provider it can launch
1971
+ // would produce matching capability tags. Enumerate providers from the node's
1972
+ // capability slots (the single source of truth — a provider that lives only in
1973
+ // slots, e.g. cursor-cli, is otherwise invisible to providerPriority-keyed
1974
+ // enumeration), falling back to the legacy providerPriority.
1972
1975
  if (task.requiredTags?.length) {
1973
- const priorities = normalizeProviderPriority(node?.policy);
1976
+ const slotProviders = resolveNodeCapabilitySlots(node).map(s => s.provider).filter(Boolean);
1977
+ const priorities = slotProviders.length ? slotProviders : normalizeProviderPriority(node?.policy);
1974
1978
  const providerCandidates = priorities.length ? priorities : [undefined as unknown as string];
1975
1979
  return providerCandidates.some(p =>
1976
1980
  nodeSatisfiesRequiredTags(task.requiredTags, buildMeshNodeCapabilityTags(node, p))
@@ -79,6 +79,7 @@ import { pullRemoteNodeQueues } from './mesh-remote-event-pull.js';
79
79
  import {
80
80
  reconcileUnterminatedDirectDispatches,
81
81
  autoPruneStaleDirectDispatches,
82
+ pollAssignedTaskTerminalEvidence,
82
83
  } from './mesh-completion-synthesis.js';
83
84
 
84
85
  // Re-export the extracted public API so existing importers (mesh-events.ts barrel;
@@ -737,7 +738,12 @@ export function __resetReclaimUnknownStreakForTests(): void {
737
738
  // never reclaimed here. And the deadline is generous so a slow-but-live dispatch still in
738
739
  // its normal confirm window is never reclaimed early. Reclaimed rows return to 'pending'
739
740
  // with ownership cleared, so the PHASE 3 trigger below re-dispatches them this same tick.
740
- function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId: string, store: MeshRuntimeStore): void {
741
+ async function recoverStrandedAssignedDispatches(
742
+ components: DaemonComponents,
743
+ mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
744
+ store: MeshRuntimeStore,
745
+ ): Promise<void> {
746
+ const meshId = mesh.id;
741
747
  const assigned = getQueue(meshId, { status: ['assigned'] });
742
748
  if (!assigned.length) return;
743
749
  const nowMs = Date.now();
@@ -910,6 +916,51 @@ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId:
910
916
  }
911
917
  reclaimReason = 'reclaim_after_unknown_grace';
912
918
  }
919
+ // TASK-PROMPT-REDRIVE-AFTER-COMPLETE (Fix A-i): before re-driving, poll the worker
920
+ // transcript for terminal evidence — the SAME check PHASE 4 does for direct dispatches,
921
+ // now for this claim-path queue row. An autoLaunch/worktree worker's
922
+ // generating_started/completed events don't reliably reach the coordinator ledger, so
923
+ // the ledger check above (findTerminalLedgerEvidenceForTask) can be empty at the 15-min
924
+ // deadline for a task the worker actually FINISHED — and re-driving then re-injects the
925
+ // same prompt into the already-idle worker (the owner's symptom). If the worker is idle
926
+ // with a final assistant summary dated after dispatch, the task is done: flip it
927
+ // 'completed' instead of reclaiming. Conservative by construction (mid-turn / no
928
+ // summary / stale summary / unreadable → null → fall through to the reclaim below), so
929
+ // this can only PREVENT a wrong re-drive, never invent a completion. Runs only at the
930
+ // deadline (rare), so the extra read is not a hot-path cost.
931
+ const terminalEvidence = await pollAssignedTaskTerminalEvidence(components, mesh, row);
932
+ if (terminalEvidence) {
933
+ deliveredNoTurnUnknownStreak.delete(streakKey);
934
+ // updateTaskStatus ends the single-flight dispatch window on any transition off
935
+ // 'assigned', so a later requeue/re-dispatch is never blocked by a stale mark.
936
+ updateTaskStatus(meshId, row.id, terminalEvidence);
937
+ if (!findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })) {
938
+ try {
939
+ appendLedgerEntry(meshId, {
940
+ kind: terminalEvidence === 'completed' ? 'task_completed' : 'task_failed',
941
+ nodeId: row.assignedNodeId,
942
+ sessionId: row.assignedSessionId,
943
+ providerType: row.assignedProviderType,
944
+ payload: {
945
+ taskId: row.id,
946
+ event: 'agent:generating_completed',
947
+ source: 'redrive_deadline_transcript_evidence',
948
+ },
949
+ });
950
+ } catch { /* best-effort ledger write */ }
951
+ }
952
+ LOG.warn('MeshReconcile', `Skipped delivered-no-turn re-drive for task ${row.id} on mesh ${meshId} `
953
+ + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}): worker transcript is idle with a `
954
+ + `final assistant message after dispatch — the completion event was lost/late, task is ${terminalEvidence}, NOT re-driving`);
955
+ traceMeshEventDrop('redrive_deadline_transcript_completed', {
956
+ taskId: row.id,
957
+ sessionId: row.assignedSessionId,
958
+ nodeId: row.assignedNodeId,
959
+ meshId,
960
+ event: 'agent:generating_completed',
961
+ }, `${reclaimReason} → transcript ${terminalEvidence}`);
962
+ continue;
963
+ }
913
964
  const reclaimedLost = reclaimStrandedAssignedTask(meshId, row.id, {
914
965
  reason: reclaimReason,
915
966
  ageMs: nowMs - dispatchedAtMs,
@@ -1108,7 +1159,7 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
1108
1159
  const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
1109
1160
  if (!daemonHostsMesh(mesh, selfIds)) continue;
1110
1161
  try {
1111
- recoverStrandedAssignedDispatches(components, mesh.id, store);
1162
+ await recoverStrandedAssignedDispatches(components, mesh, store);
1112
1163
  } catch (e: any) {
1113
1164
  LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
1114
1165
  }