@adhdev/daemon-core 0.9.82-rc.196 → 0.9.82-rc.198

package/src/mesh/mesh-events.ts

@@ -10,6 +10,7 @@ import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
 import { buildMeshNodeCapabilityTags, claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches } from './mesh-work-queue.js';
 import { MeshRuntimeStore } from './mesh-runtime-store.js';
 import { fastForwardMeshNode } from './mesh-fast-forward.js';
+import { createSessionDelivery, updateSessionDeliveryStatus, recordCompletionConflict } from './mesh-delivery-policy.js';
 
 // ---------------------------------------------------------------------------
 // Remote Node Idle Session Tracking
@@ -499,10 +500,27 @@ function isDuplicateMeshCompletionEvent(args: {
     timestamp?: number | null;
     finalSummary?: string;
     coordinatorDaemonId?: string;
+    taskId?: string;
+    nodeId?: string;
 }): boolean {
     const fingerprint = buildMeshCompletionFingerprint(args);
     if (!fingerprint) return false;
-    if (hasFingerprintSeen(fingerprint)) return true;
+    if (hasFingerprintSeen(fingerprint)) {
+        // Suppressed duplicate — but if we have a taskId and it differs from what the
+        // fingerprint was stamped for, record a conflict diagnostic so it doesn't disappear silently.
+        // (We can't recover the original taskId from the fingerprint alone, so we record
+        //  the conflicting taskId/session as a diagnostic for coordinator inspection.)
+        if (args.taskId) {
+            recordCompletionConflict({
+                meshId: args.meshId,
+                fingerprint,
+                conflictingTaskId: args.taskId,
+                conflictingSessionId: args.sessionId,
+                event: args.event,
+            });
+        }
+        return true;
+    }
     recordFingerprintSeen(fingerprint);
     return false;
 }
@@ -837,13 +855,27 @@ export function tryAssignQueueTask(
     if (node?.daemonId && components.dispatchMeshCommand) {
         const isLocalNode = components.cliManager.adapters.has(sessionId);
         if (!isLocalNode) {
+            // Create delivery record before attempting P2P send
+            const delivery = createSessionDelivery({
+                meshId,
+                nodeId,
+                sessionId,
+                providerType,
+                taskId: task.id,
+                kind: 'task',
+                message: task.message,
+                status: 'delivering',
+            });
             components.dispatchMeshCommand(node.daemonId, 'agent_command', {
                 targetSessionId: sessionId,
                 cliType: providerType,
                 action: 'send_chat',
                 message: task.message,
+            }).then(() => {
+                updateSessionDeliveryStatus(delivery.id, 'delivered');
             }).catch((e: any) => {
                 LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
+                updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
                 // Revert to pending so the task can be retried rather than permanently failing
                 updateTaskStatus(meshId, task.id, 'pending');
                 try {
@@ -851,7 +883,7 @@ export function tryAssignQueueTask(
                         kind: 'dispatch_failed' as any,
                         nodeId,
                         sessionId,
-                        payload: { taskId: task.id, error: e?.message, retryable: true },
+                        payload: { taskId: task.id, deliveryId: delivery.id, error: e?.message, retryable: true },
                     });
                 } catch { /* ledger write is best-effort */ }
             });
@@ -859,14 +891,27 @@ export function tryAssignQueueTask(
         }
     }
 
-    // Local routing
+    // Local routing — create delivery record before send_chat
+    const delivery = createSessionDelivery({
+        meshId,
+        nodeId,
+        sessionId,
+        providerType,
+        taskId: task.id,
+        kind: 'task',
+        message: task.message,
+        status: 'delivering',
+    });
     components.cliManager.handleCliCommand('agent_command', {
         targetSessionId: sessionId,
         cliType: providerType,
         action: 'send_chat',
         message: task.message,
+    }).then(() => {
+        updateSessionDeliveryStatus(delivery.id, 'delivered');
     }).catch((e: any) => {
         LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
+        updateSessionDeliveryStatus(delivery.id, 'failed', { lastError: e?.message, incrementAttempt: true });
         updateTaskStatus(meshId, task.id, 'failed');
     });
 
@@ -1625,6 +1670,8 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
             // Scope dedup to the coordinator daemon so two coordinators for the same mesh
             // don't suppress each other's completion events via shared fingerprint table.
             coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
+            taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
+            nodeId: eventNodeId || undefined,
         });
         if (duplicateCompletion) {
             LOG.info('MeshEvents', `Suppressed duplicate completion for mesh ${args.meshId} session ${eventSessionId}`);
@@ -1641,6 +1688,8 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
             timestamp: eventTimestamp,
             finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
             coordinatorDaemonId: workerCoordinatorDaemonId || undefined,
+            taskId: readNonEmptyString(args.metadataEvent.taskId) || undefined,
+            nodeId: eventNodeId || undefined,
         });
         if (duplicateStopped) {
             LOG.info('MeshEvents', `Suppressed duplicate stopped event for mesh ${args.meshId} session ${eventSessionId}`);

package/src/mesh/mesh-runtime-store.ts

@@ -143,6 +143,49 @@ export class MeshRuntimeStore {
                 metadata TEXT,
                 PRIMARY KEY (node_id, session_id)
             );
+
+            CREATE TABLE IF NOT EXISTS mesh_session_delivery (
+                id TEXT PRIMARY KEY,
+                mesh_id TEXT NOT NULL,
+                node_id TEXT,
+                session_id TEXT,
+                provider_type TEXT,
+                task_id TEXT,
+                kind TEXT NOT NULL,
+                priority INTEGER NOT NULL DEFAULT 0,
+                message TEXT NOT NULL,
+                status TEXT NOT NULL DEFAULT 'queued',
+                deliver_after TEXT,
+                expires_at TEXT,
+                attempt_count INTEGER NOT NULL DEFAULT 0,
+                source_coordinator_session_id TEXT,
+                source_coordinator_daemon_id TEXT,
+                last_error TEXT,
+                created_at TEXT NOT NULL,
+                updated_at TEXT NOT NULL
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_mesh_session_delivery_mesh_status
+                ON mesh_session_delivery(mesh_id, status, created_at);
+            CREATE INDEX IF NOT EXISTS idx_mesh_session_delivery_session
+                ON mesh_session_delivery(mesh_id, session_id, status);
+            CREATE INDEX IF NOT EXISTS idx_mesh_session_delivery_task
+                ON mesh_session_delivery(mesh_id, task_id);
+
+            CREATE TABLE IF NOT EXISTS mesh_completion_conflicts (
+                id TEXT PRIMARY KEY,
+                mesh_id TEXT NOT NULL,
+                fingerprint TEXT NOT NULL,
+                conflicting_task_id TEXT,
+                conflicting_session_id TEXT,
+                original_task_id TEXT,
+                original_session_id TEXT,
+                event TEXT NOT NULL,
+                created_at TEXT NOT NULL
+            );
+
+            CREATE INDEX IF NOT EXISTS idx_mesh_completion_conflicts_mesh
+                ON mesh_completion_conflicts(mesh_id, created_at);
         `);
     }
 
@@ -547,4 +590,180 @@ export class MeshRuntimeStore {
     pruneExpiredRemoteIdleSessions(): void {
         this.db.prepare('DELETE FROM remote_idle_sessions WHERE expires_at <= ?').run(Date.now());
     }
+
+    // ── Session Delivery Queue ───────────────────────────────────────────────
+
+    insertSessionDelivery(entry: {
+        id: string;
+        meshId: string;
+        nodeId?: string;
+        sessionId?: string;
+        providerType?: string;
+        taskId?: string;
+        kind: string;
+        priority?: number;
+        message: string;
+        status: string;
+        deliverAfter?: string;
+        expiresAt?: string;
+        sourceCoordinatorSessionId?: string;
+        sourceCoordinatorDaemonId?: string;
+        createdAt: string;
+        updatedAt: string;
+    }): void {
+        this.db.prepare(`
+            INSERT OR REPLACE INTO mesh_session_delivery (
+                id, mesh_id, node_id, session_id, provider_type, task_id, kind, priority,
+                message, status, deliver_after, expires_at, attempt_count,
+                source_coordinator_session_id, source_coordinator_daemon_id,
+                last_error, created_at, updated_at
+            ) VALUES (
+                @id, @meshId, @nodeId, @sessionId, @providerType, @taskId, @kind, @priority,
+                @message, @status, @deliverAfter, @expiresAt, 0,
+                @sourceCoordinatorSessionId, @sourceCoordinatorDaemonId,
+                NULL, @createdAt, @updatedAt
+            )
+        `).run({
+            id: entry.id,
+            meshId: entry.meshId,
+            nodeId: entry.nodeId ?? null,
+            sessionId: entry.sessionId ?? null,
+            providerType: entry.providerType ?? null,
+            taskId: entry.taskId ?? null,
+            kind: entry.kind,
+            priority: entry.priority ?? 0,
+            message: entry.message,
+            status: entry.status,
+            deliverAfter: entry.deliverAfter ?? null,
+            expiresAt: entry.expiresAt ?? null,
+            sourceCoordinatorSessionId: entry.sourceCoordinatorSessionId ?? null,
+            sourceCoordinatorDaemonId: entry.sourceCoordinatorDaemonId ?? null,
+            createdAt: entry.createdAt,
+            updatedAt: entry.updatedAt,
+        });
+        this.maybeCheckpointWal();
+    }
+
+    updateSessionDeliveryStatus(id: string, status: string, opts?: { lastError?: string; incrementAttempt?: boolean }): void {
+        const now = new Date().toISOString();
+        if (opts?.incrementAttempt) {
+            this.db.prepare(`
+                UPDATE mesh_session_delivery
+                SET status = @status, last_error = @lastError, attempt_count = attempt_count + 1, updated_at = @updatedAt
+                WHERE id = @id
+            `).run({ id, status, lastError: opts?.lastError ?? null, updatedAt: now });
+        } else {
+            this.db.prepare(`
+                UPDATE mesh_session_delivery
+                SET status = @status, last_error = @lastError, updated_at = @updatedAt
+                WHERE id = @id
+            `).run({ id, status, lastError: opts?.lastError ?? null, updatedAt: now });
+        }
+    }
+
+    getActiveSessionDeliveries(meshId: string, sessionId?: string): Array<{
+        id: string; meshId: string; nodeId: string | null; sessionId: string | null;
+        providerType: string | null; taskId: string | null; kind: string; priority: number;
+        message: string; status: string; deliverAfter: string | null; expiresAt: string | null;
+        attemptCount: number; sourceCoordinatorSessionId: string | null;
+        sourceCoordinatorDaemonId: string | null; lastError: string | null;
+        createdAt: string; updatedAt: string;
+    }> {
+        const now = new Date().toISOString();
+        const sql = sessionId
+            ? `SELECT * FROM mesh_session_delivery WHERE mesh_id = ? AND session_id = ? AND status NOT IN ('delivered','completed','failed','expired','cancelled') AND (expires_at IS NULL OR expires_at > ?) ORDER BY priority DESC, created_at ASC`
+            : `SELECT * FROM mesh_session_delivery WHERE mesh_id = ? AND status NOT IN ('delivered','completed','failed','expired','cancelled') AND (expires_at IS NULL OR expires_at > ?) ORDER BY priority DESC, created_at ASC`;
+        const rows = sessionId
+            ? this.db.prepare(sql).all(meshId, sessionId, now) as Array<Record<string, unknown>>
+            : this.db.prepare(sql).all(meshId, now) as Array<Record<string, unknown>>;
+        return rows.map(r => ({
+            id: r.id as string,
+            meshId: r.mesh_id as string,
+            nodeId: r.node_id as string | null,
+            sessionId: r.session_id as string | null,
+            providerType: r.provider_type as string | null,
+            taskId: r.task_id as string | null,
+            kind: r.kind as string,
+            priority: r.priority as number,
+            message: r.message as string,
+            status: r.status as string,
+            deliverAfter: r.deliver_after as string | null,
+            expiresAt: r.expires_at as string | null,
+            attemptCount: r.attempt_count as number,
+            sourceCoordinatorSessionId: r.source_coordinator_session_id as string | null,
+            sourceCoordinatorDaemonId: r.source_coordinator_daemon_id as string | null,
+            lastError: r.last_error as string | null,
+            createdAt: r.created_at as string,
+            updatedAt: r.updated_at as string,
+        }));
+    }
+
+    expireStaleSessionDeliveries(meshId: string): void {
+        const now = new Date().toISOString();
+        this.db.prepare(`
+            UPDATE mesh_session_delivery
+            SET status = 'expired', updated_at = ?
+            WHERE mesh_id = ? AND expires_at IS NOT NULL AND expires_at <= ?
+              AND status NOT IN ('delivered','completed','failed','expired','cancelled')
+        `).run(now, meshId, now);
+    }
+
+    deleteSessionDeliveries(meshId: string): void {
+        this.db.prepare('DELETE FROM mesh_session_delivery WHERE mesh_id = ?').run(meshId);
+    }
+
+    // ── Completion Conflict Diagnostics ──────────────────────────────────────
+
+    recordCompletionConflict(entry: {
+        id: string;
+        meshId: string;
+        fingerprint: string;
+        conflictingTaskId?: string;
+        conflictingSessionId?: string;
+        originalTaskId?: string;
+        originalSessionId?: string;
+        event: string;
+        createdAt: string;
+    }): void {
+        this.db.prepare(`
+            INSERT OR IGNORE INTO mesh_completion_conflicts
+                (id, mesh_id, fingerprint, conflicting_task_id, conflicting_session_id,
+                 original_task_id, original_session_id, event, created_at)
+            VALUES (@id, @meshId, @fingerprint, @conflictingTaskId, @conflictingSessionId,
+                    @originalTaskId, @originalSessionId, @event, @createdAt)
+        `).run({
+            id: entry.id,
+            meshId: entry.meshId,
+            fingerprint: entry.fingerprint,
+            conflictingTaskId: entry.conflictingTaskId ?? null,
+            conflictingSessionId: entry.conflictingSessionId ?? null,
+            originalTaskId: entry.originalTaskId ?? null,
+            originalSessionId: entry.originalSessionId ?? null,
+            event: entry.event,
+            createdAt: entry.createdAt,
+        });
+        this.maybeCheckpointWal();
+    }
+
+    getRecentCompletionConflicts(meshId: string, limitMs: number = 60 * 60 * 1000): Array<{
+        id: string; meshId: string; fingerprint: string; conflictingTaskId: string | null;
+        conflictingSessionId: string | null; originalTaskId: string | null;
+        originalSessionId: string | null; event: string; createdAt: string;
+    }> {
+        const cutoff = new Date(Date.now() - limitMs).toISOString();
+        const rows = this.db.prepare(
+            'SELECT * FROM mesh_completion_conflicts WHERE mesh_id = ? AND created_at >= ? ORDER BY created_at DESC LIMIT 50'
+        ).all(meshId, cutoff) as Array<Record<string, unknown>>;
+        return rows.map(r => ({
+            id: r.id as string,
+            meshId: r.mesh_id as string,
+            fingerprint: r.fingerprint as string,
+            conflictingTaskId: r.conflicting_task_id as string | null,
+            conflictingSessionId: r.conflicting_session_id as string | null,
+            originalTaskId: r.original_task_id as string | null,
+            originalSessionId: r.original_session_id as string | null,
+            event: r.event as string,
+            createdAt: r.created_at as string,
+        }));
+    }
 }

package/src/providers/spec/cli-adapter.ts

@@ -349,6 +349,10 @@ export class SpecCliAdapter implements CliAdapter {
             exited: this.exited,
             screen,
             sections,
+            stateHistory: this.driver.getStateHistory(),
+            idleHoldPending: this.driver.hasIdleHoldPending(),
+            lastBusyAt: this.driver.getLastBusyAt(),
+            specPath: this.driver.getSpecPath(),
         };
     }
     getRuntimeMetadata(): unknown {

package/src/providers/spec/driver.ts

@@ -218,7 +218,15 @@ export class SpecDriver {
      *  explicit wake-up there's nothing to trigger the busy → idle
      *  downshift. */
     private busyExpiryTimer: ReturnType<typeof setTimeout> | null = null;
+    /** Pending idle-commit timer. Armed when the evaluator first returns idle;
+     *  fires after idle_hold_ms if no non-idle reading has cancelled it. */
+    private idleHoldTimer: ReturnType<typeof setTimeout> | null = null;
+    /** State snapshot captured when the idle hold was armed — emitted on commit. */
+    private pendingIdleState: SpecEvaluation['state'] | null = null;
     private specWatcher: fs.FSWatcher | null = null;
+    /** Ring buffer of committed state transitions (max 50). */
+    private stateHistory: Array<{ stateId: string; label: string; at: number; durationMs: number }> = [];
+    private prevStateAt = 0;
 
     constructor(private readonly opts: SpecDriverOpts) {
         this.loadSpecOrThrow();
@@ -277,10 +285,33 @@ export class SpecDriver {
     shutdown(): void {
         for (const t of this.delegateTimers.values()) clearTimeout(t);
         this.delegateTimers.clear();
+        this.cancelIdleHold();
+        if (this.busyExpiryTimer) { clearTimeout(this.busyExpiryTimer); this.busyExpiryTimer = null; }
         this.specWatcher?.close();
         this.adapter.kill();
     }
 
+    private cancelIdleHold(): void {
+        if (this.idleHoldTimer) { clearTimeout(this.idleHoldTimer); this.idleHoldTimer = null; }
+        this.pendingIdleState = null;
+    }
+
+    private pushHistory(stateId: string, label: string): void {
+        const now = Date.now();
+        const durationMs = this.prevStateAt > 0 ? now - this.prevStateAt : 0;
+        this.prevStateAt = now;
+        this.stateHistory.push({ stateId, label, at: now, durationMs });
+        if (this.stateHistory.length > 50) this.stateHistory.shift();
+    }
+
+    getStateHistory(): ReadonlyArray<{ stateId: string; label: string; at: number; durationMs: number }> {
+        return this.stateHistory;
+    }
+
+    getLastBusyAt(): number { return this.lastBusyAt; }
+    hasIdleHoldPending(): boolean { return this.idleHoldTimer !== null; }
+    getSpecPath(): string { return this.opts.specPath; }
+
     // ────────────────────────────────────────────────────────────────────
     // Loading & adapter wiring
     // ────────────────────────────────────────────────────────────────────
@@ -307,9 +338,13 @@ export class SpecDriver {
 
     private armSpecWatcher(): void {
         try {
-            this.specWatcher = fs.watch(this.opts.specPath, { persistent: false }, () => {
-                // Re-read; if it parses, replace and re-evaluate. Errors are
-                // surfaced to the dashboard so the spec author sees them live.
+            // Watch the parent directory so we catch atomic replacements
+            // (cp, install scripts) that create a new inode — a file-level
+            // watch misses those on macOS because the original inode is gone.
+            const dir = path.dirname(this.opts.specPath);
+            const base = path.basename(this.opts.specPath);
+            this.specWatcher = fs.watch(dir, { persistent: false }, (_event, filename) => {
+                if (filename && filename !== base) return;
                 const res = loadSpec(this.opts.specPath);
                 if (!res.ok) { this.emit({ kind: 'spec_error', errors: res.errors }); return; }
                 this.spec = res.spec;
@@ -411,12 +446,58 @@ export class SpecDriver {
         if (evState.id === 'busy') {
             this.lastBusyAt = Date.now();
             this.lastBusyState = evState;
+            // Cancel any pending idle commit — non-idle reading invalidates it.
+            this.cancelIdleHold();
             // Schedule a re-evaluation when the hold window expires. PTYs
             // typically stop emitting once the agent stops printing (the
             // footer settles), so without an explicit timer the driver
             // never wakes up to downshift to idle and the dashboard sees
             // status stuck at generating long after the turn ended.
             this.scheduleBusyExpiry(busyWakeMs);
+        } else if (evState.id !== this.currentStateId && evState.id !== 'busy') {
+            // Non-busy modal states (approval, picker, signing_in) also cancel
+            // any in-flight idle hold — they are higher-priority than idle.
+            if (evState.id !== (this.spec.default_state ?? 'idle')) {
+                this.cancelIdleHold();
+            }
+        }
+
+        // Idle hold: if idle_hold_ms is set, don't commit idle immediately.
+        // Arm a timer; if a non-idle reading arrives before it fires, cancel.
+        const idleHoldMs = this.spec.debounce?.idle_hold_ms ?? 0;
+        const isIdleState = evState.id === (this.spec.default_state ?? 'idle');
+        if (isIdleState && idleHoldMs > 0 && this.currentStateId !== evState.id) {
+            if (!this.idleHoldTimer) {
+                this.pendingIdleState = evState;
+                this.idleHoldTimer = setTimeout(() => {
+                    this.idleHoldTimer = null;
+                    const committed = this.pendingIdleState;
+                    this.pendingIdleState = null;
+                    if (!committed) return;
+                    LOG.debug('SpecDriver', `[${this.opts.specPath.split('/').slice(-3).join('/')}] idleHold committed after ${idleHoldMs}ms`);
+                    this.currentStateId = committed.id;
+                    this.currentEval = ev;
+                    this.pushHistory(committed.id, committed.label);
+                    this.emit({
+                        kind: 'state_changed',
+                        state: committed,
+                        modal: null,
+                        controls: ev.controls.map(c => ({ id: c.id, label: c.label, action_type: c.actionType })),
+                    });
+                    this.armOrCancelDelegateTimers(committed.id);
+                    if (this.opts.emitTrace) this.emit({ kind: 'spec_trace', entries: ev.trace });
+                }, idleHoldMs);
+            }
+            // Don't fall through to the normal changed/emit path for idle.
+            this.currentEval = ev;
+            const graceMs2 = this.spec.debounce?.startup_grace_ms ?? STARTUP_GRACE_MS;
+            if (!this.idleSeenOnce && Date.now() - this.startedAtMs >= graceMs2) {
+                this.idleSeenOnce = true;
+                const queued = this.pendingSends.splice(0);
+                for (const text of queued) setTimeout(() => this.actuallySendMessage(text), 50);
+            }
+            if (this.pickerInProgress) this.tryAdvancePicker(screen);
+            return;
         }
 
         const changed = forceEmit
@@ -449,6 +530,7 @@ export class SpecDriver {
         }
         if (changed) {
             this.currentStateId = evState.id;
+            this.pushHistory(evState.id, evState.label);
             this.emit({
                 kind: 'state_changed',
                 state: evState,

package/src/providers/spec/types.ts

@@ -268,6 +268,12 @@ export interface CliSpec {
          *  Absorbs per-frame flicker in TUIs that stream output through
          *  the same region as the spinner. */
         busy_hold_ms?: number;
+        /** Min time the idle state must remain matched before it is
+         *  committed. Filters transient idle flickers that appear during
+         *  approval dismissals, layout reflows, or brief spinner gaps.
+         *  Any non-idle reading within the window cancels the transition.
+         *  When omitted the idle transition is immediate (legacy behaviour). */
+        idle_hold_ms?: number;
         /** Min time after start() before a send_message is allowed to
          *  reach the PTY. Banner paints + auth flows + skill listings
          *  can keep the agent unable to accept input for several seconds