@adhdev/daemon-core 0.9.82-rc.533 → 0.9.82-rc.534

package/dist/mesh/mesh-delivery-policy.d.ts

@@ -96,6 +96,13 @@ export declare function getActiveSessionDeliveries(meshId: string, sessionId?: s
     createdAt: string;
     updatedAt: string;
 }[];
+/**
+ * DELIVERED-NOT-CONSUMED-REDRIVE consume path. Advance a task's delivery record(s) to a
+ * CONSUMED status ('acked'/'completed') by (mesh, session[, task]), INCLUDING rows already in
+ * 'delivered' — unlike getActiveSessionDeliveries which excludes 'delivered'. The store's
+ * monotonic guard only ever advances the row. Returns the number of rows advanced.
+ */
+export declare function consumeSessionDelivery(meshId: string, sessionId: string, status: 'acked' | 'completed', taskId?: string): number;
 export declare function __clearSessionDeliveriesForTests(meshId: string): void;
 /**
  * Mark all active (queued/delivering/delivered/acked) deliveries for a session as completed or failed.

package/dist/mesh/mesh-runtime-store.d.ts

@@ -207,10 +207,35 @@ export declare class MeshRuntimeStore {
         createdAt: string;
         updatedAt: string;
     }): void;
+    private static readonly DELIVERY_PROGRESS_RANK;
     updateSessionDeliveryStatus(id: string, status: string, opts?: {
         lastError?: string;
         incrementAttempt?: boolean;
     }): void;
+    /**
+     * DELIVERED-NOT-CONSUMED-REDRIVE consume path. Advance a task's delivery record(s) to a
+     * CONSUMED status ('acked' or 'completed'), matching on mesh + session (+ taskId when the
+     * event names one) and INCLUDING rows already in 'delivered'/'acked'/'delivering'.
+     *
+     * The ack/terminal callers previously routed through getActiveSessionDeliveries(), whose SQL
+     * EXCLUDES 'delivered' — so in the normal event order (transport confirm flips 'delivered'
+     * BEFORE the worker's generating_started fires) the ack matched zero rows and the delivery
+     * was stranded 'delivered', never 'acked'. This finds the row by (mesh, session[, task])
+     * directly and relies on updateSessionDeliveryStatus's monotonic guard to only advance it.
+     * Returns the number of rows advanced.
+     */
+    consumeSessionDelivery(meshId: string, sessionId: string, status: 'acked' | 'completed', taskId?: string): number;
+    /**
+     * DELIVERED-NOT-CONSUMED-REDRIVE terminal path. Mark every OPEN delivery for a session
+     * (queued/delivering/delivered/acked) terminal on task completion/failure. The prior
+     * markSessionDeliveriesTerminal() routed through getActiveSessionDeliveries(), whose SQL
+     * EXCLUDES 'delivered'/'completed' — so a 'delivered' row (the common case, since the
+     * transport confirm flips it before the completion event) was never marked terminal and
+     * stayed 'delivered', keeping taskDeliveryConsumed() false and feeding the false re-drive.
+     * We match rows in OPEN states directly here. 'completed' advances monotonically (it is the
+     * top progress rank); 'failed' is an absorbing outcome written unconditionally.
+     */
+    markOpenSessionDeliveriesTerminal(meshId: string, sessionId: string, terminalStatus: 'completed' | 'failed'): number;
     getActiveSessionDeliveries(meshId: string, sessionId?: string): Array<{
         id: string;
         meshId: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adhdev/daemon-core",
-  "version": "0.9.82-rc.533",
+  "version": "0.9.82-rc.534",
   "description": "ADHDev daemon core — CDP, IDE detection, providers, command execution",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -47,8 +47,8 @@
   "author": "vilmire",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@adhdev/mesh-shared": "0.9.82-rc.533",
-    "@adhdev/session-host-core": "0.9.82-rc.533",
+    "@adhdev/mesh-shared": "0.9.82-rc.534",
+    "@adhdev/session-host-core": "0.9.82-rc.534",
     "@agentclientprotocol/sdk": "^0.16.1",
     "ajv": "^8.20.0",
     "ajv-formats": "^3.0.1",

package/src/mesh/mesh-delivery-policy.ts

@@ -254,6 +254,25 @@ export function getActiveSessionDeliveries(meshId: string, sessionId?: string) {
     }
 }
 
+/**
+ * DELIVERED-NOT-CONSUMED-REDRIVE consume path. Advance a task's delivery record(s) to a
+ * CONSUMED status ('acked'/'completed') by (mesh, session[, task]), INCLUDING rows already in
+ * 'delivered' — unlike getActiveSessionDeliveries which excludes 'delivered'. The store's
+ * monotonic guard only ever advances the row. Returns the number of rows advanced.
+ */
+export function consumeSessionDelivery(
+    meshId: string,
+    sessionId: string,
+    status: 'acked' | 'completed',
+    taskId?: string,
+): number {
+    try {
+        return MeshRuntimeStore.getInstance().consumeSessionDelivery(meshId, sessionId, status, taskId);
+    } catch {
+        return 0;
+    }
+}
+
 // MESH-COMPLEXITY-AUDIT Part 8-2: the completion-conflict diagnostic
 // (recordCompletionConflict / getRecentCompletionConflicts, backed by
 // mesh_completion_conflicts) was dropped. It recorded WHICH task lost a
@@ -276,9 +295,10 @@ export function markSessionDeliveriesTerminal(
     terminalStatus: 'completed' | 'failed',
 ): void {
     try {
-        const active = MeshRuntimeStore.getInstance().getActiveSessionDeliveries(meshId, sessionId);
-        for (const delivery of active) {
-            MeshRuntimeStore.getInstance().updateSessionDeliveryStatus(delivery.id, terminalStatus);
-        }
+        // Route through markOpenSessionDeliveriesTerminal, which matches OPEN rows including
+        // 'delivered' — getActiveSessionDeliveries EXCLUDES 'delivered' and would silently
+        // leave the common (already-delivered) row un-terminated, keeping taskDeliveryConsumed()
+        // false and feeding the delivered_not_consumed_redrive false re-drive.
+        MeshRuntimeStore.getInstance().markOpenSessionDeliveriesTerminal(meshId, sessionId, terminalStatus);
     } catch { /* best-effort */ }
 }

package/src/mesh/mesh-event-forwarding.ts

@@ -5,7 +5,7 @@ import { LOG } from '../logging/logger.js';
 import { appendLedgerEntry, buildTaskCompletionEvidence, getSessionRecoveryContext, isIntentionalCleanupStopEntry, readLedgerEntries } from './mesh-ledger.js';
 import type { SessionRecoveryContext } from './mesh-ledger.js';
 import { updateSessionTaskStatus, enqueueTask, updateDirectDispatchStatus, cleanupTerminalDirectDispatches, getActiveDirectDispatches, hasPendingDependents, getQueue } from './mesh-work-queue.js';
-import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus } from './mesh-delivery-policy.js';
+import { markSessionDeliveriesTerminal, updateSessionDeliveryStatus, consumeSessionDelivery } from './mesh-delivery-policy.js';
 import { MeshRuntimeStore, pruneMeshRuntimeRetention } from './mesh-runtime-store.js';
 import { maybeInjectIdleActiveMissionReminder } from './mesh-idle-reminder.js';
 import { queuePendingMeshCoordinatorEvent, drainPendingMeshCoordinatorEvents, prunePendingMeshCoordinatorEventsRetention, readV2EnvelopeFromWire, type PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
@@ -1273,16 +1273,14 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
                     updateDirectDispatchStatus(args.meshId, sessionId, 'acked', soleTaskId);
                 }
             }
-            const activeDeliveries = ((): { id: string; taskId: string | null }[] => {
-                try { return MeshRuntimeStore.getInstance().getActiveSessionDeliveries(args.meshId, sessionId); }
-                catch { return []; }
-            })();
-            const deliveriesToAck = startedTaskId
-                ? activeDeliveries.filter(d => d.taskId === startedTaskId)
-                : activeDeliveries;
-            for (const d of deliveriesToAck) {
-                updateSessionDeliveryStatus(d.id, 'acked');
-            }
+            // DELIVERED-NOT-CONSUMED-REDRIVE: ack the delivery via consumeSessionDelivery, which
+            // matches rows INCLUDING 'delivered'. The prior path filtered getActiveSessionDeliveries
+            // — whose SQL EXCLUDES 'delivered' — so in the normal event order (transport confirm
+            // flips 'delivered' BEFORE generating_started fires) the ack matched zero rows and the
+            // delivery was stranded 'delivered', never reaching the 'acked' consume signal that
+            // taskDeliveryConsumed() keys on. The store's monotonic guard only advances the row.
+            // With a named taskId, key on (mesh, task); otherwise fall back to the whole session.
+            consumeSessionDelivery(args.meshId, sessionId, 'acked', startedTaskId);
         }
     } else if (args.event === 'agent:stopped') {
         const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);

package/src/mesh/mesh-reconcile-loop.ts

@@ -710,9 +710,19 @@ const RECLAIM_UNKNOWN_GRACE_TICKS = 3;
 // completed/reclaimed/claimed-elsewhere row's counter is dropped (no unbounded growth).
 const deliveredNoTurnUnknownStreak = new Map<string, number>();
 
-// Test hook: clear the delivered-no-turn UNKNOWN streak between cases.
+// DELIVERED-NOT-CONSUMED-REDRIVE (fix d): the SHORT-grace re-drive (delivered-but-unconsumed,
+// 25s window) previously reclaimed on a SINGLE non-GENERATING tick — and a REMOTE worker's local
+// busy verdict is UNKNOWN, not GENERATING, so a genuinely-mid-turn remote worker whose ack merely
+// hadn't propagated yet was torn off its task and the SAME prompt re-injected. Give the short path
+// the same bounded consecutive-UNKNOWN grace the long delivered-no-turn path uses: only an
+// IDLE_CONFIRMED verdict (positive LOCAL evidence the session is present-and-idle) re-drives
+// immediately; UNKNOWN accrues a streak and re-drives only after RECLAIM_UNKNOWN_GRACE_TICKS.
+const deliveredUnconsumedUnknownStreak = new Map<string, number>();
+
+// Test hook: clear the delivered-no-turn UNKNOWN streaks between cases.
 export function __resetReclaimUnknownStreakForTests(): void {
     deliveredNoTurnUnknownStreak.clear();
+    deliveredUnconsumedUnknownStreak.clear();
 }
 
 // PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
@@ -739,6 +749,9 @@ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId:
     for (const key of [...deliveredNoTurnUnknownStreak.keys()]) {
         if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) deliveredNoTurnUnknownStreak.delete(key);
     }
+    for (const key of [...deliveredUnconsumedUnknownStreak.keys()]) {
+        if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) deliveredUnconsumedUnknownStreak.delete(key);
+    }
     for (const row of assigned) {
         const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
         if (!Number.isFinite(dispatchedAtMs)) continue;              // no dispatch ts → can't age it
@@ -774,15 +787,42 @@ function recoverStrandedAssignedDispatches(components: DaemonComponents, meshId:
                 updateTaskStatus(meshId, row.id, status);
                 continue;
             }
+            const shortStreakKey = `${meshId}::${row.id}`;
             const verdict = row.assignedSessionId
                 ? resolveSessionBusyVerdict(components, row.assignedSessionId)
                 : 'IDLE_CONFIRMED'; // no session bound → nothing live generating to protect
-            if (verdict !== 'GENERATING') {
+            // GENERATING → demonstrably alive: never re-drive, reset the grace.
+            // IDLE_CONFIRMED → positive LOCAL evidence the session is present-and-idle: re-drive now.
+            // UNKNOWN → remote / gone / id-form-skewed session: DEFER. A remote worker whose ack
+            //   merely hasn't propagated reads UNKNOWN here — reclaiming on a single UNKNOWN tick
+            //   tears a live remote worker off its task and re-injects the same prompt (the exact
+            //   delivered_not_consumed_redrive symptom). Accrue a bounded consecutive-UNKNOWN streak
+            //   and only re-drive after RECLAIM_UNKNOWN_GRACE_TICKS, matching the long path.
+            if (verdict === 'GENERATING') {
+                deliveredUnconsumedUnknownStreak.delete(shortStreakKey);
+            } else {
+                if (verdict === 'IDLE_CONFIRMED') {
+                    deliveredUnconsumedUnknownStreak.delete(shortStreakKey);
+                } else {
+                    const streak = (deliveredUnconsumedUnknownStreak.get(shortStreakKey) ?? 0) + 1;
+                    deliveredUnconsumedUnknownStreak.set(shortStreakKey, streak);
+                    if (streak < RECLAIM_UNKNOWN_GRACE_TICKS) {
+                        traceMeshEventDrop('short_redrive_deferred_unknown_verdict', {
+                            taskId: row.id,
+                            sessionId: row.assignedSessionId,
+                            nodeId: row.assignedNodeId,
+                            meshId,
+                            event: 'agent:generating_started',
+                        }, `unknown ${streak}/${RECLAIM_UNKNOWN_GRACE_TICKS}`);
+                        continue;
+                    }
+                }
                 const redriven = reclaimStrandedAssignedTask(meshId, row.id, {
                     reason: 'delivered_not_consumed_redrive',
                     ageMs,
                 });
                 if (redriven) {
+                    deliveredUnconsumedUnknownStreak.delete(shortStreakKey);
                     LOG.warn('MeshReconcile', `Re-drove delivered-but-unconsumed task ${row.id} on mesh ${meshId} `
                         + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}, delivered but no `
                         + `generating_started in ${Math.round(ageMs / 1000)}s, verdict ${verdict} → ${redriven.status})`);

package/src/mesh/mesh-runtime-store.ts

@@ -1429,21 +1429,123 @@ export class MeshRuntimeStore {
         this.maybeCheckpointWal();
     }
 
+    // DELIVERED-NOT-CONSUMED-REDRIVE monotonic FSM: the forward-progress lifecycle of a
+    // delivery is a strictly increasing rank — a status may only advance, never regress.
+    // The redrive bug was a NON-monotonic FSM: the transport-confirm callback
+    // (mesh-queue-assignment :384) writes 'delivered' unconditionally by PK, so when the
+    // worker's agent:generating_started raced AHEAD of the confirm and already flipped the
+    // row 'delivering'→'acked', the late confirm CLOBBERED 'acked' back to 'delivered'.
+    // taskDeliveryConsumed() (which keys on 'acked'/'completed') then read false forever,
+    // and the short-grace re-drive re-opened an already-consumed task. Enforcing the rank
+    // ordering here makes the two event orders converge on the same monotone terminal state
+    // regardless of arrival order, so a late confirm can never demote a consumed delivery.
+    // 'failed'/'expired'/'cancelled' are absorbing OUTCOMES, not progress ranks — they are
+    // always allowed (a genuine dispatch failure must be recordable even from 'acked').
+    private static readonly DELIVERY_PROGRESS_RANK: Record<string, number> = {
+        queued: 0,
+        delivering: 1,
+        delivered: 2,
+        acked: 3,
+        completed: 4,
+    };
+
     updateSessionDeliveryStatus(id: string, status: string, opts?: { lastError?: string; incrementAttempt?: boolean }): void {
         const now = new Date().toISOString();
         if (opts?.incrementAttempt) {
+            // Retry/requeue path (transport failure → 'failed', or an explicit re-queue): this is
+            // the deliberate reset signal, NOT the racing progress writes that cause the clobber, so
+            // it is exempt from the monotonic guard and always applies (preserves attempt_count
+            // bookkeeping and the failure ledger). The clobber bug lives only in the plain
+            // progress write below.
             this.db.prepare(`
                 UPDATE mesh_session_delivery
                 SET status = @status, last_error = @lastError, attempt_count = attempt_count + 1, updated_at = @updatedAt
                 WHERE id = @id
             `).run({ id, status, lastError: opts?.lastError ?? null, updatedAt: now });
-        } else {
+            return;
+        }
+        // Monotonic guard for forward-progress statuses: a plain status write may ADVANCE or
+        // rewrite the SAME rank, but NEVER regress to a strictly-lower rank. This is what stops the
+        // late transport-confirm ('delivered', rank 2) from clobbering an already-consumed row
+        // ('acked', rank 3): the `@targetRank >= current` predicate fetches zero rows for 3→2, so
+        // 'acked' survives. Absorbing failure outcomes (failed/expired/cancelled) have no rank and
+        // are written unconditionally.
+        const targetRank = MeshRuntimeStore.DELIVERY_PROGRESS_RANK[status];
+        if (targetRank === undefined) {
             this.db.prepare(`
                 UPDATE mesh_session_delivery
                 SET status = @status, last_error = @lastError, updated_at = @updatedAt
                 WHERE id = @id
             `).run({ id, status, lastError: opts?.lastError ?? null, updatedAt: now });
+            return;
+        }
+        // Absorbing failure states (failed/expired/cancelled) map to rank 99 so no progress write
+        // (max rank 4) can ever resurrect a dead delivery.
+        this.db.prepare(`
+            UPDATE mesh_session_delivery
+            SET status = @status, last_error = @lastError, updated_at = @updatedAt
+            WHERE id = @id AND (@targetRank >= CASE status
+                WHEN 'queued' THEN 0 WHEN 'delivering' THEN 1 WHEN 'delivered' THEN 2
+                WHEN 'acked' THEN 3 WHEN 'completed' THEN 4 ELSE 99 END)
+        `).run({ id, status, lastError: opts?.lastError ?? null, updatedAt: now, targetRank });
+    }
+
+    /**
+     * DELIVERED-NOT-CONSUMED-REDRIVE consume path. Advance a task's delivery record(s) to a
+     * CONSUMED status ('acked' or 'completed'), matching on mesh + session (+ taskId when the
+     * event names one) and INCLUDING rows already in 'delivered'/'acked'/'delivering'.
+     *
+     * The ack/terminal callers previously routed through getActiveSessionDeliveries(), whose SQL
+     * EXCLUDES 'delivered' — so in the normal event order (transport confirm flips 'delivered'
+     * BEFORE the worker's generating_started fires) the ack matched zero rows and the delivery
+     * was stranded 'delivered', never 'acked'. This finds the row by (mesh, session[, task])
+     * directly and relies on updateSessionDeliveryStatus's monotonic guard to only advance it.
+     * Returns the number of rows advanced.
+     */
+    consumeSessionDelivery(meshId: string, sessionId: string, status: 'acked' | 'completed', taskId?: string): number {
+        const rows = this.db.prepare(
+            taskId
+                ? `SELECT id, session_id FROM mesh_session_delivery
+                     WHERE mesh_id = ? AND task_id = ?
+                       AND status IN ('queued','delivering','delivered','acked')`
+                : `SELECT id, session_id FROM mesh_session_delivery
+                     WHERE mesh_id = ? AND session_id = ?
+                       AND status IN ('queued','delivering','delivered','acked')`,
+        ).all(meshId, taskId ?? sessionId) as Array<{ id: string; session_id: string | null }>;
+        // Filter session membership in JS with the trimming equivalence predicate (mirrors
+        // findAssignedBySession): a taskId match must still belong to this session, and the
+        // session-only match already selected by column may carry serialization skew.
+        let advanced = 0;
+        for (const r of rows) {
+            if (!sessionIdsEquivalent(r.session_id ?? undefined, sessionId)) continue;
+            this.updateSessionDeliveryStatus(r.id, status);
+            advanced++;
+        }
+        return advanced;
+    }
+
+    /**
+     * DELIVERED-NOT-CONSUMED-REDRIVE terminal path. Mark every OPEN delivery for a session
+     * (queued/delivering/delivered/acked) terminal on task completion/failure. The prior
+     * markSessionDeliveriesTerminal() routed through getActiveSessionDeliveries(), whose SQL
+     * EXCLUDES 'delivered'/'completed' — so a 'delivered' row (the common case, since the
+     * transport confirm flips it before the completion event) was never marked terminal and
+     * stayed 'delivered', keeping taskDeliveryConsumed() false and feeding the false re-drive.
+     * We match rows in OPEN states directly here. 'completed' advances monotonically (it is the
+     * top progress rank); 'failed' is an absorbing outcome written unconditionally.
+     */
+    markOpenSessionDeliveriesTerminal(meshId: string, sessionId: string, terminalStatus: 'completed' | 'failed'): number {
+        const rows = this.db.prepare(
+            `SELECT id, session_id FROM mesh_session_delivery
+               WHERE mesh_id = ? AND status IN ('queued','delivering','delivered','acked')`,
+        ).all(meshId) as Array<{ id: string; session_id: string | null }>;
+        let marked = 0;
+        for (const r of rows) {
+            if (!sessionIdsEquivalent(r.session_id ?? undefined, sessionId)) continue;
+            this.updateSessionDeliveryStatus(r.id, terminalStatus);
+            marked++;
         }
+        return marked;
     }
 
     getActiveSessionDeliveries(meshId: string, sessionId?: string): Array<{