@adhdev/daemon-core 1.0.18-rc.2 → 1.0.18-rc.20

package/src/mesh/mesh-reconcile-loop.ts

@@ -69,13 +69,16 @@ import {
     resolveCoordinatorDaemonIds,
     daemonHostsMesh,
     resolveCoordinatorSelfIds,
+    daemonIdListIncludes,
 } from './mesh-reconcile-identity.js';
 import {
     resolveAutoPruneMinAgeMs,
     resolvePendingHeldDrainEscalateMs,
     resolveReconcileIntervalMs,
 } from './mesh-reconcile-config.js';
-import { pullRemoteNodeQueues } from './mesh-remote-event-pull.js';
+import { pullRemoteNodeQueues, pullPendingEventsFromNode, reprobeWorkerStatus } from './mesh-remote-event-pull.js';
+import { isPurePtyTranscriptProvider } from '../cli-adapters/provider-cli-shared.js';
+import { runDiskRetentionSweep, detectAndSignalOrphanWorktrees } from './mesh-disk-retention.js';
 import {
     reconcileUnterminatedDirectDispatches,
     autoPruneStaleDirectDispatches,
@@ -126,6 +129,16 @@ interface LiveCoordinator {
 // "restart does not clear it" symptom the operator needs visibility into).
 const coordinatorModalParkState = new Map<string, boolean>();
 
+// Disk/worktree retention throttle. The reconcile tick runs every ~4s, but the
+// retention sweep (fs walks + git worktree list) is far too heavy for that cadence
+// and its artifacts age in days, so it runs at most once per hour. `undefined`
+// means "never run yet" → runs on the first tick after daemon start so a long-lived
+// backlog is reclaimed promptly rather than an hour later. Per-process; a restart
+// re-runs it immediately, which is the desired behavior (a restart is exactly when
+// stale artifacts from the previous run should be swept).
+const DISK_RETENTION_INTERVAL_MS = 60 * 60 * 1000; // 1h
+let lastDiskRetentionRunAt: number | undefined;
+
 // Find live CLI coordinator instances on THIS daemon, keyed by mesh.
 function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
     const out: LiveCoordinator[] = [];
@@ -721,10 +734,32 @@ const deliveredNoTurnUnknownStreak = new Map<string, number>();
 // immediately; UNKNOWN accrues a streak and re-drives only after RECLAIM_UNKNOWN_GRACE_TICKS.
 const deliveredUnconsumedUnknownStreak = new Map<string, number>();
 
+// KIMI-PURE-PTY-COMPLETION-EMIT (Fix 2): how long a delivered assigned row whose worker is
+// CONTINUOUSLY idle-WITH-a-final-assistant-message must stay so before we short-circuit it to
+// 'completed' from transcript evidence — WITHOUT waiting the full DELIVERED_NO_TURN_DEADLINE_MS
+// (15min). A pure-PTY worker (kimi and kin — no native transcript, no provider authority) whose
+// generating_completed never emitted (the onTurnStarted idle→idle collapse Fix 1 addresses at the
+// source) leaves the row 'assigned' with an idle worker that already rendered its answer. The
+// F3 long-deadline reclaim's pollAssignedTaskTerminalEvidence would eventually recover it — but only
+// after 15min, during which the coordinator ledger wrongly shows the task in flight. This EARLY
+// reconcile applies the SAME idle-with-final-assistant-after-dispatch evidence bar (via
+// pollAssignedTaskTerminalEvidence) after a few continuous-idle ticks, so a finished worker's task
+// is marked complete promptly. Conservative by construction: the streak resets the instant a read is
+// non-idle / lacks a final summary / is unreadable, and the poll itself enforces the after-dispatch
+// stale-summary guard, so a mid-turn or warming-up worker is never falsely completed.
+const ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS = 8_000;
+
+// Per-row continuous idle-with-final-assistant streak for the early transcript-evidence completion,
+// keyed `${meshId}::${taskId}`, storing the wall-clock ms when the continuous run began. Pruned each
+// pass to the currently-assigned rows (same as the UNKNOWN streaks). A non-idle / no-summary /
+// unreadable tick clears the entry so the grace must re-accumulate from scratch.
+const assignedIdleFinalAssistantSince = new Map<string, number>();
+
 // Test hook: clear the delivered-no-turn UNKNOWN streaks between cases.
 export function __resetReclaimUnknownStreakForTests(): void {
     deliveredNoTurnUnknownStreak.clear();
     deliveredUnconsumedUnknownStreak.clear();
+    assignedIdleFinalAssistantSince.clear();
 }
 
 // APPROVAL-INBOX-BLINDSPOT (Fix A.3): true when the assigned row's bound session is, per the
@@ -743,12 +778,129 @@ function assignedRowLiveStatusIsAwaitingApproval(
 ): boolean {
     if (!nodeId || !sessionId) return false;
     try {
-        return sessionStatusFromNodes(mesh.nodes, nodeId, sessionId).status === 'awaiting_approval';
+        // Both blocked-on-input states hold the row: an approval (waiting_approval) and a
+        // question (awaiting_choice) legitimately park the worker awaiting the coordinator's
+        // mesh_approve / mesh_answer_question, so neither should accrue the reclaim streak
+        // (mission f1d25e11 extends the APPROVAL-INBOX-BLINDSPOT guard to questions).
+        const liveStatus = sessionStatusFromNodes(mesh.nodes, nodeId, sessionId).status;
+        return liveStatus === 'awaiting_approval' || liveStatus === 'awaiting_choice';
     } catch {
         return false;
     }
 }
 
+// EARLY-IDLE-COMPLETION-FALSE-POSITIVE — resolve whether the assigned session's provider is
+// the pure-PTY transcript class (kimi and kin — no native history, no provider authority), from
+// the LOCAL instance when it is present. Returns:
+//   true      → local instance is a pure-PTY provider (the class the early rescue exists for)
+//   false     → local instance is NOT pure-PTY (native / provider-authoritative)
+//   undefined → no local instance (remote worker / gone / id-form skew) — pure-PTY unknowable here
+// A remote pure-PTY worker is handled by the reprobe path (a fresh-idle read positively confirms
+// the session is settled before the streak accrues), not by this local-only capability check.
+function resolveLocalSessionPurePty(components: DaemonComponents, sessionId: string): boolean | undefined {
+    try {
+        const instances = components.instanceManager?.getByCategory?.('cli') || [];
+        const inst = instances.find((i: any) => {
+            const sid = readNonEmptyString(i?.getState?.().instanceId);
+            return sid && sessionIdsEquivalent(sid, sessionId);
+        }) as { provider?: unknown } | undefined;
+        if (!inst) return undefined; // remote / gone / id-form skew — not locally observable
+        const provider = inst.provider;
+        if (!provider || typeof provider !== 'object') return undefined;
+        return isPurePtyTranscriptProvider(provider as Parameters<typeof isPurePtyTranscriptProvider>[0]);
+    } catch {
+        return undefined;
+    }
+}
+
+// EARLY-IDLE-COMPLETION-FALSE-POSITIVE — decide whether the early transcript-evidence completion
+// streak may ACCRUE this tick for an assigned row. This hardens the original arm gate, which only
+// checked `resolveSessionBusyVerdict(...) !== 'GENERATING'` — a gate a REMOTE worker (local verdict
+// UNKNOWN, not GENERATING) passed unconditionally, so its "8s continuous idle" streak degenerated to
+// 8s of wall-clock even while the worker was genuinely mid-turn, and a startup-grace boot-window idle
+// (before the first turn ever started) also passed. The row was then early-completed off a preamble.
+//
+// Two added requirements, defense-in-depth on top of the existing confirmed-delivery / no-terminal-
+// ledger / bound-session gate the caller applies:
+//   (a) POSITIVE idle evidence, not merely "not GENERATING". A LOCAL IDLE_CONFIRMED verdict qualifies
+//       directly. A remote/UNKNOWN verdict is RE-PROBED with a fresh read_chat status this tick: only
+//       a definitively 'idle' read lets the streak accrue; any active status (generating/waiting/…),
+//       or an inconclusive read, resets it. This is what stops the remote worker's mid-turn busy from
+//       silently passing as "continuous idle".
+//   (b) TURN-START evidence, so a boot-window / never-consumed idle cannot early-complete a preamble.
+//       taskDeliveryConsumed===true (the worker emitted agent:generating_started) satisfies it. When
+//       the delivery was never consumed, only a LOCAL pure-PTY provider — the exact class that never
+//       emits generating_started and that this rescue exists for — is still allowed; a native/
+//       provider-authoritative worker that hasn't started its turn is NOT armed (it will complete via
+//       its own emit or the normal grace). A remote-not-consumed worker whose provider class is
+//       unknowable locally falls back to the reprobe: it may accrue only once (a) reads it positively
+//       idle, and the poll's post-dispatch + trailing-tool-activity guards remain the final net.
+async function evaluateEarlyIdleTranscriptArm(
+    components: DaemonComponents,
+    mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
+    store: MeshRuntimeStore,
+    row: { id: string; assignedSessionId?: string; assignedNodeId?: string; assignedProviderType?: string },
+    localDaemonId?: string,
+    selfIds: string[] = [],
+): Promise<boolean> {
+    const sessionId = readNonEmptyString(row.assignedSessionId);
+    if (!sessionId) return false;
+
+    // (a) POSITIVE idle evidence.
+    const verdict = resolveSessionBusyVerdict(components, sessionId);
+    if (verdict === 'GENERATING') return false; // locally observed mid-turn — never accrue
+    if (verdict === 'UNKNOWN') {
+        // Remote / gone / id-form skew. First consult the LIVE mesh node snapshot (the same
+        // cross-daemon status feed the approval-hold guard and mesh_status use): if it already
+        // reports a definitively NON-idle status (generating / waiting_approval / awaiting_choice /
+        // …), the worker is not settled — reset the streak WITHOUT a read_chat. This keeps the
+        // streak the cheap per-tick gate for the common case and avoids probing a worker the graph
+        // already knows is busy or parked at an approval.
+        const nodeId = readNonEmptyString(row.assignedNodeId);
+        const node = (mesh.nodes ?? []).find(n => n.id === nodeId);
+        const liveStatus = nodeId
+            ? readNonEmptyString(sessionStatusFromNodes(mesh.nodes, nodeId, sessionId).status).toLowerCase()
+            : '';
+        if (liveStatus && liveStatus !== 'idle') return false; // live graph says non-idle → not a turn-end
+        if (!liveStatus) {
+            // No live snapshot for this session — re-probe the worker's own status this tick so a
+            // genuinely busy remote worker breaks the streak instead of coasting through as "not
+            // GENERATING". getInstance may be absent on some component surfaces (tests / standalone),
+            // so guard it.
+            const nodeDaemonId = readNonEmptyString(node?.daemonId);
+            const isLocalNode = !nodeDaemonId
+                || daemonIdsEquivalent(nodeDaemonId, localDaemonId)
+                || daemonIdListIncludes(selfIds, nodeDaemonId)
+                || !!components.instanceManager?.getInstance?.(sessionId);
+            const providerType = readNonEmptyString(row.assignedProviderType);
+            const readArgs: Record<string, unknown> = {
+                sessionId,
+                targetSessionId: sessionId,
+                tailLimit: 1,
+                ...(node?.workspace ? { workspace: node.workspace } : {}),
+                ...(providerType ? { agentType: providerType, providerType } : {}),
+            };
+            const probedStatus = await reprobeWorkerStatus(components, { isLocalNode, nodeDaemonId, readArgs });
+            // Only a positively-idle re-probe lets the streak accrue. A non-idle status (the worker
+            // IS busy) or an inconclusive read (null — transport error / no payload) resets it: we
+            // never treat "couldn't tell" as idle for this early-completion path.
+            if (probedStatus !== 'idle') return false;
+        }
+    }
+
+    // (b) TURN-START evidence (guards the boot-window / preamble false positive).
+    if (store.taskDeliveryConsumed(mesh.id, row.id)) return true; // worker emitted generating_started
+    // Delivery never consumed. Preserve the kimi rescue ONLY for the LOCAL pure-PTY class (which
+    // never emits generating_started); a native/provider-authoritative local worker that hasn't
+    // started its turn must NOT be armed off a boot-window idle. For a remote worker (pure-PTY
+    // unknowable here) the positive-idle reprobe in (a) is the gate — allow accrual and let the poll
+    // enforce post-dispatch + trailing-tool-activity finality.
+    const localPurePty = resolveLocalSessionPurePty(components, sessionId);
+    if (localPurePty === true) return true;   // local pure-PTY — the class this rescue targets
+    if (localPurePty === false) return false; // local native/provider-owned, turn not started — hold
+    return verdict === 'UNKNOWN';             // remote, unknowable class — trust the (a) reprobe
+}
+
 // PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
 // flips a row to 'assigned' BEFORE the fire-and-forget dispatch runs. If that dispatch
 // neither rejects (→ no .catch requeue) nor is confirmed delivered — a relay that hangs
@@ -765,6 +917,8 @@ async function recoverStrandedAssignedDispatches(
     components: DaemonComponents,
     mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
     store: MeshRuntimeStore,
+    localDaemonId?: string,
+    selfIds: string[] = [],
 ): Promise<void> {
     const meshId = mesh.id;
     const assigned = getQueue(meshId, { status: ['assigned'] });
@@ -781,10 +935,82 @@ async function recoverStrandedAssignedDispatches(
     for (const key of [...deliveredUnconsumedUnknownStreak.keys()]) {
         if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) deliveredUnconsumedUnknownStreak.delete(key);
     }
+    for (const key of [...assignedIdleFinalAssistantSince.keys()]) {
+        if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) assignedIdleFinalAssistantSince.delete(key);
+    }
     for (const row of assigned) {
         const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
         if (!Number.isFinite(dispatchedAtMs)) continue;              // no dispatch ts → can't age it
         const ageMs = nowMs - dispatchedAtMs;
+        const idleTranscriptStreakKey = `${meshId}::${row.id}`;
+        // KIMI-PURE-PTY-COMPLETION-EMIT (Fix 2): EARLY transcript-evidence completion for a
+        // delivered assigned row whose worker is already idle with a final assistant answer, so a
+        // pure-PTY worker whose generating_completed never emitted is marked done PROMPTLY instead of
+        // sitting 'assigned' until the 15-min DELIVERED_NO_TURN_DEADLINE_MS reclaim. Gate on a
+        // CONFIRMED delivery (a never-delivered row is handled by dispatch/redrive, not completion)
+        // and a bound session, then require the worker to read idle-WITH-final-assistant continuously
+        // for ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS before polling the decisive transcript evidence.
+        // The streak is the cheap gate (avoids a read every tick); the read only runs once the grace
+        // elapses. pollAssignedTaskTerminalEvidence enforces idle + final-assistant-after-dispatch, so
+        // a mid-turn / stale-tail / unreadable worker is never falsely completed — a non-qualifying
+        // tick clears the streak below.
+        // EARLY-IDLE-COMPLETION-FALSE-POSITIVE: the arm gate now requires POSITIVE idle evidence
+        // (a fresh-idle reprobe for a remote/UNKNOWN worker, not merely "not GENERATING") AND
+        // turn-start evidence (delivery consumed, or the local pure-PTY class this rescue targets),
+        // so a mid-turn remote worker or a boot-window preamble can no longer coast the continuous-
+        // idle streak to a false completion. See evaluateEarlyIdleTranscriptArm.
+        const earlyArm =
+            store.taskHasConfirmedDelivery(meshId, row.id)
+            && !findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })
+            && readNonEmptyString(row.assignedSessionId)
+                ? await evaluateEarlyIdleTranscriptArm(components, mesh, store, row, localDaemonId, selfIds)
+                : false;
+        if (earlyArm) {
+            const since = assignedIdleFinalAssistantSince.get(idleTranscriptStreakKey);
+            if (since === undefined) {
+                assignedIdleFinalAssistantSince.set(idleTranscriptStreakKey, nowMs);
+            } else if (nowMs - since >= ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS) {
+                const terminalEvidence = await pollAssignedTaskTerminalEvidence(components, mesh, row);
+                if (terminalEvidence) {
+                    assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
+                    updateTaskStatus(meshId, row.id, terminalEvidence);
+                    if (!findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })) {
+                        try {
+                            appendLedgerEntry(meshId, {
+                                kind: terminalEvidence === 'completed' ? 'task_completed' : 'task_failed',
+                                nodeId: row.assignedNodeId,
+                                sessionId: row.assignedSessionId,
+                                providerType: row.assignedProviderType,
+                                payload: {
+                                    taskId: row.id,
+                                    event: 'agent:generating_completed',
+                                    source: 'early_idle_transcript_evidence',
+                                },
+                            });
+                        } catch { /* best-effort ledger write */ }
+                    }
+                    LOG.warn('MeshReconcile', `Early-completed assigned task ${row.id} on mesh ${meshId} `
+                        + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}): worker idle with a `
+                        + `final assistant message after dispatch for ≥${Math.round(ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS / 1000)}s — `
+                        + `the completion event was lost/late (pure-PTY provider), task is ${terminalEvidence} without waiting the 15-min deadline`);
+                    traceMeshEventDrop('assigned_early_transcript_completed', {
+                        taskId: row.id,
+                        sessionId: row.assignedSessionId,
+                        nodeId: row.assignedNodeId,
+                        meshId,
+                        event: 'agent:generating_completed',
+                    }, terminalEvidence);
+                    continue;
+                }
+                // Poll was inconclusive (mid-turn re-check / stale tail / unreadable) — reset the
+                // streak so it must re-accumulate a fresh continuous-idle run before the next poll.
+                assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
+            }
+        } else {
+            // Not a qualifying tick (no confirmed delivery, generating, terminal ledger present, or
+            // no bound session) — break the continuous-idle run.
+            assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
+        }
         // DELIVERED-NOT-CONSUMED short-grace re-drive (remote autoLaunch delivered≠consumed gap).
         // Runs BEFORE the ASSIGNED_STRANDED_DEADLINE_MS confirm-window gate below because its whole
         // point is to recover a delivered-but-unconsumed row well inside that window. A remote
@@ -816,6 +1042,45 @@ async function recoverStrandedAssignedDispatches(
                 updateTaskStatus(meshId, row.id, status);
                 continue;
             }
+            // GENERATING-STARTED-CONSUME-RACE (fix B): the delivery-consume that clears this
+            // gate (delivered → acked) runs ONLY when the coordinator pulls the worker's queued
+            // agent:generating_started in PHASE 1 (handleMeshForwardEvent → consumeSessionDelivery).
+            // A worker may have emitted generating_started ALREADY, but if PHASE 1 has not yet
+            // pulled that specific event (a skipped/slow peer tick, or the event was queued after
+            // this tick's pull ran), the delivery row still reads 'delivered' here and the redrive
+            // below tears a genuinely-generating remote worker off its task and re-injects the
+            // prompt (the delivered_not_consumed_redrive symptom). Close the race by issuing a
+            // TARGETED, in-process last-chance pull of THIS node's queue right now, then
+            // re-reading taskDeliveryConsumed(): if the worker's generating_started was waiting to
+            // be pulled, the pull consumes the delivery this tick and we skip the redrive entirely.
+            // Best-effort and self-scoped (the same skip/peer guards as PHASE 1) — a miss just
+            // falls through to the existing UNKNOWN-streak grace, so this only ever removes false
+            // redrives, never adds one. A local (co-hosted) worker has nothing to pull (the node's
+            // daemon is a self id) and this is a fast no-op.
+            const assignedNode = row.assignedNodeId
+                ? mesh.nodes?.find(n => meshNodeIdMatches(n, row.assignedNodeId))
+                : undefined;
+            if (assignedNode?.daemonId) {
+                try {
+                    await pullPendingEventsFromNode(
+                        components,
+                        meshId,
+                        assignedNode,
+                        localDaemonId,
+                        selfIds,
+                        selfIds.length > 0
+                            ? selfIds.map(id => ({ meshId, coordinatorDaemonId: id }))
+                            : [{ meshId }],
+                    );
+                } catch { /* best-effort last-chance pull */ }
+                if (store.taskDeliveryConsumed(meshId, row.id)) {
+                    // The pull delivered the worker's generating_started (or a terminal event) and
+                    // consumed the delivery — the task is genuinely live/handled. Reset any accrued
+                    // streak and leave the row to PHASE 4's completion reconcile.
+                    deliveredUnconsumedUnknownStreak.delete(`${meshId}::${row.id}`);
+                    continue;
+                }
+            }
             const shortStreakKey = `${meshId}::${row.id}`;
             const verdict = row.assignedSessionId
                 ? resolveSessionBusyVerdict(components, row.assignedSessionId)
@@ -1211,7 +1476,7 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
             const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
             if (!daemonHostsMesh(mesh, selfIds)) continue;
             try {
-                await recoverStrandedAssignedDispatches(components, mesh, store);
+                await recoverStrandedAssignedDispatches(components, mesh, store, localDaemonId, selfIds);
             } catch (e: any) {
                 LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
             }
@@ -1354,6 +1619,44 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
         }
     }
 
+    // ── PHASE 6: disk / worktree retention (hourly, mission 86def38d) ──────────
+    // Legacy on-disk artifacts under ~/.adhdev accumulated with NO lifetime and grew
+    // the data volume until a refine bootstrap failed at 98% disk. This throttled pass
+    // (≤1×/hour — the artifacts age in days and the fs/git walk is too heavy for the 4s
+    // tick) reclaims them:
+    //   • JSONL ledger files older than 30d (legacy after the SQLite ledger)
+    //   • terminated session-host runtime files older than 14d (live never touched)
+    //   • mesh-runtime.db.bak-* backups older than 7d
+    //   • orphan worktree DETECTION → cleanup_candidate ledger signal (NEVER deleted).
+    // Runs BEFORE PHASE 2's early return so it fires whether or not a live CLI
+    // coordinator exists on this daemon. Isolated in try/catch so it can never kill the
+    // tick. All file deletions are defensive (age + live-runtime guards in the pure
+    // selectors); worktree cleanup stays manual/coordinator-driven.
+    {
+        const nowMs = Date.now();
+        const due = lastDiskRetentionRunAt === undefined
+            || (nowMs - lastDiskRetentionRunAt) >= DISK_RETENTION_INTERVAL_MS;
+        if (due) {
+            lastDiskRetentionRunAt = nowMs;
+            try {
+                runDiskRetentionSweep(nowMs);
+            } catch (e: any) {
+                LOG.warn('MeshReconcile', `Disk retention sweep failed: ${e?.message || e}`);
+            }
+            // Orphan-worktree detection is per-mesh (needs the mesh's base repo + node set)
+            // and only for meshes this daemon hosts (its local base checkout is the git anchor).
+            for (const mesh of listMeshes()) {
+                const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
+                if (!daemonHostsMesh(mesh, selfIds)) continue;
+                try {
+                    await detectAndSignalOrphanWorktrees(mesh, nowMs);
+                } catch (e: any) {
+                    LOG.warn('MeshReconcile', `Orphan worktree detection failed for mesh ${mesh.id}: ${e?.message || e}`);
+                }
+            }
+        }
+    }
+
     // ── PHASE 2: inject into live CLI coordinators on this daemon ──────────────
     const coordinators = findLiveCoordinators(components);
     if (coordinators.length === 0) {