@adhdev/daemon-core 1.0.18-rc.20 → 1.0.18-rc.4

package/src/mesh/mesh-reconcile-loop.ts

@@ -69,16 +69,13 @@ import {
     resolveCoordinatorDaemonIds,
     daemonHostsMesh,
     resolveCoordinatorSelfIds,
-    daemonIdListIncludes,
 } from './mesh-reconcile-identity.js';
 import {
     resolveAutoPruneMinAgeMs,
     resolvePendingHeldDrainEscalateMs,
     resolveReconcileIntervalMs,
 } from './mesh-reconcile-config.js';
-import { pullRemoteNodeQueues, pullPendingEventsFromNode, reprobeWorkerStatus } from './mesh-remote-event-pull.js';
-import { isPurePtyTranscriptProvider } from '../cli-adapters/provider-cli-shared.js';
-import { runDiskRetentionSweep, detectAndSignalOrphanWorktrees } from './mesh-disk-retention.js';
+import { pullRemoteNodeQueues } from './mesh-remote-event-pull.js';
 import {
     reconcileUnterminatedDirectDispatches,
     autoPruneStaleDirectDispatches,
@@ -129,16 +126,6 @@ interface LiveCoordinator {
 // "restart does not clear it" symptom the operator needs visibility into).
 const coordinatorModalParkState = new Map<string, boolean>();
 
-// Disk/worktree retention throttle. The reconcile tick runs every ~4s, but the
-// retention sweep (fs walks + git worktree list) is far too heavy for that cadence
-// and its artifacts age in days, so it runs at most once per hour. `undefined`
-// means "never run yet" → runs on the first tick after daemon start so a long-lived
-// backlog is reclaimed promptly rather than an hour later. Per-process; a restart
-// re-runs it immediately, which is the desired behavior (a restart is exactly when
-// stale artifacts from the previous run should be swept).
-const DISK_RETENTION_INTERVAL_MS = 60 * 60 * 1000; // 1h
-let lastDiskRetentionRunAt: number | undefined;
-
 // Find live CLI coordinator instances on THIS daemon, keyed by mesh.
 function findLiveCoordinators(components: DaemonComponents): LiveCoordinator[] {
     const out: LiveCoordinator[] = [];
@@ -734,32 +721,10 @@ const deliveredNoTurnUnknownStreak = new Map<string, number>();
 // immediately; UNKNOWN accrues a streak and re-drives only after RECLAIM_UNKNOWN_GRACE_TICKS.
 const deliveredUnconsumedUnknownStreak = new Map<string, number>();
 
-// KIMI-PURE-PTY-COMPLETION-EMIT (Fix 2): how long a delivered assigned row whose worker is
-// CONTINUOUSLY idle-WITH-a-final-assistant-message must stay so before we short-circuit it to
-// 'completed' from transcript evidence — WITHOUT waiting the full DELIVERED_NO_TURN_DEADLINE_MS
-// (15min). A pure-PTY worker (kimi and kin — no native transcript, no provider authority) whose
-// generating_completed never emitted (the onTurnStarted idle→idle collapse Fix 1 addresses at the
-// source) leaves the row 'assigned' with an idle worker that already rendered its answer. The
-// F3 long-deadline reclaim's pollAssignedTaskTerminalEvidence would eventually recover it — but only
-// after 15min, during which the coordinator ledger wrongly shows the task in flight. This EARLY
-// reconcile applies the SAME idle-with-final-assistant-after-dispatch evidence bar (via
-// pollAssignedTaskTerminalEvidence) after a few continuous-idle ticks, so a finished worker's task
-// is marked complete promptly. Conservative by construction: the streak resets the instant a read is
-// non-idle / lacks a final summary / is unreadable, and the poll itself enforces the after-dispatch
-// stale-summary guard, so a mid-turn or warming-up worker is never falsely completed.
-const ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS = 8_000;
-
-// Per-row continuous idle-with-final-assistant streak for the early transcript-evidence completion,
-// keyed `${meshId}::${taskId}`, storing the wall-clock ms when the continuous run began. Pruned each
-// pass to the currently-assigned rows (same as the UNKNOWN streaks). A non-idle / no-summary /
-// unreadable tick clears the entry so the grace must re-accumulate from scratch.
-const assignedIdleFinalAssistantSince = new Map<string, number>();
-
 // Test hook: clear the delivered-no-turn UNKNOWN streaks between cases.
 export function __resetReclaimUnknownStreakForTests(): void {
     deliveredNoTurnUnknownStreak.clear();
     deliveredUnconsumedUnknownStreak.clear();
-    assignedIdleFinalAssistantSince.clear();
 }
 
 // APPROVAL-INBOX-BLINDSPOT (Fix A.3): true when the assigned row's bound session is, per the
@@ -778,129 +743,12 @@ function assignedRowLiveStatusIsAwaitingApproval(
 ): boolean {
     if (!nodeId || !sessionId) return false;
     try {
-        // Both blocked-on-input states hold the row: an approval (waiting_approval) and a
-        // question (awaiting_choice) legitimately park the worker awaiting the coordinator's
-        // mesh_approve / mesh_answer_question, so neither should accrue the reclaim streak
-        // (mission f1d25e11 extends the APPROVAL-INBOX-BLINDSPOT guard to questions).
-        const liveStatus = sessionStatusFromNodes(mesh.nodes, nodeId, sessionId).status;
-        return liveStatus === 'awaiting_approval' || liveStatus === 'awaiting_choice';
+        return sessionStatusFromNodes(mesh.nodes, nodeId, sessionId).status === 'awaiting_approval';
     } catch {
         return false;
     }
 }
 
-// EARLY-IDLE-COMPLETION-FALSE-POSITIVE — resolve whether the assigned session's provider is
-// the pure-PTY transcript class (kimi and kin — no native history, no provider authority), from
-// the LOCAL instance when it is present. Returns:
-//   true      → local instance is a pure-PTY provider (the class the early rescue exists for)
-//   false     → local instance is NOT pure-PTY (native / provider-authoritative)
-//   undefined → no local instance (remote worker / gone / id-form skew) — pure-PTY unknowable here
-// A remote pure-PTY worker is handled by the reprobe path (a fresh-idle read positively confirms
-// the session is settled before the streak accrues), not by this local-only capability check.
-function resolveLocalSessionPurePty(components: DaemonComponents, sessionId: string): boolean | undefined {
-    try {
-        const instances = components.instanceManager?.getByCategory?.('cli') || [];
-        const inst = instances.find((i: any) => {
-            const sid = readNonEmptyString(i?.getState?.().instanceId);
-            return sid && sessionIdsEquivalent(sid, sessionId);
-        }) as { provider?: unknown } | undefined;
-        if (!inst) return undefined; // remote / gone / id-form skew — not locally observable
-        const provider = inst.provider;
-        if (!provider || typeof provider !== 'object') return undefined;
-        return isPurePtyTranscriptProvider(provider as Parameters<typeof isPurePtyTranscriptProvider>[0]);
-    } catch {
-        return undefined;
-    }
-}
-
-// EARLY-IDLE-COMPLETION-FALSE-POSITIVE — decide whether the early transcript-evidence completion
-// streak may ACCRUE this tick for an assigned row. This hardens the original arm gate, which only
-// checked `resolveSessionBusyVerdict(...) !== 'GENERATING'` — a gate a REMOTE worker (local verdict
-// UNKNOWN, not GENERATING) passed unconditionally, so its "8s continuous idle" streak degenerated to
-// 8s of wall-clock even while the worker was genuinely mid-turn, and a startup-grace boot-window idle
-// (before the first turn ever started) also passed. The row was then early-completed off a preamble.
-//
-// Two added requirements, defense-in-depth on top of the existing confirmed-delivery / no-terminal-
-// ledger / bound-session gate the caller applies:
-//   (a) POSITIVE idle evidence, not merely "not GENERATING". A LOCAL IDLE_CONFIRMED verdict qualifies
-//       directly. A remote/UNKNOWN verdict is RE-PROBED with a fresh read_chat status this tick: only
-//       a definitively 'idle' read lets the streak accrue; any active status (generating/waiting/…),
-//       or an inconclusive read, resets it. This is what stops the remote worker's mid-turn busy from
-//       silently passing as "continuous idle".
-//   (b) TURN-START evidence, so a boot-window / never-consumed idle cannot early-complete a preamble.
-//       taskDeliveryConsumed===true (the worker emitted agent:generating_started) satisfies it. When
-//       the delivery was never consumed, only a LOCAL pure-PTY provider — the exact class that never
-//       emits generating_started and that this rescue exists for — is still allowed; a native/
-//       provider-authoritative worker that hasn't started its turn is NOT armed (it will complete via
-//       its own emit or the normal grace). A remote-not-consumed worker whose provider class is
-//       unknowable locally falls back to the reprobe: it may accrue only once (a) reads it positively
-//       idle, and the poll's post-dispatch + trailing-tool-activity guards remain the final net.
-async function evaluateEarlyIdleTranscriptArm(
-    components: DaemonComponents,
-    mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
-    store: MeshRuntimeStore,
-    row: { id: string; assignedSessionId?: string; assignedNodeId?: string; assignedProviderType?: string },
-    localDaemonId?: string,
-    selfIds: string[] = [],
-): Promise<boolean> {
-    const sessionId = readNonEmptyString(row.assignedSessionId);
-    if (!sessionId) return false;
-
-    // (a) POSITIVE idle evidence.
-    const verdict = resolveSessionBusyVerdict(components, sessionId);
-    if (verdict === 'GENERATING') return false; // locally observed mid-turn — never accrue
-    if (verdict === 'UNKNOWN') {
-        // Remote / gone / id-form skew. First consult the LIVE mesh node snapshot (the same
-        // cross-daemon status feed the approval-hold guard and mesh_status use): if it already
-        // reports a definitively NON-idle status (generating / waiting_approval / awaiting_choice /
-        // …), the worker is not settled — reset the streak WITHOUT a read_chat. This keeps the
-        // streak the cheap per-tick gate for the common case and avoids probing a worker the graph
-        // already knows is busy or parked at an approval.
-        const nodeId = readNonEmptyString(row.assignedNodeId);
-        const node = (mesh.nodes ?? []).find(n => n.id === nodeId);
-        const liveStatus = nodeId
-            ? readNonEmptyString(sessionStatusFromNodes(mesh.nodes, nodeId, sessionId).status).toLowerCase()
-            : '';
-        if (liveStatus && liveStatus !== 'idle') return false; // live graph says non-idle → not a turn-end
-        if (!liveStatus) {
-            // No live snapshot for this session — re-probe the worker's own status this tick so a
-            // genuinely busy remote worker breaks the streak instead of coasting through as "not
-            // GENERATING". getInstance may be absent on some component surfaces (tests / standalone),
-            // so guard it.
-            const nodeDaemonId = readNonEmptyString(node?.daemonId);
-            const isLocalNode = !nodeDaemonId
-                || daemonIdsEquivalent(nodeDaemonId, localDaemonId)
-                || daemonIdListIncludes(selfIds, nodeDaemonId)
-                || !!components.instanceManager?.getInstance?.(sessionId);
-            const providerType = readNonEmptyString(row.assignedProviderType);
-            const readArgs: Record<string, unknown> = {
-                sessionId,
-                targetSessionId: sessionId,
-                tailLimit: 1,
-                ...(node?.workspace ? { workspace: node.workspace } : {}),
-                ...(providerType ? { agentType: providerType, providerType } : {}),
-            };
-            const probedStatus = await reprobeWorkerStatus(components, { isLocalNode, nodeDaemonId, readArgs });
-            // Only a positively-idle re-probe lets the streak accrue. A non-idle status (the worker
-            // IS busy) or an inconclusive read (null — transport error / no payload) resets it: we
-            // never treat "couldn't tell" as idle for this early-completion path.
-            if (probedStatus !== 'idle') return false;
-        }
-    }
-
-    // (b) TURN-START evidence (guards the boot-window / preamble false positive).
-    if (store.taskDeliveryConsumed(mesh.id, row.id)) return true; // worker emitted generating_started
-    // Delivery never consumed. Preserve the kimi rescue ONLY for the LOCAL pure-PTY class (which
-    // never emits generating_started); a native/provider-authoritative local worker that hasn't
-    // started its turn must NOT be armed off a boot-window idle. For a remote worker (pure-PTY
-    // unknowable here) the positive-idle reprobe in (a) is the gate — allow accrual and let the poll
-    // enforce post-dispatch + trailing-tool-activity finality.
-    const localPurePty = resolveLocalSessionPurePty(components, sessionId);
-    if (localPurePty === true) return true;   // local pure-PTY — the class this rescue targets
-    if (localPurePty === false) return false; // local native/provider-owned, turn not started — hold
-    return verdict === 'UNKNOWN';             // remote, unknowable class — trust the (a) reprobe
-}
-
 // PHASE 2.5 — assigned-stranded dispatch watchdog (Bug B). claimNextTask atomically
 // flips a row to 'assigned' BEFORE the fire-and-forget dispatch runs. If that dispatch
 // neither rejects (→ no .catch requeue) nor is confirmed delivered — a relay that hangs
@@ -917,8 +765,6 @@ async function recoverStrandedAssignedDispatches(
     components: DaemonComponents,
     mesh: { id: string; nodes?: Array<{ id: string; daemonId?: string; workspace?: string }> },
     store: MeshRuntimeStore,
-    localDaemonId?: string,
-    selfIds: string[] = [],
 ): Promise<void> {
     const meshId = mesh.id;
     const assigned = getQueue(meshId, { status: ['assigned'] });
@@ -935,82 +781,10 @@ async function recoverStrandedAssignedDispatches(
     for (const key of [...deliveredUnconsumedUnknownStreak.keys()]) {
         if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) deliveredUnconsumedUnknownStreak.delete(key);
     }
-    for (const key of [...assignedIdleFinalAssistantSince.keys()]) {
-        if (key.startsWith(meshKeyPrefix) && !assignedKeys.has(key)) assignedIdleFinalAssistantSince.delete(key);
-    }
     for (const row of assigned) {
         const dispatchedAtMs = Date.parse(row.dispatchTimestamp ?? '');
         if (!Number.isFinite(dispatchedAtMs)) continue;              // no dispatch ts → can't age it
         const ageMs = nowMs - dispatchedAtMs;
-        const idleTranscriptStreakKey = `${meshId}::${row.id}`;
-        // KIMI-PURE-PTY-COMPLETION-EMIT (Fix 2): EARLY transcript-evidence completion for a
-        // delivered assigned row whose worker is already idle with a final assistant answer, so a
-        // pure-PTY worker whose generating_completed never emitted is marked done PROMPTLY instead of
-        // sitting 'assigned' until the 15-min DELIVERED_NO_TURN_DEADLINE_MS reclaim. Gate on a
-        // CONFIRMED delivery (a never-delivered row is handled by dispatch/redrive, not completion)
-        // and a bound session, then require the worker to read idle-WITH-final-assistant continuously
-        // for ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS before polling the decisive transcript evidence.
-        // The streak is the cheap gate (avoids a read every tick); the read only runs once the grace
-        // elapses. pollAssignedTaskTerminalEvidence enforces idle + final-assistant-after-dispatch, so
-        // a mid-turn / stale-tail / unreadable worker is never falsely completed — a non-qualifying
-        // tick clears the streak below.
-        // EARLY-IDLE-COMPLETION-FALSE-POSITIVE: the arm gate now requires POSITIVE idle evidence
-        // (a fresh-idle reprobe for a remote/UNKNOWN worker, not merely "not GENERATING") AND
-        // turn-start evidence (delivery consumed, or the local pure-PTY class this rescue targets),
-        // so a mid-turn remote worker or a boot-window preamble can no longer coast the continuous-
-        // idle streak to a false completion. See evaluateEarlyIdleTranscriptArm.
-        const earlyArm =
-            store.taskHasConfirmedDelivery(meshId, row.id)
-            && !findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })
-            && readNonEmptyString(row.assignedSessionId)
-                ? await evaluateEarlyIdleTranscriptArm(components, mesh, store, row, localDaemonId, selfIds)
-                : false;
-        if (earlyArm) {
-            const since = assignedIdleFinalAssistantSince.get(idleTranscriptStreakKey);
-            if (since === undefined) {
-                assignedIdleFinalAssistantSince.set(idleTranscriptStreakKey, nowMs);
-            } else if (nowMs - since >= ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS) {
-                const terminalEvidence = await pollAssignedTaskTerminalEvidence(components, mesh, row);
-                if (terminalEvidence) {
-                    assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
-                    updateTaskStatus(meshId, row.id, terminalEvidence);
-                    if (!findTerminalLedgerEvidenceForTask({ meshId, taskId: row.id })) {
-                        try {
-                            appendLedgerEntry(meshId, {
-                                kind: terminalEvidence === 'completed' ? 'task_completed' : 'task_failed',
-                                nodeId: row.assignedNodeId,
-                                sessionId: row.assignedSessionId,
-                                providerType: row.assignedProviderType,
-                                payload: {
-                                    taskId: row.id,
-                                    event: 'agent:generating_completed',
-                                    source: 'early_idle_transcript_evidence',
-                                },
-                            });
-                        } catch { /* best-effort ledger write */ }
-                    }
-                    LOG.warn('MeshReconcile', `Early-completed assigned task ${row.id} on mesh ${meshId} `
-                        + `(node=${row.assignedNodeId ?? '?'} session=${row.assignedSessionId ?? '?'}): worker idle with a `
-                        + `final assistant message after dispatch for ≥${Math.round(ASSIGNED_IDLE_TRANSCRIPT_COMPLETE_MS / 1000)}s — `
-                        + `the completion event was lost/late (pure-PTY provider), task is ${terminalEvidence} without waiting the 15-min deadline`);
-                    traceMeshEventDrop('assigned_early_transcript_completed', {
-                        taskId: row.id,
-                        sessionId: row.assignedSessionId,
-                        nodeId: row.assignedNodeId,
-                        meshId,
-                        event: 'agent:generating_completed',
-                    }, terminalEvidence);
-                    continue;
-                }
-                // Poll was inconclusive (mid-turn re-check / stale tail / unreadable) — reset the
-                // streak so it must re-accumulate a fresh continuous-idle run before the next poll.
-                assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
-            }
-        } else {
-            // Not a qualifying tick (no confirmed delivery, generating, terminal ledger present, or
-            // no bound session) — break the continuous-idle run.
-            assignedIdleFinalAssistantSince.delete(idleTranscriptStreakKey);
-        }
         // DELIVERED-NOT-CONSUMED short-grace re-drive (remote autoLaunch delivered≠consumed gap).
         // Runs BEFORE the ASSIGNED_STRANDED_DEADLINE_MS confirm-window gate below because its whole
         // point is to recover a delivered-but-unconsumed row well inside that window. A remote
@@ -1042,45 +816,6 @@ async function recoverStrandedAssignedDispatches(
                 updateTaskStatus(meshId, row.id, status);
                 continue;
             }
-            // GENERATING-STARTED-CONSUME-RACE (fix B): the delivery-consume that clears this
-            // gate (delivered → acked) runs ONLY when the coordinator pulls the worker's queued
-            // agent:generating_started in PHASE 1 (handleMeshForwardEvent → consumeSessionDelivery).
-            // A worker may have emitted generating_started ALREADY, but if PHASE 1 has not yet
-            // pulled that specific event (a skipped/slow peer tick, or the event was queued after
-            // this tick's pull ran), the delivery row still reads 'delivered' here and the redrive
-            // below tears a genuinely-generating remote worker off its task and re-injects the
-            // prompt (the delivered_not_consumed_redrive symptom). Close the race by issuing a
-            // TARGETED, in-process last-chance pull of THIS node's queue right now, then
-            // re-reading taskDeliveryConsumed(): if the worker's generating_started was waiting to
-            // be pulled, the pull consumes the delivery this tick and we skip the redrive entirely.
-            // Best-effort and self-scoped (the same skip/peer guards as PHASE 1) — a miss just
-            // falls through to the existing UNKNOWN-streak grace, so this only ever removes false
-            // redrives, never adds one. A local (co-hosted) worker has nothing to pull (the node's
-            // daemon is a self id) and this is a fast no-op.
-            const assignedNode = row.assignedNodeId
-                ? mesh.nodes?.find(n => meshNodeIdMatches(n, row.assignedNodeId))
-                : undefined;
-            if (assignedNode?.daemonId) {
-                try {
-                    await pullPendingEventsFromNode(
-                        components,
-                        meshId,
-                        assignedNode,
-                        localDaemonId,
-                        selfIds,
-                        selfIds.length > 0
-                            ? selfIds.map(id => ({ meshId, coordinatorDaemonId: id }))
-                            : [{ meshId }],
-                    );
-                } catch { /* best-effort last-chance pull */ }
-                if (store.taskDeliveryConsumed(meshId, row.id)) {
-                    // The pull delivered the worker's generating_started (or a terminal event) and
-                    // consumed the delivery — the task is genuinely live/handled. Reset any accrued
-                    // streak and leave the row to PHASE 4's completion reconcile.
-                    deliveredUnconsumedUnknownStreak.delete(`${meshId}::${row.id}`);
-                    continue;
-                }
-            }
             const shortStreakKey = `${meshId}::${row.id}`;
             const verdict = row.assignedSessionId
                 ? resolveSessionBusyVerdict(components, row.assignedSessionId)
@@ -1476,7 +1211,7 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
             const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
             if (!daemonHostsMesh(mesh, selfIds)) continue;
             try {
-                await recoverStrandedAssignedDispatches(components, mesh, store, localDaemonId, selfIds);
+                await recoverStrandedAssignedDispatches(components, mesh, store);
             } catch (e: any) {
                 LOG.warn('MeshReconcile', `Assigned-stranded watchdog failed for mesh ${mesh.id}: ${e?.message || e}`);
             }
@@ -1619,44 +1354,6 @@ export async function runMeshReconcileTick(components: DaemonComponents): Promis
         }
     }
 
-    // ── PHASE 6: disk / worktree retention (hourly, mission 86def38d) ──────────
-    // Legacy on-disk artifacts under ~/.adhdev accumulated with NO lifetime and grew
-    // the data volume until a refine bootstrap failed at 98% disk. This throttled pass
-    // (≤1×/hour — the artifacts age in days and the fs/git walk is too heavy for the 4s
-    // tick) reclaims them:
-    //   • JSONL ledger files older than 30d (legacy after the SQLite ledger)
-    //   • terminated session-host runtime files older than 14d (live never touched)
-    //   • mesh-runtime.db.bak-* backups older than 7d
-    //   • orphan worktree DETECTION → cleanup_candidate ledger signal (NEVER deleted).
-    // Runs BEFORE PHASE 2's early return so it fires whether or not a live CLI
-    // coordinator exists on this daemon. Isolated in try/catch so it can never kill the
-    // tick. All file deletions are defensive (age + live-runtime guards in the pure
-    // selectors); worktree cleanup stays manual/coordinator-driven.
-    {
-        const nowMs = Date.now();
-        const due = lastDiskRetentionRunAt === undefined
-            || (nowMs - lastDiskRetentionRunAt) >= DISK_RETENTION_INTERVAL_MS;
-        if (due) {
-            lastDiskRetentionRunAt = nowMs;
-            try {
-                runDiskRetentionSweep(nowMs);
-            } catch (e: any) {
-                LOG.warn('MeshReconcile', `Disk retention sweep failed: ${e?.message || e}`);
-            }
-            // Orphan-worktree detection is per-mesh (needs the mesh's base repo + node set)
-            // and only for meshes this daemon hosts (its local base checkout is the git anchor).
-            for (const mesh of listMeshes()) {
-                const selfIds = resolveCoordinatorSelfIds(mesh, drainDaemonIds);
-                if (!daemonHostsMesh(mesh, selfIds)) continue;
-                try {
-                    await detectAndSignalOrphanWorktrees(mesh, nowMs);
-                } catch (e: any) {
-                    LOG.warn('MeshReconcile', `Orphan worktree detection failed for mesh ${mesh.id}: ${e?.message || e}`);
-                }
-            }
-        }
-    }
-
     // ── PHASE 2: inject into live CLI coordinators on this daemon ──────────────
     const coordinators = findLiveCoordinators(components);
     if (coordinators.length === 0) {