@adhdev/daemon-core 0.9.82-rc.468 → 0.9.82-rc.469

package/src/mesh/mesh-events-pending.ts

@@ -189,6 +189,10 @@ function ledgerRecordQuarantinedEvent(event: PendingMeshCoordinatorEvent, reason
                 ...(readNonEmptyString(event.eventId) ? { eventId: event.eventId } : {}),
                 queuedAt: event.queuedAt,
                 ...(finalSummary ? { finalSummary } : {}),
+                // Full original event so mesh_requeue_held_events can restore it
+                // losslessly (event_held→pending). The summary/label fields above stay
+                // for human-readable audit; `heldEvent` is the machine recovery copy.
+                heldEvent: event,
             },
         });
     } catch (e: any) {
@@ -746,8 +750,11 @@ function trimPendingEventsIfNeeded(path: string): void {
                         nodeLabel: event.nodeLabel,
                         ...(event.workspace ? { workspace: event.workspace } : {}),
                         targetCoordinatorDaemonId: event.targetCoordinatorDaemonId ?? null,
+                        ...(readNonEmptyString(event.eventId) ? { eventId: event.eventId } : {}),
                         queuedAt: event.queuedAt,
                         ...(finalSummary ? { finalSummary } : {}),
+                        // Full original event for lossless mesh_requeue_held_events restore.
+                        heldEvent: event,
                     },
                 });
                 LOG.warn('MeshEvents', `Pending-events trim dropping undelivered ${event.event} for mesh ${event.meshId} — recorded to ledger (recoverable)`);
@@ -1303,6 +1310,175 @@ export function __persistUnstampedPendingEventForTests(event: PendingMeshCoordin
     return persistPendingMeshCoordinatorEvent(event);
 }
 
+// ---------------------------------------------------------------------------
+// event_held → pending requeue (T6 recovery path)
+// ---------------------------------------------------------------------------
+// T6 quarantine (v2 enforce) and the pending-events trim both mirror a
+// destructively-drained-but-undelivered event into the ledger as a recoverable
+// `event_held` entry. Until now that recovery channel was audit-only: the comment
+// said "an operator can requeue it" but no code path did. This restores those held
+// events to the pending queue so a coordinator drains them on its next poll.
+//
+// No-loss + no-double-requeue invariants:
+//   • The full original event rides on the held payload as `heldEvent`, so the
+//     restore is byte-for-byte (metadataEvent, coordinatorMessage, v2 envelope).
+//   • queuePendingMeshCoordinatorEvent runs the normal dedup (fingerprint / eventId),
+//     so an event still live in the queue is not duplicated.
+//   • Every held entry that has ALREADY been requeued is marked with an
+//     `event_held_requeued` ledger entry keyed by the source held-entry id; a later
+//     pass skips those ids, so calling the tool twice does not requeue the same held
+//     event twice.
+
+/** Narrowing filter for {@link requeueHeldMeshCoordinatorEvents}, scoped within one mesh. */
+export interface MeshHeldEventRequeueFilter {
+    /** Restore only held events whose worker task id matches (from the held event's metadata/taskId). */
+    taskId?: string;
+    /** Restore only held events originating from this node. */
+    nodeId?: string;
+    /** Restore only held events of this event name (e.g. 'session:completed'). */
+    event?: string;
+    /** Restore only held entries recorded at/after this ISO timestamp. */
+    since?: string;
+    /** Restore only held entries with this hold reason (e.g. 'pending_trim_dropped'). */
+    reason?: string;
+}
+
+export interface MeshHeldEventRequeueResult {
+    meshId: string;
+    /** event_held entries considered after the filter. */
+    matched: number;
+    /** entries skipped because a prior requeue already recovered them. */
+    alreadyRequeued: number;
+    /** entries skipped because they carried no restorable original event / were not recoverable. */
+    unrecoverable: number;
+    /** entries handed to the pending queue (some may have been dedup-suppressed downstream). */
+    requeued: number;
+    /** of `requeued`, how many the pending-queue dedup collapsed onto a live event. */
+    dedupSuppressed: number;
+    entries: Array<{
+        heldEntryId: string;
+        event: string;
+        nodeId?: string;
+        taskId?: string;
+        reason?: string;
+        outcome: 'requeued' | 'already_requeued' | 'unrecoverable';
+    }>;
+}
+
+/** Read the taskId a held event carried, checking the restored event then the audit payload. */
+function readHeldTaskId(restored: PendingMeshCoordinatorEvent | undefined, payload: Record<string, unknown>): string {
+    const fromMeta = restored?.metadataEvent && typeof restored.metadataEvent === 'object'
+        ? readNonEmptyString((restored.metadataEvent as Record<string, unknown>).taskId)
+        : '';
+    return fromMeta || readNonEmptyString(payload.taskId) || '';
+}
+
+/**
+ * Restore recoverable `event_held` ledger entries back to the pending coordinator
+ * queue for `meshId`. See the block comment above for the no-loss / no-double-requeue
+ * invariants. Returns per-entry outcomes for the caller to surface.
+ */
+export function requeueHeldMeshCoordinatorEvents(
+    meshId: string,
+    filter?: MeshHeldEventRequeueFilter,
+): MeshHeldEventRequeueResult {
+    const result: MeshHeldEventRequeueResult = {
+        meshId,
+        matched: 0,
+        alreadyRequeued: 0,
+        unrecoverable: 0,
+        requeued: 0,
+        dedupSuppressed: 0,
+        entries: [],
+    };
+
+    const all = readLedgerEntries(meshId);
+    // held-entry ids already recovered by a prior requeue pass (dedup key = source id).
+    const requeuedIds = new Set<string>();
+    for (const entry of all) {
+        if (entry.kind !== 'event_held_requeued') continue;
+        const id = readNonEmptyString(entry.payload?.heldEntryId);
+        if (id) requeuedIds.add(id);
+    }
+
+    const sinceMs = filter?.since ? new Date(filter.since).getTime() : NaN;
+    const wantEvent = readNonEmptyString(filter?.event);
+    const wantNode = readNonEmptyString(filter?.nodeId);
+    const wantTask = readNonEmptyString(filter?.taskId);
+    const wantReason = readNonEmptyString(filter?.reason);
+
+    for (const entry of all) {
+        if (entry.kind !== 'event_held') continue;
+        const payload = (entry.payload && typeof entry.payload === 'object') ? entry.payload : {};
+        if (payload.recoverable !== true) continue;
+
+        // Reconstruct the original event: prefer the full `heldEvent` copy; fall back to
+        // the flat audit fields for entries written before the copy was embedded.
+        const restored: PendingMeshCoordinatorEvent | undefined =
+            (payload.heldEvent && typeof payload.heldEvent === 'object')
+                ? { ...(payload.heldEvent as PendingMeshCoordinatorEvent) }
+                : undefined;
+
+        const eventName = restored?.event || readNonEmptyString(payload.event);
+        const nodeId = restored?.nodeId || entry.nodeId || readNonEmptyString((payload as any).nodeId) || undefined;
+        const taskId = readHeldTaskId(restored, payload as Record<string, unknown>);
+        const reason = readNonEmptyString(payload.reason) || undefined;
+
+        // Apply the caller filter within the mesh scope.
+        if (wantEvent && eventName !== wantEvent) continue;
+        if (wantNode && nodeId !== wantNode) continue;
+        if (wantTask && taskId !== wantTask) continue;
+        if (wantReason && reason !== wantReason) continue;
+        if (filter?.since && !Number.isNaN(sinceMs) && new Date(entry.timestamp).getTime() < sinceMs) continue;
+
+        result.matched++;
+
+        if (requeuedIds.has(entry.id)) {
+            result.alreadyRequeued++;
+            result.entries.push({ heldEntryId: entry.id, event: eventName, ...(nodeId ? { nodeId } : {}), ...(taskId ? { taskId } : {}), ...(reason ? { reason } : {}), outcome: 'already_requeued' });
+            continue;
+        }
+
+        if (!restored || !readNonEmptyString(restored.event) || !readNonEmptyString(restored.meshId)) {
+            result.unrecoverable++;
+            result.entries.push({ heldEntryId: entry.id, event: eventName, ...(nodeId ? { nodeId } : {}), ...(taskId ? { taskId } : {}), ...(reason ? { reason } : {}), outcome: 'unrecoverable' });
+            continue;
+        }
+
+        // Restore to pending. queuePendingMeshCoordinatorEvent re-stamps/dedups; a live
+        // duplicate is suppressed there (returns true) so we never double-deliver.
+        const beforeDup = hasPendingCoordinatorEventDuplicate(restored);
+        let ok = false;
+        try {
+            ok = queuePendingMeshCoordinatorEvent(restored);
+        } catch (e: any) {
+            LOG.warn('MeshEvents', `Requeue of held ${eventName} for mesh ${meshId} failed: ${e?.message || e}`);
+        }
+
+        // Mark the source held entry so a second pass skips it, regardless of whether the
+        // queue dedup collapsed it (the recovery attempt is what we dedup on, not delivery).
+        appendLedgerEntry(meshId, {
+            kind: 'event_held_requeued',
+            ...(nodeId ? { nodeId } : {}),
+            payload: {
+                heldEntryId: entry.id,
+                event: eventName,
+                requeued: ok,
+                ...(taskId ? { taskId } : {}),
+                ...(reason ? { reason } : {}),
+                ...(beforeDup ? { dedupSuppressed: true } : {}),
+            },
+        });
+        requeuedIds.add(entry.id);
+
+        result.requeued++;
+        if (beforeDup) result.dedupSuppressed++;
+        result.entries.push({ heldEntryId: entry.id, event: eventName, ...(nodeId ? { nodeId } : {}), ...(taskId ? { taskId } : {}), ...(reason ? { reason } : {}), outcome: 'requeued' });
+    }
+
+    return result;
+}
+
 /** Explicitly clear all pending coordinator events for a mesh (and coordinator if scoped). */
 export function clearPendingMeshCoordinatorEvents(meshId?: string, coordinatorDaemonId?: string): void {
     if (!meshId) return;

package/src/mesh/mesh-events.ts

@@ -5,12 +5,17 @@
 // New code should import directly from the relevant sub-module.
 // ---------------------------------------------------------------------------
 
-export type { PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
+export type {
+    PendingMeshCoordinatorEvent,
+    MeshHeldEventRequeueFilter,
+    MeshHeldEventRequeueResult,
+} from './mesh-events-pending.js';
 export {
     queuePendingMeshCoordinatorEvent,
     drainPendingMeshCoordinatorEvents,
     getPendingMeshCoordinatorEvents,
     clearPendingMeshCoordinatorEvents,
+    requeueHeldMeshCoordinatorEvents,
     serializeV2EnvelopeToWire,
     readV2EnvelopeFromWire,
     getMeshV2DrainCounters,

package/src/mesh/mesh-ledger.ts

@@ -49,6 +49,11 @@ export type MeshLedgerKind =
     | 'delivery_unroutable'
     | 'direct_dispatch_pruned'
     | 'event_held'
+    // Audit marker written by mesh_requeue_held_events when a recoverable `event_held`
+    // entry is restored to the pending queue (event_held→pending). Keyed by the source
+    // held-ledger-entry id so a second requeue pass skips already-recovered entries
+    // (no double-requeue). payload: { heldEntryId, event, requeued: boolean, reason?, dedupSuppressed?: boolean }
+    | 'event_held_requeued'
     | 'task_reclaimed'
     // Gap2-A: a coordinator-recorded operating note — a runtime-accumulated
     // lesson (provider quirk, pattern to avoid, recovery lesson) persisted in

package/src/mesh/mesh-queue-assignment.ts

@@ -6,7 +6,7 @@ import { getMesh } from '../config/mesh-config.js';
 import { detectCLI } from '../detection/cli-detector.js';
 import { LOG } from '../logging/logger.js';
 import { appendLedgerEntry } from './mesh-ledger.js';
-import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches, isTaskReadonly, taskDependenciesSatisfied } from './mesh-work-queue.js';
+import { buildMeshNodeCapabilityTags, nodeSatisfiesRequiredTags, claimNextTask, updateTaskStatus, getQueue, recordTaskAutoLaunch, getActiveDirectDispatches, isTaskReadonly, taskDependenciesSatisfied, meshTaskNotBeforeReady, meshTaskPriorityRank } from './mesh-work-queue.js';
 import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
 import { fastForwardMeshNode } from './mesh-fast-forward.js';
 import { createSessionDelivery, updateSessionDeliveryStatus } from './mesh-delivery-policy.js';
@@ -1563,7 +1563,13 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
     // so the dependency gate below sees the terminal state of every referenced
     // dependency, not just the still-active rows.
     const statusById = new Map(queue.map(task => [task.id, task.status] as const));
-    const pending = queue.filter(task => task.status === 'pending');
+    // G6: scan higher task-level priority first so a high-priority task auto-launches its
+    // session ahead of an older normal/low task (getQueue is FIFO; a stable sort by priority
+    // rank descending keeps created_at order within a priority band). The claim path applies
+    // the same ordering, so the launched session pulls the same task the scan chose.
+    const pending = queue
+        .filter(task => task.status === 'pending')
+        .sort((a, b) => meshTaskPriorityRank(b.priority) - meshTaskPriorityRank(a.priority));
     // AUTOLAUNCH-CLAIM-CHURN: prune await-claim backoff state for tasks of this mesh that are no
     // longer pending (claimed/completed/cancelled) so the map cannot grow without bound.
     {
@@ -1595,6 +1601,14 @@ async function maybeAutoLaunchOneQueueSession(components: DaemonComponents, mesh
             markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'dependencies_unsatisfied' });
             continue;
         }
+        // G7: never spawn a session for a task still held by its notBefore gate — the launched
+        // session would idle→claim and be refused by the SAME gate in claimNextQueueTask,
+        // producing orphan-session churn. Skip it so a later tick (after not_before passes)
+        // launches it. Tasks with no notBefore pass through unchanged.
+        if (!meshTaskNotBeforeReady(task)) {
+            markAutoLaunch(meshId, task.id, { status: 'skipped', reason: 'not_before_delayed' });
+            continue;
+        }
         const isReadonly = isTaskReadonly(task);
         if (isReadonly) {
             if (activeReadonlyAssignedCount(meshId) >= maxReadonlyParallelTasks) {

package/src/mesh/mesh-reconcile-config.ts

@@ -0,0 +1,66 @@
+// ---------------------------------------------------------------------------
+// mesh-reconcile-config — reconcile-loop timing tunables + env resolvers
+// ---------------------------------------------------------------------------
+// Extracted from mesh-reconcile-loop.ts (A-3 god-module decomposition, pure move,
+// no behavior change). Holds the loop-cadence tunables and their env-override
+// resolvers. Each resolver reads MESH_*_MS from the environment and clamps the
+// value so a mis-set env cannot make the loop pathological. Single-consumer
+// deadline constants that live next to their sole reader (e.g.
+// ASSIGNED_STRANDED_DEADLINE_MS, STRICT_SESSION_MATCH_TTL_MS) intentionally stay
+// in mesh-reconcile-loop.ts — only the shared loop-cadence tunables move here.
+// ---------------------------------------------------------------------------
+
+import { readNonEmptyString } from './mesh-events-utils.js';
+import { resolveTunedReconcileMs } from './mesh-reconcile-acked-hold.js';
+
+// Default reconcile cadence. approval/completion notifications to a live CLI
+// coordinator land within at most one interval. Overridable via env for tuning.
+export const DEFAULT_RECONCILE_INTERVAL_MS = 4_000;
+
+// PHASE 5 (auto-prune) conservative age gate. A direct dispatch whose node/session is
+// orphaned (no longer in the live mesh) is only auto-pruned once it is at least this old,
+// measured from its dispatch time. This protects against a node/session that is only
+// *transiently* invisible (a momentary probe failure, a daemon restart) being pruned the
+// instant it disappears. The MANUAL prune (mesh_prune_stale_direct) has no age gate — an
+// operator pruning explicitly wants the orphan gone now. Overridable via env for tuning.
+export const DEFAULT_AUTO_PRUNE_MIN_AGE_MS = 24 * 60 * 60_000; // 24h
+
+export function resolveAutoPruneMinAgeMs(): number {
+    const raw = readNonEmptyString(process.env.MESH_AUTO_PRUNE_MIN_AGE_MS);
+    if (raw) {
+        const parsed = Number.parseInt(raw, 10);
+        // Clamp to [1h, 30d] so a mis-set env can't make the gate pathologically aggressive
+        // (prune the moment something blinks) or effectively disable it forever.
+        if (Number.isFinite(parsed) && parsed >= 60 * 60_000 && parsed <= 30 * 24 * 60 * 60_000) return parsed;
+    }
+    return DEFAULT_AUTO_PRUNE_MIN_AGE_MS;
+}
+
+// PTY-OVERTRUST-DRAIN (Defect B, fix B). Age-based escape for the
+// `generating_no_idle_coordinator` hold. Fix A makes the drain predicate read the RAW
+// adapter (mask-stripped), so the common mask-driven false-busy is gone. But a hold can
+// still arise from a genuine status-source desync that fix A does not reach (e.g. the
+// adapter raw itself momentarily reads generating while the coordinator is actually at a
+// turn end). This is a TIME-BASED BACKSTOP: when a mesh's pending terminal events have
+// been held this long, re-confirm the coordinator's RAW adapter idle on the tick and, if
+// it is genuinely idle, drain ONCE. It NEVER injects into a genuinely-generating PTY —
+// the re-confirmation gates on raw adapter idle, so the intentional removal of
+// force-inject-into-generating (data-loss) is preserved. Default 12s = 3 reconcile ticks
+// at the 4s cadence: long enough that a normal mid-turn settle is not pre-empted, short
+// enough that a desync-stranded completion is not held for minutes. Env-tunable.
+export const DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS = 12_000;
+
+export function resolvePendingHeldDrainEscalateMs(): number {
+    // Floor 4s (one tick) so a mis-set env cannot make the escape race a normal settle;
+    // ceiling 5min so it cannot be disabled into a permanent strand.
+    return resolveTunedReconcileMs('MESH_PENDING_HELD_DRAIN_ESCALATE_MS', DEFAULT_PENDING_HELD_DRAIN_ESCALATE_MS, 4_000, 5 * 60_000);
+}
+
+export function resolveReconcileIntervalMs(): number {
+    const raw = readNonEmptyString(process.env.MESH_RECONCILE_INTERVAL_MS);
+    if (raw) {
+        const parsed = Number.parseInt(raw, 10);
+        if (Number.isFinite(parsed) && parsed >= 1_000 && parsed <= 60_000) return parsed;
+    }
+    return DEFAULT_RECONCILE_INTERVAL_MS;
+}