@adhdev/daemon-core 0.9.82-rc.37 → 0.9.82-rc.371

package/src/mesh/mesh-task-stats.ts

@@ -0,0 +1,161 @@
+/**
+ * M7: Operational stats (time/attempts) derived from existing truth.
+ *
+ * No cost/token accounting — ADHDev observes PTY/CDP and cannot see API
+ * tokens (explicit non-goal). Everything here is derived at query time from
+ * the SQLite ledger and queue rows; there is no separate aggregate table.
+ * Tasks with missing ledger evidence report incompleteEvidence instead of
+ * estimated numbers.
+ */
+
+import { readLedgerEntries } from './mesh-ledger.js';
+import type { MeshLedgerEntry } from './mesh-ledger.js';
+import { getQueue } from './mesh-work-queue.js';
+
+export interface MeshTaskStats {
+    taskId: string;
+    status: string;
+    dispatchedAt: string | null;
+    terminalAt: string | null;
+    terminalKind: 'task_completed' | 'task_failed' | null;
+    /** dispatched → terminal wall clock; null when evidence is incomplete. */
+    durationMs: number | null;
+    /** Number of task_dispatched ledger entries observed for this task. */
+    dispatchCount: number;
+    /** Queue requeueCount (0 when the row is gone or never requeued). */
+    requeueCount: number;
+    /** True when dispatch or terminal ledger evidence is missing — numbers are withheld, never estimated. */
+    incompleteEvidence?: true;
+}
+
+export interface MeshMissionStats {
+    missionId: string;
+    taskCount: number;
+    completed: number;
+    failed: number;
+    /** Sum of per-task durations with complete evidence. */
+    totalDurationMs: number;
+    /** First dispatch → last terminal across the mission's tasks; null without complete endpoints. */
+    wallClockMs: number | null;
+    /** Total requeue attempts across the mission's tasks. */
+    retries: number;
+    /** Task ids whose ledger evidence was incomplete (excluded from sums). */
+    incompleteTaskIds: string[];
+}
+
+function readPayloadTaskId(entry: MeshLedgerEntry): string {
+    const value = entry.payload?.taskId;
+    return typeof value === 'string' && value.trim() ? value.trim() : '';
+}
+
+function parseTime(value: string | null | undefined): number | null {
+    if (!value) return null;
+    const parsed = new Date(value).getTime();
+    return Number.isFinite(parsed) ? parsed : null;
+}
+
+/**
+ * Compute per-task stats from ledger entries. Scans a bounded tail window —
+ * stats are an operational view of recent work, not a full historical report.
+ */
+export function computeMeshTaskStats(meshId: string, opts?: { taskIds?: string[]; missionId?: string; tail?: number }): MeshTaskStats[] {
+    const queue = getQueue(meshId);
+    const queueById = new Map(queue.map(task => [task.id, task]));
+
+    let targetIds: string[];
+    if (opts?.taskIds?.length) {
+        targetIds = [...new Set(opts.taskIds)];
+    } else if (opts?.missionId) {
+        targetIds = queue.filter(task => task.missionId === opts.missionId).map(task => task.id);
+    } else {
+        targetIds = queue.map(task => task.id);
+    }
+    if (targetIds.length === 0) return [];
+    const targetSet = new Set(targetIds);
+
+    const entries = readLedgerEntries(meshId, { tail: opts?.tail ?? 1000 });
+    const dispatches = new Map<string, { first: string; count: number }>();
+    const terminals = new Map<string, { at: string; kind: 'task_completed' | 'task_failed' }>();
+    for (const entry of entries) {
+        const taskId = readPayloadTaskId(entry);
+        if (!taskId || !targetSet.has(taskId)) continue;
+        if (entry.kind === 'task_dispatched') {
+            const existing = dispatches.get(taskId);
+            if (existing) existing.count += 1;
+            else dispatches.set(taskId, { first: entry.timestamp, count: 1 });
+        } else if (entry.kind === 'task_completed' || entry.kind === 'task_failed') {
+            // Last terminal wins (requeued tasks can have multiple terminals).
+            terminals.set(taskId, { at: entry.timestamp, kind: entry.kind });
+        }
+    }
+
+    return targetIds.map(taskId => {
+        const queueEntry = queueById.get(taskId);
+        const dispatch = dispatches.get(taskId);
+        const terminal = terminals.get(taskId);
+        // Direct dispatches (mesh_send_task) have no work-queue row, so queueEntry is undefined.
+        // Derive a terminal status from the attributed terminal ledger entry instead of reporting
+        // status='unknown' — without this a completed direct task showed unknown + terminalKind=null
+        // even though its task_completed event fired (see mesh-events-coordinator Fix B attribution).
+        const status = queueEntry?.status
+            ?? (terminal
+                ? (terminal.kind === 'task_completed' ? 'completed' : 'failed')
+                : 'unknown');
+        const isTerminalStatus = status === 'completed' || status === 'failed' || status === 'cancelled';
+        const dispatchTime = parseTime(dispatch?.first ?? queueEntry?.dispatchTimestamp);
+        const terminalTime = parseTime(terminal?.at);
+        const stats: MeshTaskStats = {
+            taskId,
+            status,
+            dispatchedAt: dispatch?.first ?? queueEntry?.dispatchTimestamp ?? null,
+            terminalAt: terminal?.at ?? null,
+            terminalKind: terminal?.kind ?? null,
+            durationMs: null,
+            dispatchCount: dispatch?.count ?? 0,
+            requeueCount: queueEntry?.requeueCount ?? 0,
+        };
+        if (dispatchTime !== null && terminalTime !== null && terminalTime >= dispatchTime) {
+            stats.durationMs = terminalTime - dispatchTime;
+        } else if (isTerminalStatus) {
+            // Terminal task without complete dispatch+terminal ledger evidence:
+            // withhold numbers rather than estimate (M7 rule).
+            stats.incompleteEvidence = true;
+        }
+        return stats;
+    });
+}
+
+/** Mission rollup — derived from per-task stats, no stored aggregates. */
+export function computeMeshMissionStats(meshId: string, missionId: string): MeshMissionStats {
+    const tasks = computeMeshTaskStats(meshId, { missionId });
+    const stats: MeshMissionStats = {
+        missionId,
+        taskCount: tasks.length,
+        completed: 0,
+        failed: 0,
+        totalDurationMs: 0,
+        wallClockMs: null,
+        retries: 0,
+        incompleteTaskIds: [],
+    };
+    let firstDispatch: number | null = null;
+    let lastTerminal: number | null = null;
+    for (const task of tasks) {
+        if (task.status === 'completed') stats.completed += 1;
+        else if (task.status === 'failed') stats.failed += 1;
+        stats.retries += task.requeueCount;
+        if (task.incompleteEvidence) {
+            stats.incompleteTaskIds.push(task.taskId);
+            continue;
+        }
+        if (task.durationMs !== null) stats.totalDurationMs += task.durationMs;
+        const dispatchTime = parseTime(task.dispatchedAt);
+        const terminalTime = parseTime(task.terminalAt);
+        if (dispatchTime !== null && (firstDispatch === null || dispatchTime < firstDispatch)) firstDispatch = dispatchTime;
+        if (terminalTime !== null && (lastTerminal === null || terminalTime > lastTerminal)) lastTerminal = terminalTime;
+    }
+    if (firstDispatch !== null && lastTerminal !== null && lastTerminal >= firstDispatch) {
+        stats.wallClockMs = lastTerminal - firstDispatch;
+    }
+    return stats;
+}

package/src/mesh/mesh-unresolved-forward-outbox.ts

@@ -0,0 +1,185 @@
+// ---------------------------------------------------------------------------
+// mesh-unresolved-forward-outbox — durable retry for unresolved-delegate forwards
+// ---------------------------------------------------------------------------
+// A REMOTE worker daemon that is P2P-remote-controlled by a coordinator is NOT a
+// member of the coordinator's mesh — it has no local mesh record. So when its
+// completion event reaches setupMeshEventForwarding, resolveWorkerDelegateRouting
+// returns mesh_unresolved: the worker carries the coordinator daemon anchor but no
+// resolvable mesh id, so the normal queue-then-coordinator-pull path can't be used
+// (the coordinator's reconcile PHASE 1 only pulls mesh.nodes, and this worker is in
+// none of them — see docs/refactoring/2026-06-16-mesh-completion-polling-single-model.md).
+//
+// The ONLY delivery route for this case is a directed push to the coordinator
+// daemon (mesh_forward_event over P2P). Previously that push was fire-and-forget:
+// a single dispatchMeshCommand whose rejection was only logged, so one transient
+// P2P failure dropped the completion forever (delivery_unroutable).
+//
+// This outbox makes that push DURABLE without inventing a new persistence layer:
+// it reuses the existing mesh_pending_events SQLite table (the same store every
+// other mesh coordinator event is persisted to), under a synthetic, reserved
+// "mesh id" namespace so it never collides with a real mesh's queue. The worker's
+// reconcile tick (PHASE 0) peeks the outbox, pushes each entry to its coordinator,
+// and marks it drained (acked) ONLY on a successful push — a failed push leaves the
+// row undrained for the next tick. Entries that exceed a max age are expired so the
+// outbox can't grow unbounded when a coordinator stays permanently unreachable.
+//
+// Idempotency:
+//   - Enqueue is idempotent on the event fingerprint (mesh_pending_events has a
+//     UNIQUE (mesh_id, fingerprint) index + INSERT OR IGNORE), so the same
+//     completion fired twice on the worker queues once.
+//   - The receiver dedups independently: handleMeshForwardEvent → injectMeshSystemMessage
+//     → queuePendingMeshCoordinatorEvent recomputes the fingerprint and suppresses a
+//     duplicate, so an at-least-once retry that double-delivers is harmless.
+// ---------------------------------------------------------------------------
+
+import { randomUUID } from 'crypto';
+import { LOG } from '../logging/logger.js';
+import { MeshRuntimeStore } from './mesh-runtime-store.js';
+import { buildPendingEventFingerprint, type PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
+import { readNonEmptyString } from './mesh-events-utils.js';
+
+// Reserved synthetic mesh id for the worker-side unresolved-forward outbox. The `::`
+// and leading `__` make it impossible to collide with a real mesh id (which are
+// `mesh_*` / `mreg_*` slugs). Scoping rows by coordinator_daemon_id inside this one
+// namespace lets a single worker forward to multiple coordinators.
+export const UNRESOLVED_FORWARD_OUTBOX_MESH_ID = '__unresolved_forward_outbox__';
+
+// Entries older than this are expired (dropped) rather than retried forever — a
+// coordinator that has been unreachable this long is treated as gone. Generous
+// enough to ride out a long coordinator outage / restart, bounded enough that a
+// permanently-dead coordinator can't accumulate outbox rows indefinitely.
+const UNRESOLVED_FORWARD_MAX_AGE_MS = 30 * 60 * 1000; // 30 minutes
+
+export interface UnresolvedForwardEntry {
+    /** Row id in mesh_pending_events; pass back to ack/expire after a push attempt. */
+    id: string;
+    /** The coordinator daemon to push this event to (mesh_forward_event target). */
+    coordinatorDaemonId: string;
+    /** The flat payload to forward (already shaped for handleMeshForwardEvent). */
+    payload: Record<string, unknown>;
+    /** When the entry was first enqueued (epoch ms) — used for age-based expiry. */
+    queuedAt: number;
+}
+
+function getStore(): MeshRuntimeStore | undefined {
+    try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
+}
+
+/**
+ * Durably enqueue an unresolved-delegate forward for a coordinator daemon. The
+ * `forwardPayload` is the flat shape handleMeshForwardEvent reads on the coordinator.
+ * Idempotent on the event fingerprint. Returns true when a new row was written (or a
+ * duplicate was harmlessly ignored), false on a hard persistence failure.
+ */
+export function enqueueUnresolvedDelegateForward(
+    coordinatorDaemonId: string,
+    eventName: string,
+    forwardPayload: Record<string, unknown>,
+): boolean {
+    const target = readNonEmptyString(coordinatorDaemonId);
+    const event = readNonEmptyString(eventName);
+    if (!target || !event) return false;
+    const store = getStore();
+    if (!store) return false;
+
+    const queuedAt = Date.now();
+    // Reuse the standard pending-event fingerprint so enqueue is idempotent AND the
+    // receiver's own dedup keys on a comparable identity. We synthesise the minimal
+    // PendingMeshCoordinatorEvent shape buildPendingEventFingerprint needs.
+    const fingerprintSource: PendingMeshCoordinatorEvent = {
+        event,
+        meshId: UNRESOLVED_FORWARD_OUTBOX_MESH_ID,
+        nodeLabel: readNonEmptyString(forwardPayload.nodeId) || readNonEmptyString(forwardPayload.workspace) || 'unresolved-delegate',
+        nodeId: readNonEmptyString(forwardPayload.nodeId) || undefined,
+        workspace: readNonEmptyString(forwardPayload.workspace) || undefined,
+        metadataEvent: forwardPayload,
+        queuedAt,
+        targetCoordinatorDaemonId: target,
+    };
+    // Scope the fingerprint to the coordinator so two coordinators awaiting the same
+    // worker session don't collapse into one outbox row.
+    const fingerprint = `${target}::${buildPendingEventFingerprint(fingerprintSource)}`;
+
+    try {
+        const inserted = store.insertPendingEvent({
+            id: randomUUID(),
+            meshId: UNRESOLVED_FORWARD_OUTBOX_MESH_ID,
+            coordinatorDaemonId: target,
+            event,
+            // Store the flat forward payload + the queue timestamp so the retry tick can
+            // rebuild the push args and apply age-based expiry without a schema change.
+            payload: { forwardPayload, coordinatorDaemonId: target, queuedAt },
+            fingerprint,
+            queuedAt,
+        });
+        if (inserted) {
+            LOG.info('MeshEvents', `Durably queued unresolved-delegate ${event} for coordinator ${target} (outbox)`);
+        }
+        return true;
+    } catch (e: any) {
+        LOG.warn('MeshEvents', `Failed to persist unresolved-delegate forward to outbox: ${e?.message || e}`);
+        return false;
+    }
+}
+
+/** Peek (non-destructive) every undrained outbox entry across all coordinators. */
+export function peekUnresolvedDelegateForwards(): UnresolvedForwardEntry[] {
+    const store = getStore();
+    if (!store) return [];
+    let rows: Array<{ id: string; event: string; payload: unknown }>;
+    try {
+        rows = store.peekPendingEvents(UNRESOLVED_FORWARD_OUTBOX_MESH_ID);
+    } catch {
+        return [];
+    }
+    const out: UnresolvedForwardEntry[] = [];
+    for (const row of rows) {
+        const stored = row.payload && typeof row.payload === 'object' ? row.payload as Record<string, unknown> : {};
+        const coordinatorDaemonId = readNonEmptyString(stored.coordinatorDaemonId);
+        const forwardPayload = stored.forwardPayload && typeof stored.forwardPayload === 'object'
+            ? stored.forwardPayload as Record<string, unknown>
+            : undefined;
+        if (!coordinatorDaemonId || !forwardPayload) continue;
+        const queuedAt = typeof stored.queuedAt === 'number' ? stored.queuedAt : 0;
+        out.push({ id: row.id, coordinatorDaemonId, payload: forwardPayload, queuedAt });
+    }
+    return out;
+}
+
+/** Mark an outbox entry delivered (acked) after a successful push. */
+export function ackUnresolvedDelegateForward(id: string): void {
+    const store = getStore();
+    if (!store) return;
+    try { store.markPendingEventsDrainedById([id]); } catch { /* best-effort */ }
+}
+
+/**
+ * Drop outbox entries that have exceeded the max retry age. Returns the count
+ * expired so the caller can log a fail-loud trace (a dropped completion is a real
+ * loss; it must be visible, not silent).
+ */
+export function expireStaleUnresolvedDelegateForwards(nowMs: number = Date.now()): number {
+    const entries = peekUnresolvedDelegateForwards();
+    const staleIds = entries
+        .filter(e => e.queuedAt > 0 && nowMs - e.queuedAt >= UNRESOLVED_FORWARD_MAX_AGE_MS)
+        .map(e => e.id);
+    if (staleIds.length === 0) return 0;
+    const store = getStore();
+    if (!store) return 0;
+    try {
+        const removed = store.deletePendingEventsById(staleIds);
+        if (removed > 0) {
+            LOG.warn('MeshEvents', `Expired ${removed} unresolved-delegate forward(s) after ${Math.round(UNRESOLVED_FORWARD_MAX_AGE_MS / 60000)}m of failed retries — coordinator unreachable, completion dropped`);
+        }
+        return removed;
+    } catch {
+        return 0;
+    }
+}
+
+/** Test helper: purge the entire outbox. */
+export function __clearUnresolvedDelegateForwardOutboxForTests(): void {
+    const store = getStore();
+    if (!store) return;
+    try { store.clearPendingEventsForMesh(UNRESOLVED_FORWARD_OUTBOX_MESH_ID); } catch { /* nothing to clear */ }
+}

package/src/mesh/mesh-warmup-deadline.ts

@@ -0,0 +1,152 @@
+// ---------------------------------------------------------------------------
+// mesh-warmup-deadline — cold-open-aware deadline for mesh P2P dispatches
+// ---------------------------------------------------------------------------
+// A dependency-free leaf so the warmup deadline + its connection-probe resolution
+// can be shared by BOTH the dashboard git_status probe path (commands/router.ts)
+// and the general task-dispatch path (mesh/mesh-events-coordinator.ts) without an
+// import cycle (router ⇄ mesh-events). Pure except for timers + the injected
+// `isConnected` probe, so it is unit-testable under fake timers with no real WebRTC.
+// ---------------------------------------------------------------------------
+
+/**
+ * Await `work` under a warmup-aware deadline so a cold-open DataChannel handshake
+ * is NOT charged against the command response budget — the root cause of the
+ * "first mesh dispatch to a cold peer false-times-out, the warm retry succeeds"
+ * signature. Two budgets, switched by the live peer connection state:
+ *
+ *  - While `isConnected()` returns false the peer's channel is still opening; the
+ *    cold-open `connectTimeoutMs` budget applies. This phase is deliberately
+ *    generous because a TURN-relayed cross-machine handshake legitimately needs
+ *    many seconds — but a genuine connect *failure* is surfaced by `work`
+ *    rejecting on its own (the mesh manager fails the peer the instant its
+ *    PeerConnection state goes terminal), so a real failure is never masked for
+ *    the whole window.
+ *  - The first time `isConnected()` returns true the channel is warm; from that
+ *    instant the tight `responseTimeoutMs` governs how long the handler may take.
+ *    Warm-channel callers therefore see behavior identical to the old single
+ *    `Promise.race(work, responseTimeoutMs)`.
+ *
+ * Rejects with `Error('timeout')` when either budget is exhausted, mirroring the
+ * previous single-race contract. When no connection getter is wired callers must
+ * NOT pass `() => true` ("always warm") — that re-introduces the cold-open
+ * false-timeout. Use {@link resolveWarmupDeadlineOpts} which degrades conservatively.
+ */
+export function awaitWithWarmupDeadline<T>(
+    work: Promise<T>,
+    opts: {
+        isConnected: () => boolean;
+        connectTimeoutMs: number;
+        responseTimeoutMs: number;
+        pollIntervalMs?: number;
+    },
+): Promise<T> {
+    const pollMs = Math.max(1, Math.min(opts.pollIntervalMs ?? 200, opts.connectTimeoutMs));
+    return new Promise<T>((resolve, reject) => {
+        let done = false;
+        let poll: ReturnType<typeof setInterval> | undefined;
+        let responseTimer: ReturnType<typeof setTimeout> | undefined;
+        const startedAt = Date.now();
+        const cleanup = () => {
+            if (poll) { clearInterval(poll); poll = undefined; }
+            if (responseTimer) { clearTimeout(responseTimer); responseTimer = undefined; }
+        };
+        const settle = (fn: () => void) => {
+            if (done) return;
+            done = true;
+            cleanup();
+            fn();
+        };
+        // Arm the response deadline exactly once, the moment the channel is warm.
+        const armResponse = () => {
+            if (responseTimer || done) return;
+            responseTimer = setTimeout(
+                () => settle(() => reject(new Error('timeout'))),
+                opts.responseTimeoutMs,
+            );
+            if (typeof responseTimer.unref === 'function') responseTimer.unref();
+        };
+        const onPoll = () => {
+            if (done) return;
+            if (opts.isConnected()) {
+                if (poll) { clearInterval(poll); poll = undefined; }
+                armResponse();
+                return;
+            }
+            if (Date.now() - startedAt >= opts.connectTimeoutMs) {
+                settle(() => reject(new Error('timeout')));
+            }
+        };
+        if (opts.isConnected()) {
+            // Already warm (e.g. a retry over an open channel) — skip the warmup
+            // phase entirely and let the response deadline govern from t0.
+            armResponse();
+        } else {
+            poll = setInterval(onPoll, pollMs);
+            if (typeof poll.unref === 'function') poll.unref();
+        }
+        work.then(
+            (val) => settle(() => resolve(val)),
+            (err) => settle(() => reject(err)),
+        );
+    });
+}
+
+/** Minimal connection-state reader: a mesh peer snapshot stamps its live state on `.state`. */
+export function readWarmupConnectionState(connection: Record<string, unknown> | null | undefined): string | undefined {
+    const state = (connection as { state?: unknown } | null | undefined)?.state;
+    return typeof state === 'string' && state.length > 0 ? state : undefined;
+}
+
+export interface ResolvedWarmupDeadlineOpts {
+    isConnected: () => boolean;
+    connectTimeoutMs: number;
+    responseTimeoutMs: number;
+}
+
+/**
+ * Build {@link awaitWithWarmupDeadline} opts from an OPTIONAL live peer-connection
+ * probe, handling the missing-getter case fail-loud instead of silently degrading.
+ *
+ * When `getConnection` is wired the normal cold-open/warm split applies: the probe
+ * is consulted live and the channel is "warm" only once it reports `connected`.
+ *
+ * When `getConnection` is ABSENT the old call sites fell back to `() => true`
+ * ("always warm"), which charges a still-opening cold channel against the response
+ * budget and silently re-introduces the exact cold-open false-timeout the warmup
+ * deadline exists to prevent. Instead we degrade CONSERVATIVELY and FAIL LOUD:
+ *   - `onMissingGetter` is invoked so the caller can warn (the degrade is visible,
+ *     never silent) — keyed/throttled by the caller as it sees fit.
+ *   - the channel is treated as NOT observably warm (`isConnected: () => false`),
+ *     so the response deadline never arms early on an unobservable channel.
+ *   - the cold peer is granted the COMBINED connect+response window as one deadline,
+ *     so a slow-but-live cold open is never false-timed at the shorter response
+ *     budget. A genuinely hung dispatch still rejects when the combined window
+ *     lapses, and `work` rejecting on its own (a real transport failure) still
+ *     settles immediately.
+ *
+ * Note: a present getter that returns a non-`connected` snapshot (or `null` for an
+ * unknown peer) correctly yields `isConnected() === false` — i.e. the generous
+ * connect budget, never "always warm". Only a wholly absent getter degrades.
+ */
+export function resolveWarmupDeadlineOpts(opts: {
+    getConnection?: ((daemonId: string) => Record<string, unknown> | null) | undefined;
+    daemonId: string;
+    connectTimeoutMs: number;
+    responseTimeoutMs: number;
+    onMissingGetter?: (daemonId: string) => void;
+}): ResolvedWarmupDeadlineOpts {
+    const { getConnection, daemonId, connectTimeoutMs, responseTimeoutMs } = opts;
+    if (getConnection) {
+        return {
+            isConnected: () => readWarmupConnectionState(getConnection(daemonId)) === 'connected',
+            connectTimeoutMs,
+            responseTimeoutMs,
+        };
+    }
+    opts.onMissingGetter?.(daemonId);
+    return {
+        isConnected: () => false,
+        connectTimeoutMs: connectTimeoutMs + responseTimeoutMs,
+        responseTimeoutMs,
+    };
+}