@adhdev/daemon-core 0.9.82-rc.43 → 0.9.82-rc.431

package/src/mesh/mesh-scheduling-runtime.ts

@@ -0,0 +1,198 @@
+// Mesh scheduling-runtime view — a read-only, derived projection of how the queue
+// scheduler currently sees the mesh: which tie-break strategy is live, the global
+// parallel caps and how much of them is consumed, and per-node load / priority /
+// per-(node, provider) caps plus a structured "why this node can't take more write
+// work right now" reason set.
+//
+// This is OBSERVABILITY ONLY. It re-derives the same numbers the live claim path
+// (maybeAutoLaunchOneQueueSession → orderEligibleNodes → claimNextQueueTask) acts
+// on, but never mutates anything and never drives a scheduling decision. It is built
+// from the in-memory mesh config + the queue snapshot, so callers (mesh_status) get a
+// consistent picture without touching the SQLite claim transaction. Keeping the
+// derivation here — next to the constants it mirrors — means the exposed numbers and
+// the enforced numbers stay defined in one module family.
+
+import type { RepoMeshNodePolicy, RepoMeshSchedulingStrategy } from '../repo-mesh-types.js';
+import {
+    normalizeMeshSchedulingStrategy,
+    resolveMaxParallelTasks,
+    resolveMaxReadonlyParallelTasks,
+    resolveNodeSchedulingPriority,
+    resolveProviderMaxParallel,
+} from '../repo-mesh-types.js';
+import { normalizeMeshNodeId } from '@adhdev/mesh-shared';
+import type { MeshWorkQueueEntry } from './mesh-work-queue.js';
+import { isTaskReadonly } from './mesh-work-queue.js';
+
+/** Per-(node, provider) cap and its current consumption. */
+export interface MeshNodeProviderSchedulingRuntime {
+    providerType: string;
+    /** Declared maxParallel cap for this (node, provider). Omitted when uncapped. */
+    maxParallel?: number;
+    /** Active (status='assigned') tasks on this node claimed by this provider. */
+    activeAssigned: number;
+    /** True when activeAssigned has reached maxParallel (a further claim is refused). */
+    capReached: boolean;
+}
+
+/** Per-node scheduling-runtime projection. */
+export interface MeshNodeSchedulingRuntime {
+    nodeId: string;
+    /** Active (status='assigned') task count on this node — the least-loaded rank key. */
+    load: number;
+    /** Soft scheduling priority (PRIORITY rank key; higher = preferred). */
+    schedulingPriority: number;
+    /** Per-node concurrent-session cap, when configured. */
+    maxConcurrentSessions?: number;
+    /** Per-(node, provider) caps + consumption, when providerRoles declares any. */
+    providerRoles?: MeshNodeProviderSchedulingRuntime[];
+    /**
+     * True when the node currently cannot claim a NEW write (non-readonly) task —
+     * either the global write cap is exhausted or a node-local gate (active write
+     * assignment / a fully-consumed provider cap / session cap) blocks it.
+     */
+    capReached: boolean;
+    /** Structured reasons backing capReached (empty when the node can take work). */
+    capReasons: string[];
+}
+
+/** Mesh-level scheduling-runtime projection. */
+export interface MeshSchedulingRuntime {
+    /** Resolved tie-break strategy (defaults to 'first_eligible'). */
+    strategy: RepoMeshSchedulingStrategy;
+    /** Effective global write-task parallel cap (clamped). */
+    maxParallelTasks: number;
+    /** Derived read-only diagnosis cap (max(2, 2× write cap)) — mirrors the claim path. */
+    maxReadonlyParallelTasks: number;
+    /** Current global write (non-readonly) assigned-task load. */
+    activeWriteAssigned: number;
+    /** Current global read-only assigned-task load. */
+    activeReadonlyAssigned: number;
+    /** True when the global write cap is exhausted (no new write task can launch). */
+    globalWriteCapReached: boolean;
+    /** True when the read-only diagnosis cap is exhausted. */
+    globalReadonlyCapReached: boolean;
+    /** Per-node projections, in mesh config order. */
+    nodes: MeshNodeSchedulingRuntime[];
+}
+
+interface MeshLike {
+    policy?: { schedulingStrategy?: unknown; maxParallelTasks?: unknown } | null;
+    nodes?: Array<{ id?: string; nodeId?: string; node_id?: string; policy?: RepoMeshNodePolicy | null; isLocalWorktree?: boolean }> | null;
+}
+
+function isAssigned(task: MeshWorkQueueEntry): boolean {
+    return task.status === 'assigned';
+}
+
+/**
+ * Build the scheduling-runtime projection for a mesh from its config + a snapshot of
+ * its work queue. Pure and side-effect free — safe to call on any read path.
+ *
+ * The derivation deliberately mirrors maybeAutoLaunchOneQueueSession's gates so the
+ * exposed "capReached/capReasons" match why a real claim would be refused:
+ *   • global write cap   → activeWriteAssigned >= maxParallelTasks
+ *   • node write conflict→ a node already holding an assigned write task (worktree isolation)
+ *   • provider cap       → a (node, provider) at its declared maxParallel
+ * It does NOT re-evaluate eligibility/required-tags (those are task-specific); it only
+ * reports the capacity picture, which is what an operator needs to read load balance.
+ */
+export function buildMeshSchedulingRuntime(
+    mesh: MeshLike | null | undefined,
+    queue: MeshWorkQueueEntry[],
+): MeshSchedulingRuntime {
+    const strategy = normalizeMeshSchedulingStrategy(mesh?.policy?.schedulingStrategy);
+    const maxParallelTasks = resolveMaxParallelTasks(mesh?.policy?.maxParallelTasks);
+    // Read-only diagnoses are exempt from the write cap and get their own higher
+    // safety cap (multiplier × the write cap, floor 2) — identical to the claim
+    // path, via the shared helper so the two can never drift.
+    const maxReadonlyParallelTasks = resolveMaxReadonlyParallelTasks(maxParallelTasks);
+
+    const assignedTasks = (Array.isArray(queue) ? queue : []).filter(isAssigned);
+    const activeWriteAssigned = assignedTasks.filter(t => !isTaskReadonly(t)).length;
+    const activeReadonlyAssigned = assignedTasks.filter(isTaskReadonly).length;
+    const globalWriteCapReached = activeWriteAssigned >= maxParallelTasks;
+    const globalReadonlyCapReached = activeReadonlyAssigned >= maxReadonlyParallelTasks;
+
+    // Pre-bucket assigned tasks by node so per-node load and per-provider counts are
+    // a single pass rather than O(nodes × queue).
+    const writeAssignedByNode = new Map<string, number>();
+    const assignedByNode = new Map<string, number>();
+    const providerCountByNode = new Map<string, Map<string, number>>();
+    for (const task of assignedTasks) {
+        const nodeId = typeof task.assignedNodeId === 'string' ? task.assignedNodeId.trim() : '';
+        if (!nodeId) continue;
+        assignedByNode.set(nodeId, (assignedByNode.get(nodeId) ?? 0) + 1);
+        if (!isTaskReadonly(task)) {
+            writeAssignedByNode.set(nodeId, (writeAssignedByNode.get(nodeId) ?? 0) + 1);
+        }
+        const provider = typeof task.assignedProviderType === 'string' ? task.assignedProviderType : '';
+        if (provider) {
+            let byProvider = providerCountByNode.get(nodeId);
+            if (!byProvider) { byProvider = new Map(); providerCountByNode.set(nodeId, byProvider); }
+            byProvider.set(provider, (byProvider.get(provider) ?? 0) + 1);
+        }
+    }
+
+    const nodes: MeshNodeSchedulingRuntime[] = [];
+    for (const rawNode of (Array.isArray(mesh?.nodes) ? mesh!.nodes! : [])) {
+        const nodeId = normalizeMeshNodeId(rawNode);
+        if (!nodeId) continue;
+        const policy = (rawNode?.policy || undefined) as RepoMeshNodePolicy | undefined;
+        const load = assignedByNode.get(nodeId) ?? 0;
+        const schedulingPriority = resolveNodeSchedulingPriority(policy);
+
+        const capReasons: string[] = [];
+        if (globalWriteCapReached) capReasons.push('global_max_parallel_tasks_reached');
+        // Write isolation: a node already holding an assigned write task can't take another.
+        if ((writeAssignedByNode.get(nodeId) ?? 0) > 0) capReasons.push('node_has_active_assignment');
+
+        // Per-(node, provider) caps, with live consumption.
+        let providerRoles: MeshNodeProviderSchedulingRuntime[] | undefined;
+        const declaredRoles = Array.isArray(policy?.providerRoles) ? policy!.providerRoles! : [];
+        if (declaredRoles.length) {
+            const byProvider = providerCountByNode.get(nodeId);
+            providerRoles = [];
+            for (const role of declaredRoles) {
+                if (!role || typeof role !== 'object') continue;
+                const providerType = typeof role.providerType === 'string' ? role.providerType.trim() : '';
+                if (!providerType) continue;
+                const maxParallel = resolveProviderMaxParallel(policy, providerType);
+                const activeAssigned = byProvider?.get(providerType) ?? 0;
+                const capReached = maxParallel !== undefined && activeAssigned >= maxParallel;
+                providerRoles.push({
+                    providerType,
+                    ...(maxParallel !== undefined ? { maxParallel } : {}),
+                    activeAssigned,
+                    capReached,
+                });
+                if (capReached) capReasons.push(`max_provider_parallel_reached:${providerType}`);
+            }
+            if (!providerRoles.length) providerRoles = undefined;
+        }
+
+        const maxConcurrentSessions = Number(policy?.maxConcurrentSessions);
+        const hasSessionCap = Number.isFinite(maxConcurrentSessions) && maxConcurrentSessions >= 0;
+
+        nodes.push({
+            nodeId,
+            load,
+            schedulingPriority,
+            ...(hasSessionCap ? { maxConcurrentSessions: Math.floor(maxConcurrentSessions) } : {}),
+            ...(providerRoles ? { providerRoles } : {}),
+            capReached: capReasons.length > 0,
+            capReasons,
+        });
+    }
+
+    return {
+        strategy,
+        maxParallelTasks,
+        maxReadonlyParallelTasks,
+        activeWriteAssigned,
+        activeReadonlyAssigned,
+        globalWriteCapReached,
+        globalReadonlyCapReached,
+        nodes,
+    };
+}

package/src/mesh/mesh-task-inflight.ts

@@ -0,0 +1,70 @@
+/**
+ * CANON-IDENTITY single-flight dispatch guard.
+ *
+ * A queue task is double-dispatched into two worker sessions when a SECOND dispatch
+ * path opens after the first has already claimed + dispatched the task. The two
+ * observed paths:
+ *   - the idle-claim drain / auto-launch both funnel through tryAssignQueueTask,
+ *     which is serialized by the atomic claim transaction (claimNextQueueTask) — so
+ *     two concurrent claims can never both win; and
+ *   - the operator requeue tool (mesh_queue_requeue → requeueTask), which flips an
+ *     already-`assigned` row back to `pending` REGARDLESS of whether the worker
+ *     holding it is still generating. A requeue issued while the worker is mid-turn
+ *     re-opens the task for a SECOND session to claim — the live `ade8586d` race.
+ *
+ * The atomic claim already discriminates pending-vs-assigned, but it cannot tell a
+ * GENUINELY-in-flight assigned row (dispatched, worker generating) from a STALE
+ * assigned row (dead session, dispatch never confirmed) — and the requeue contract
+ * must still reopen the stale case. This module is that discriminator: a task id is
+ * registered here ONLY by the dispatch path at the moment it hands the claimed task
+ * to a transport, and cleared the moment the task leaves the `assigned` state
+ * (terminal completion/failure, dispatch-failure requeue, cancel, reclaim, or a
+ * forced requeue). requeueTask consults it and refuses (no-op) to reopen a task that
+ * is still in-flight unless the caller passes `force`.
+ *
+ * Dependency-free leaf (no imports) so both mesh-queue-assignment (begin) and
+ * mesh-work-queue (clear + the requeue guard) can import it without a cycle. The key
+ * is `${meshId}::${taskId}`; a task id is a single-form UUID, so no daemon-id
+ * normalization is needed on the key itself.
+ */
+
+const inFlight = new Set<string>();
+
+function key(meshId: string, taskId: string): string {
+    return `${meshId}::${taskId}`;
+}
+
+/**
+ * Mark a task as actively dispatched/generating (in-flight). Called by the dispatch
+ * path right after a successful claim, before/as the task is handed to a transport.
+ * Returns true when this call transitioned the task into the in-flight set, false
+ * when it was already in-flight (a redundant begin — the caller may treat that as a
+ * signal that a dispatch is already live for this task).
+ */
+export function beginTaskDispatchInFlight(meshId: string, taskId: string): boolean {
+    if (!meshId || !taskId) return false;
+    const k = key(meshId, taskId);
+    if (inFlight.has(k)) return false;
+    inFlight.add(k);
+    return true;
+}
+
+/** True while a task is actively dispatched/generating (registered by the dispatch
+ *  path and not yet cleared by a terminal/requeue/cancel transition). */
+export function isTaskDispatchInFlight(meshId: string, taskId: string): boolean {
+    if (!meshId || !taskId) return false;
+    return inFlight.has(key(meshId, taskId));
+}
+
+/** Clear a task's in-flight mark. Called whenever the task leaves the `assigned`
+ *  state (terminal completion/failure, dispatch-failure requeue, cancel, reclaim, or
+ *  a forced requeue). Idempotent / safe to call when the task was never in-flight. */
+export function endTaskDispatchInFlight(meshId: string, taskId: string): void {
+    if (!meshId || !taskId) return;
+    inFlight.delete(key(meshId, taskId));
+}
+
+/** Test-only: drop all in-flight marks so a fresh test starts from a clean guard. */
+export function __resetTaskDispatchInFlightForTests(): void {
+    inFlight.clear();
+}

package/src/mesh/mesh-task-stats.ts

@@ -0,0 +1,161 @@
+/**
+ * M7: Operational stats (time/attempts) derived from existing truth.
+ *
+ * No cost/token accounting — ADHDev observes PTY/CDP and cannot see API
+ * tokens (explicit non-goal). Everything here is derived at query time from
+ * the SQLite ledger and queue rows; there is no separate aggregate table.
+ * Tasks with missing ledger evidence report incompleteEvidence instead of
+ * estimated numbers.
+ */
+
+import { readLedgerEntries } from './mesh-ledger.js';
+import type { MeshLedgerEntry } from './mesh-ledger.js';
+import { getQueue } from './mesh-work-queue.js';
+
+export interface MeshTaskStats {
+    taskId: string;
+    status: string;
+    dispatchedAt: string | null;
+    terminalAt: string | null;
+    terminalKind: 'task_completed' | 'task_failed' | null;
+    /** dispatched → terminal wall clock; null when evidence is incomplete. */
+    durationMs: number | null;
+    /** Number of task_dispatched ledger entries observed for this task. */
+    dispatchCount: number;
+    /** Queue requeueCount (0 when the row is gone or never requeued). */
+    requeueCount: number;
+    /** True when dispatch or terminal ledger evidence is missing — numbers are withheld, never estimated. */
+    incompleteEvidence?: true;
+}
+
+export interface MeshMissionStats {
+    missionId: string;
+    taskCount: number;
+    completed: number;
+    failed: number;
+    /** Sum of per-task durations with complete evidence. */
+    totalDurationMs: number;
+    /** First dispatch → last terminal across the mission's tasks; null without complete endpoints. */
+    wallClockMs: number | null;
+    /** Total requeue attempts across the mission's tasks. */
+    retries: number;
+    /** Task ids whose ledger evidence was incomplete (excluded from sums). */
+    incompleteTaskIds: string[];
+}
+
+function readPayloadTaskId(entry: MeshLedgerEntry): string {
+    const value = entry.payload?.taskId;
+    return typeof value === 'string' && value.trim() ? value.trim() : '';
+}
+
+function parseTime(value: string | null | undefined): number | null {
+    if (!value) return null;
+    const parsed = new Date(value).getTime();
+    return Number.isFinite(parsed) ? parsed : null;
+}
+
+/**
+ * Compute per-task stats from ledger entries. Scans a bounded tail window —
+ * stats are an operational view of recent work, not a full historical report.
+ */
+export function computeMeshTaskStats(meshId: string, opts?: { taskIds?: string[]; missionId?: string; tail?: number }): MeshTaskStats[] {
+    const queue = getQueue(meshId);
+    const queueById = new Map(queue.map(task => [task.id, task]));
+
+    let targetIds: string[];
+    if (opts?.taskIds?.length) {
+        targetIds = [...new Set(opts.taskIds)];
+    } else if (opts?.missionId) {
+        targetIds = queue.filter(task => task.missionId === opts.missionId).map(task => task.id);
+    } else {
+        targetIds = queue.map(task => task.id);
+    }
+    if (targetIds.length === 0) return [];
+    const targetSet = new Set(targetIds);
+
+    const entries = readLedgerEntries(meshId, { tail: opts?.tail ?? 1000 });
+    const dispatches = new Map<string, { first: string; count: number }>();
+    const terminals = new Map<string, { at: string; kind: 'task_completed' | 'task_failed' }>();
+    for (const entry of entries) {
+        const taskId = readPayloadTaskId(entry);
+        if (!taskId || !targetSet.has(taskId)) continue;
+        if (entry.kind === 'task_dispatched') {
+            const existing = dispatches.get(taskId);
+            if (existing) existing.count += 1;
+            else dispatches.set(taskId, { first: entry.timestamp, count: 1 });
+        } else if (entry.kind === 'task_completed' || entry.kind === 'task_failed') {
+            // Last terminal wins (requeued tasks can have multiple terminals).
+            terminals.set(taskId, { at: entry.timestamp, kind: entry.kind });
+        }
+    }
+
+    return targetIds.map(taskId => {
+        const queueEntry = queueById.get(taskId);
+        const dispatch = dispatches.get(taskId);
+        const terminal = terminals.get(taskId);
+        // Direct dispatches (mesh_send_task) have no work-queue row, so queueEntry is undefined.
+        // Derive a terminal status from the attributed terminal ledger entry instead of reporting
+        // status='unknown' — without this a completed direct task showed unknown + terminalKind=null
+        // even though its task_completed event fired (see mesh-events-coordinator Fix B attribution).
+        const status = queueEntry?.status
+            ?? (terminal
+                ? (terminal.kind === 'task_completed' ? 'completed' : 'failed')
+                : 'unknown');
+        const isTerminalStatus = status === 'completed' || status === 'failed' || status === 'cancelled';
+        const dispatchTime = parseTime(dispatch?.first ?? queueEntry?.dispatchTimestamp);
+        const terminalTime = parseTime(terminal?.at);
+        const stats: MeshTaskStats = {
+            taskId,
+            status,
+            dispatchedAt: dispatch?.first ?? queueEntry?.dispatchTimestamp ?? null,
+            terminalAt: terminal?.at ?? null,
+            terminalKind: terminal?.kind ?? null,
+            durationMs: null,
+            dispatchCount: dispatch?.count ?? 0,
+            requeueCount: queueEntry?.requeueCount ?? 0,
+        };
+        if (dispatchTime !== null && terminalTime !== null && terminalTime >= dispatchTime) {
+            stats.durationMs = terminalTime - dispatchTime;
+        } else if (isTerminalStatus) {
+            // Terminal task without complete dispatch+terminal ledger evidence:
+            // withhold numbers rather than estimate (M7 rule).
+            stats.incompleteEvidence = true;
+        }
+        return stats;
+    });
+}
+
+/** Mission rollup — derived from per-task stats, no stored aggregates. */
+export function computeMeshMissionStats(meshId: string, missionId: string): MeshMissionStats {
+    const tasks = computeMeshTaskStats(meshId, { missionId });
+    const stats: MeshMissionStats = {
+        missionId,
+        taskCount: tasks.length,
+        completed: 0,
+        failed: 0,
+        totalDurationMs: 0,
+        wallClockMs: null,
+        retries: 0,
+        incompleteTaskIds: [],
+    };
+    let firstDispatch: number | null = null;
+    let lastTerminal: number | null = null;
+    for (const task of tasks) {
+        if (task.status === 'completed') stats.completed += 1;
+        else if (task.status === 'failed') stats.failed += 1;
+        stats.retries += task.requeueCount;
+        if (task.incompleteEvidence) {
+            stats.incompleteTaskIds.push(task.taskId);
+            continue;
+        }
+        if (task.durationMs !== null) stats.totalDurationMs += task.durationMs;
+        const dispatchTime = parseTime(task.dispatchedAt);
+        const terminalTime = parseTime(task.terminalAt);
+        if (dispatchTime !== null && (firstDispatch === null || dispatchTime < firstDispatch)) firstDispatch = dispatchTime;
+        if (terminalTime !== null && (lastTerminal === null || terminalTime > lastTerminal)) lastTerminal = terminalTime;
+    }
+    if (firstDispatch !== null && lastTerminal !== null && lastTerminal >= firstDispatch) {
+        stats.wallClockMs = lastTerminal - firstDispatch;
+    }
+    return stats;
+}

package/src/mesh/mesh-unresolved-forward-outbox.ts

@@ -0,0 +1,185 @@
+// ---------------------------------------------------------------------------
+// mesh-unresolved-forward-outbox — durable retry for unresolved-delegate forwards
+// ---------------------------------------------------------------------------
+// A REMOTE worker daemon that is P2P-remote-controlled by a coordinator is NOT a
+// member of the coordinator's mesh — it has no local mesh record. So when its
+// completion event reaches setupMeshEventForwarding, resolveWorkerDelegateRouting
+// returns mesh_unresolved: the worker carries the coordinator daemon anchor but no
+// resolvable mesh id, so the normal queue-then-coordinator-pull path can't be used
+// (the coordinator's reconcile PHASE 1 only pulls mesh.nodes, and this worker is in
+// none of them — see docs/refactoring/2026-06-16-mesh-completion-polling-single-model.md).
+//
+// The ONLY delivery route for this case is a directed push to the coordinator
+// daemon (mesh_forward_event over P2P). Previously that push was fire-and-forget:
+// a single dispatchMeshCommand whose rejection was only logged, so one transient
+// P2P failure dropped the completion forever (delivery_unroutable).
+//
+// This outbox makes that push DURABLE without inventing a new persistence layer:
+// it reuses the existing mesh_pending_events SQLite table (the same store every
+// other mesh coordinator event is persisted to), under a synthetic, reserved
+// "mesh id" namespace so it never collides with a real mesh's queue. The worker's
+// reconcile tick (PHASE 0) peeks the outbox, pushes each entry to its coordinator,
+// and marks it drained (acked) ONLY on a successful push — a failed push leaves the
+// row undrained for the next tick. Entries that exceed a max age are expired so the
+// outbox can't grow unbounded when a coordinator stays permanently unreachable.
+//
+// Idempotency:
+//   - Enqueue is idempotent on the event fingerprint (mesh_pending_events has a
+//     UNIQUE (mesh_id, fingerprint) index + INSERT OR IGNORE), so the same
+//     completion fired twice on the worker queues once.
+//   - The receiver dedups independently: handleMeshForwardEvent → injectMeshSystemMessage
+//     → queuePendingMeshCoordinatorEvent recomputes the fingerprint and suppresses a
+//     duplicate, so an at-least-once retry that double-delivers is harmless.
+// ---------------------------------------------------------------------------
+
+import { randomUUID } from 'crypto';
+import { LOG } from '../logging/logger.js';
+import { MeshRuntimeStore } from './mesh-runtime-store.js';
+import { buildPendingEventFingerprint, type PendingMeshCoordinatorEvent } from './mesh-events-pending.js';
+import { readNonEmptyString } from './mesh-events-utils.js';
+
+// Reserved synthetic mesh id for the worker-side unresolved-forward outbox. The `::`
+// and leading `__` make it impossible to collide with a real mesh id (which are
+// `mesh_*` / `mreg_*` slugs). Scoping rows by coordinator_daemon_id inside this one
+// namespace lets a single worker forward to multiple coordinators.
+export const UNRESOLVED_FORWARD_OUTBOX_MESH_ID = '__unresolved_forward_outbox__';
+
+// Entries older than this are expired (dropped) rather than retried forever — a
+// coordinator that has been unreachable this long is treated as gone. Generous
+// enough to ride out a long coordinator outage / restart, bounded enough that a
+// permanently-dead coordinator can't accumulate outbox rows indefinitely.
+const UNRESOLVED_FORWARD_MAX_AGE_MS = 30 * 60 * 1000; // 30 minutes
+
+export interface UnresolvedForwardEntry {
+    /** Row id in mesh_pending_events; pass back to ack/expire after a push attempt. */
+    id: string;
+    /** The coordinator daemon to push this event to (mesh_forward_event target). */
+    coordinatorDaemonId: string;
+    /** The flat payload to forward (already shaped for handleMeshForwardEvent). */
+    payload: Record<string, unknown>;
+    /** When the entry was first enqueued (epoch ms) — used for age-based expiry. */
+    queuedAt: number;
+}
+
+function getStore(): MeshRuntimeStore | undefined {
+    try { return MeshRuntimeStore.getInstance(); } catch { return undefined; }
+}
+
+/**
+ * Durably enqueue an unresolved-delegate forward for a coordinator daemon. The
+ * `forwardPayload` is the flat shape handleMeshForwardEvent reads on the coordinator.
+ * Idempotent on the event fingerprint. Returns true when a new row was written (or a
+ * duplicate was harmlessly ignored), false on a hard persistence failure.
+ */
+export function enqueueUnresolvedDelegateForward(
+    coordinatorDaemonId: string,
+    eventName: string,
+    forwardPayload: Record<string, unknown>,
+): boolean {
+    const target = readNonEmptyString(coordinatorDaemonId);
+    const event = readNonEmptyString(eventName);
+    if (!target || !event) return false;
+    const store = getStore();
+    if (!store) return false;
+
+    const queuedAt = Date.now();
+    // Reuse the standard pending-event fingerprint so enqueue is idempotent AND the
+    // receiver's own dedup keys on a comparable identity. We synthesise the minimal
+    // PendingMeshCoordinatorEvent shape buildPendingEventFingerprint needs.
+    const fingerprintSource: PendingMeshCoordinatorEvent = {
+        event,
+        meshId: UNRESOLVED_FORWARD_OUTBOX_MESH_ID,
+        nodeLabel: readNonEmptyString(forwardPayload.nodeId) || readNonEmptyString(forwardPayload.workspace) || 'unresolved-delegate',
+        nodeId: readNonEmptyString(forwardPayload.nodeId) || undefined,
+        workspace: readNonEmptyString(forwardPayload.workspace) || undefined,
+        metadataEvent: forwardPayload,
+        queuedAt,
+        targetCoordinatorDaemonId: target,
+    };
+    // Scope the fingerprint to the coordinator so two coordinators awaiting the same
+    // worker session don't collapse into one outbox row.
+    const fingerprint = `${target}::${buildPendingEventFingerprint(fingerprintSource)}`;
+
+    try {
+        const inserted = store.insertPendingEvent({
+            id: randomUUID(),
+            meshId: UNRESOLVED_FORWARD_OUTBOX_MESH_ID,
+            coordinatorDaemonId: target,
+            event,
+            // Store the flat forward payload + the queue timestamp so the retry tick can
+            // rebuild the push args and apply age-based expiry without a schema change.
+            payload: { forwardPayload, coordinatorDaemonId: target, queuedAt },
+            fingerprint,
+            queuedAt,
+        });
+        if (inserted) {
+            LOG.info('MeshEvents', `Durably queued unresolved-delegate ${event} for coordinator ${target} (outbox)`);
+        }
+        return true;
+    } catch (e: any) {
+        LOG.warn('MeshEvents', `Failed to persist unresolved-delegate forward to outbox: ${e?.message || e}`);
+        return false;
+    }
+}
+
+/** Peek (non-destructive) every undrained outbox entry across all coordinators. */
+export function peekUnresolvedDelegateForwards(): UnresolvedForwardEntry[] {
+    const store = getStore();
+    if (!store) return [];
+    let rows: Array<{ id: string; event: string; payload: unknown }>;
+    try {
+        rows = store.peekPendingEvents(UNRESOLVED_FORWARD_OUTBOX_MESH_ID);
+    } catch {
+        return [];
+    }
+    const out: UnresolvedForwardEntry[] = [];
+    for (const row of rows) {
+        const stored = row.payload && typeof row.payload === 'object' ? row.payload as Record<string, unknown> : {};
+        const coordinatorDaemonId = readNonEmptyString(stored.coordinatorDaemonId);
+        const forwardPayload = stored.forwardPayload && typeof stored.forwardPayload === 'object'
+            ? stored.forwardPayload as Record<string, unknown>
+            : undefined;
+        if (!coordinatorDaemonId || !forwardPayload) continue;
+        const queuedAt = typeof stored.queuedAt === 'number' ? stored.queuedAt : 0;
+        out.push({ id: row.id, coordinatorDaemonId, payload: forwardPayload, queuedAt });
+    }
+    return out;
+}
+
+/** Mark an outbox entry delivered (acked) after a successful push. */
+export function ackUnresolvedDelegateForward(id: string): void {
+    const store = getStore();
+    if (!store) return;
+    try { store.markPendingEventsDrainedById([id]); } catch { /* best-effort */ }
+}
+
+/**
+ * Drop outbox entries that have exceeded the max retry age. Returns the count
+ * expired so the caller can log a fail-loud trace (a dropped completion is a real
+ * loss; it must be visible, not silent).
+ */
+export function expireStaleUnresolvedDelegateForwards(nowMs: number = Date.now()): number {
+    const entries = peekUnresolvedDelegateForwards();
+    const staleIds = entries
+        .filter(e => e.queuedAt > 0 && nowMs - e.queuedAt >= UNRESOLVED_FORWARD_MAX_AGE_MS)
+        .map(e => e.id);
+    if (staleIds.length === 0) return 0;
+    const store = getStore();
+    if (!store) return 0;
+    try {
+        const removed = store.deletePendingEventsById(staleIds);
+        if (removed > 0) {
+            LOG.warn('MeshEvents', `Expired ${removed} unresolved-delegate forward(s) after ${Math.round(UNRESOLVED_FORWARD_MAX_AGE_MS / 60000)}m of failed retries — coordinator unreachable, completion dropped`);
+        }
+        return removed;
+    } catch {
+        return 0;
+    }
+}
+
+/** Test helper: purge the entire outbox. */
+export function __clearUnresolvedDelegateForwardOutboxForTests(): void {
+    const store = getStore();
+    if (!store) return;
+    try { store.clearPendingEventsForMesh(UNRESOLVED_FORWARD_OUTBOX_MESH_ID); } catch { /* nothing to clear */ }
+}