@adhdev/daemon-core 0.9.82-rc.256 → 0.9.82-rc.258

package/src/mesh/mesh-events.ts

@@ -11,6 +11,7 @@ export {
     drainPendingMeshCoordinatorEvents,
     getPendingMeshCoordinatorEvents,
     clearPendingMeshCoordinatorEvents,
+    markMeshCoordinatorEventDirectDelivered,
 } from './mesh-events-pending.js';
 
 export {
@@ -25,4 +26,5 @@ export {
     setupMeshEventForwarding,
     isMeshCoordinatorEvent,
     __resetIdleAutoFastForwardForTests,
+    __resetMeshWorkspaceCacheForTests,
 } from './mesh-events-coordinator.js';

package/src/mesh/mesh-ledger.ts

@@ -40,6 +40,7 @@ export type MeshLedgerKind =
     | 'ledger_replicated'
     | 'ledger_reconciled'
     | 'direct_fast_forward'
+    | 'delivery_unroutable'
     ;
 
 export interface MeshLedgerEntry {

package/src/mesh/mesh-routing.ts

@@ -0,0 +1,272 @@
+import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
+import { getActiveDirectDispatches } from './mesh-work-queue.js';
+import { hasUnterminalDirectDispatchLedgerEntry } from './mesh-events-stale.js';
+import { appendLedgerEntry, readLedgerEntries } from './mesh-ledger.js';
+import { LOG } from '../logging/logger.js';
+import { readNonEmptyString } from './mesh-events-utils.js';
+
+// ---------------------------------------------------------------------------
+// R1: single-source coordinator routing resolution
+//
+// Worker→node→mesh→coordinator routing used to be reconstructed inline inside
+// setupMeshEventForwarding by reading six settings "stamp" fields
+// (meshNodeFor / meshNodeId / meshCoordinatorFor / meshCoordinatorDaemonId /
+// meshCoordinatorNodeId / launchedByCoordinator) plus two ledger queries plus a
+// workspace fallback, scattered across ~50 lines. Any missing stamp silently
+// changed the routing decision (delegate dropped, mesh unresolved) — design
+// vulnerability #2.
+//
+// resolveWorkerDelegateRouting() is the ONE place that interprets those inputs.
+// The stamps are inputs *internal to this function*; no external branch reads
+// them to make a routing decision. The forwarder consumes the typed result only.
+// ---------------------------------------------------------------------------
+
+export type WorkerDelegateRejectionReason =
+    | 'not_cli'
+    | 'no_workspace'
+    | 'no_worker_envelope'
+    | 'coordinator_not_dispatch_target'
+    | 'mesh_unresolved';
+
+export interface WorkerDelegateRouting {
+    /** True when the source session is a mesh worker whose events must route to a coordinator. */
+    isDelegate: boolean;
+    /** Resolved mesh id (runtime stamp, direct-dispatch recovery, or workspace lookup). */
+    meshId: string;
+    /** Resolved node id within the mesh, or '' when it cannot be determined. */
+    nodeId: string;
+    /** Human-readable node label for the coordinator system message. */
+    nodeLabel: string;
+    /** Coordinator daemon anchor (prefixed id) when the worker carries one; '' otherwise. */
+    coordinatorDaemonId: string;
+    /** Source session workspace (best-effort; '' when the session/workspace is missing). */
+    workspace: string;
+    /** Source session id echoed back, so diagnostics can name the dropped event's origin. */
+    sessionId: string;
+    /** Why routing was rejected (isDelegate=false), for fail-loud diagnostics (R4). */
+    rejectionReason?: WorkerDelegateRejectionReason;
+}
+
+interface ResolveDeps {
+    /** Resolve a mesh by id (local config + inline cache). */
+    getMeshById: (meshId: string) => any | undefined;
+    /** Resolve a mesh by workspace path (cached repo lookup). */
+    getMeshByWorkspace: (workspace: string) => any | undefined;
+}
+
+function readSettings(state: any): Record<string, unknown> {
+    return state?.settings && typeof state.settings === 'object'
+        ? state.settings as Record<string, unknown>
+        : {};
+}
+
+/**
+ * Resolve how a source CLI session's mesh events should route to a coordinator.
+ *
+ * This is the single authoritative interpretation of the worker envelope. It folds
+ * together every signal that previously lived in inline branches:
+ *  - runtime stamps (meshNodeFor / meshNodeId)
+ *  - the coordinator anchor (meshCoordinatorDaemonId) and launch marker (launchedByCoordinator,
+ *    meshCoordinatorNodeId) — any one proves delegation even if the node/mesh stamp was dropped
+ *  - direct-dispatch recovery: a coordinator session that is itself a dispatch target
+ *  - workspace→mesh fallback when no runtime mesh id survived
+ *
+ * Returns isDelegate=false (with a rejectionReason) instead of throwing, so the caller
+ * can either ignore non-delegate events or surface a diagnostic.
+ */
+export function resolveWorkerDelegateRouting(
+    components: DaemonComponents,
+    instanceId: string,
+    deps: ResolveDeps,
+): WorkerDelegateRouting {
+    const sessionId = readNonEmptyString(instanceId);
+    let workspace = '';
+    let coordinatorDaemonId = '';
+    const reject = (rejectionReason: WorkerDelegateRejectionReason): WorkerDelegateRouting => ({
+        isDelegate: false,
+        meshId: '',
+        nodeId: '',
+        nodeLabel: '',
+        coordinatorDaemonId,
+        workspace,
+        sessionId,
+        rejectionReason,
+    });
+
+    const sourceInstance = components.instanceManager.getInstance(instanceId);
+    if (!sourceInstance || sourceInstance.category !== 'cli') return reject('not_cli');
+
+    const state = sourceInstance.getState();
+    workspace = readNonEmptyString(state.workspace);
+    if (!workspace) return reject('no_workspace');
+
+    const settings = readSettings(state);
+    coordinatorDaemonId = readNonEmptyString(settings.meshCoordinatorDaemonId);
+
+    // A coordinator session (meshCoordinatorFor set) is only treated as a worker delegate
+    // when it is itself the target of an active direct dispatch — otherwise its own events
+    // must not be routed back to a coordinator (it IS the coordinator).
+    const coordinatorMeshId = readNonEmptyString(settings.meshCoordinatorFor);
+    let meshIdFromDirectDispatch = '';
+    if (coordinatorMeshId) {
+        let hasActiveDispatch = false;
+        try {
+            hasActiveDispatch =
+                getActiveDirectDispatches(coordinatorMeshId).some(d => d.sessionId === instanceId)
+                || hasUnterminalDirectDispatchLedgerEntry(coordinatorMeshId, instanceId);
+        } catch { /* best-effort */ }
+        if (!hasActiveDispatch) return reject('coordinator_not_dispatch_target');
+        meshIdFromDirectDispatch = coordinatorMeshId;
+    }
+
+    const meshIdFromRuntime = readNonEmptyString(settings.meshNodeFor) || meshIdFromDirectDispatch;
+
+    // Worker-envelope proof of delegation. A worker can arrive carrying only the routing
+    // anchor (meshCoordinatorDaemonId) without meshNodeFor — e.g. when the node/mesh stamp
+    // was dropped on a direct dispatch or relaunch. Treat any envelope marker as proof and
+    // recover the mesh id by workspace when the runtime id is absent.
+    const hasWorkerEnvelope = Boolean(
+        meshIdFromRuntime
+        || settings.launchedByCoordinator
+        || coordinatorDaemonId
+        || readNonEmptyString(settings.meshCoordinatorNodeId),
+    );
+    if (!hasWorkerEnvelope) return reject('no_worker_envelope');
+
+    const mesh = meshIdFromRuntime ? deps.getMeshById(meshIdFromRuntime) : deps.getMeshByWorkspace(workspace);
+    const meshId = meshIdFromRuntime || readNonEmptyString(mesh?.id);
+    if (!meshId) return reject('mesh_unresolved');
+
+    const targetNode = mesh?.nodes?.find((n: any) => n.workspace === workspace);
+    const runtimeNodeId = readNonEmptyString(settings.meshNodeId);
+    const nodeId = readNonEmptyString(targetNode?.id) || runtimeNodeId;
+    const nodeLabel = targetNode
+        ? `Node '${targetNode.id}'`
+        : runtimeNodeId
+            ? `Node '${runtimeNodeId}'`
+            : `Agent at ${workspace}`;
+
+    return {
+        isDelegate: true,
+        meshId,
+        nodeId,
+        nodeLabel,
+        coordinatorDaemonId,
+        workspace,
+        sessionId,
+    };
+}
+
+// ---------------------------------------------------------------------------
+// R4: fail-loud routing diagnostics
+//
+// A rejection is only worth surfacing when the session DID present a worker
+// envelope but routing still couldn't complete (`mesh_unresolved`). Before R4
+// that event was dropped silently — setupMeshEventForwarding returned, the
+// completion never reached a coordinator and never landed in the pending queue,
+// and there was no trace of why. The benign rejections (not_cli / no_workspace /
+// no_worker_envelope / coordinator_not_dispatch_target) are ordinary non-delegate
+// traffic and must NOT spam the ledger.
+//
+// The diagnostic is keyed under a single shared stream id so all unroutable drops
+// across meshes are discoverable in one place (the dropped event has no resolvable
+// mesh by definition). A short per-(session+event) dedup window keeps a chatty
+// session from flooding the ledger.
+// ---------------------------------------------------------------------------
+
+/** Ledger stream that collects delivery_unroutable diagnostics (no real mesh id exists for them). */
+export const UNROUTABLE_DIAGNOSTIC_STREAM = '__unroutable__';
+
+const UNROUTABLE_DIAGNOSTIC_DEDUP_MS = 60 * 1000;
+const recentUnroutableDiagnostics = new Map<string, number>();
+
+export function __resetUnroutableDiagnosticsForTests(): void {
+    recentUnroutableDiagnostics.clear();
+}
+
+/** True only for rejections that represent a silently-dropped delegate event worth a diagnostic. */
+export function isUnroutableDelegateRejection(routing: WorkerDelegateRouting): boolean {
+    return !routing.isDelegate && routing.rejectionReason === 'mesh_unresolved';
+}
+
+/**
+ * R4: record that a worker event with a valid envelope could not be routed to a coordinator.
+ * Writes a `delivery_unroutable` ledger entry (deduped within a short window) so operators see
+ * WHY a completion never arrived instead of it vanishing. No-op for non-diagnostic rejections.
+ */
+export function recordUnroutableDelegateEvent(routing: WorkerDelegateRouting, eventName: string): boolean {
+    if (!isUnroutableDelegateRejection(routing)) return false;
+
+    const dedupKey = `${routing.sessionId}::${eventName}::${routing.workspace}`;
+    const now = Date.now();
+    const last = recentUnroutableDiagnostics.get(dedupKey);
+    if (last !== undefined && now - last < UNROUTABLE_DIAGNOSTIC_DEDUP_MS) return false;
+    recentUnroutableDiagnostics.set(dedupKey, now);
+    // Opportunistic sweep so the map can't grow unbounded.
+    if (recentUnroutableDiagnostics.size > 256) {
+        for (const [key, ts] of recentUnroutableDiagnostics) {
+            if (now - ts >= UNROUTABLE_DIAGNOSTIC_DEDUP_MS) recentUnroutableDiagnostics.delete(key);
+        }
+    }
+
+    try {
+        appendLedgerEntry(UNROUTABLE_DIAGNOSTIC_STREAM, {
+            kind: 'delivery_unroutable',
+            sessionId: routing.sessionId || undefined,
+            payload: {
+                event: eventName,
+                reason: routing.rejectionReason,
+                workspace: routing.workspace || undefined,
+                coordinatorDaemonId: routing.coordinatorDaemonId || undefined,
+                detail: 'Worker envelope was present but no mesh could be resolved; the event could not be routed to a coordinator.',
+            },
+        });
+        LOG.warn('MeshEvents', `delivery_unroutable: ${eventName} from session ${routing.sessionId || '(unknown)'} at ${routing.workspace || '(no workspace)'} — envelope present but mesh unresolved`);
+        return true;
+    } catch (e: any) {
+        LOG.warn('MeshEvents', `Failed to record delivery_unroutable diagnostic: ${e?.message || e}`);
+        return false;
+    }
+}
+
+export interface UnroutableDeliveryDiagnostic {
+    timestamp: string;
+    event: string;
+    sessionId?: string;
+    workspace?: string;
+    coordinatorDaemonId?: string;
+}
+
+/**
+ * R4 visibility: recent delivery_unroutable diagnostics, newest first. Surfaced in mesh_status
+ * so an operator/coordinator can see that completions were dropped (envelope present, mesh
+ * unresolved) rather than the drops being invisible in a ledger nobody reads. The diagnostics
+ * live in a single shared stream because an unroutable event has no resolvable mesh.
+ */
+export function getRecentUnroutableDeliveries(opts?: { sinceMs?: number; limit?: number }): UnroutableDeliveryDiagnostic[] {
+    const sinceMs = opts?.sinceMs ?? 60 * 60 * 1000; // last hour by default
+    const limit = opts?.limit ?? 20;
+    let entries: ReturnType<typeof readLedgerEntries>;
+    try {
+        entries = readLedgerEntries(UNROUTABLE_DIAGNOSTIC_STREAM, { kind: ['delivery_unroutable'], tail: 200 });
+    } catch {
+        return [];
+    }
+    const cutoff = Date.now() - sinceMs;
+    const out: UnroutableDeliveryDiagnostic[] = [];
+    for (let i = entries.length - 1; i >= 0; i--) {
+        const entry = entries[i];
+        const ts = new Date(entry.timestamp).getTime();
+        if (!Number.isNaN(ts) && ts < cutoff) continue;
+        const payload = entry.payload && typeof entry.payload === 'object' ? entry.payload as Record<string, unknown> : {};
+        out.push({
+            timestamp: entry.timestamp,
+            event: readNonEmptyString(payload.event),
+            sessionId: readNonEmptyString(entry.sessionId) || readNonEmptyString(payload.sessionId) || undefined,
+            workspace: readNonEmptyString(payload.workspace) || undefined,
+            coordinatorDaemonId: readNonEmptyString(payload.coordinatorDaemonId) || undefined,
+        });
+        if (out.length >= limit) break;
+    }
+    return out;
+}

package/src/mesh/mesh-runtime-store.ts

@@ -117,6 +117,21 @@ export class MeshRuntimeStore {
                 expires_at INTEGER NOT NULL
             );
 
+            -- R3: idempotent coordinator inbox. When a terminal/force-inject event is
+            -- direct-injected into a LIVE local CLI coordinator (coord.onEvent('send_message')),
+            -- we record (coordinator_daemon_id, fingerprint) here. That same coordinator also
+            -- polls get_pending_mesh_events, which would re-deliver the queued copy of the very
+            -- event it just received in its PTY → user sees the completion twice. The drain for
+            -- a coordinator daemon filters out events already direct-delivered to it, giving
+            -- exactly-once-per-coordinator while keeping the queue for other consumers (idle /
+            -- MCP-only / remote) that did NOT receive the direct inject.
+            CREATE TABLE IF NOT EXISTS mesh_direct_delivered_events (
+                coordinator_daemon_id TEXT NOT NULL,
+                fingerprint TEXT NOT NULL,
+                expires_at INTEGER NOT NULL,
+                PRIMARY KEY (coordinator_daemon_id, fingerprint)
+            );
+
             CREATE TABLE IF NOT EXISTS mesh_direct_dispatches (
                 task_id TEXT PRIMARY KEY,
                 mesh_id TEXT NOT NULL,
@@ -281,6 +296,28 @@ export class MeshRuntimeStore {
         this.db.prepare('DELETE FROM mesh_completion_fingerprints WHERE expires_at <= ?').run(Date.now());
     }
 
+    // R3: record that an event (by pending-event fingerprint) was direct-injected into a live
+    // coordinator on the given daemon, so that coordinator's own drain skips the queued copy.
+    recordDirectDelivered(coordinatorDaemonId: string, fingerprint: string, ttlMs: number): void {
+        if (!coordinatorDaemonId || !fingerprint) return;
+        this.db.prepare(
+            'INSERT OR REPLACE INTO mesh_direct_delivered_events (coordinator_daemon_id, fingerprint, expires_at) VALUES (?, ?, ?)'
+        ).run(coordinatorDaemonId, fingerprint, Date.now() + ttlMs);
+        this.maybeCheckpointWal();
+    }
+
+    wasDirectDelivered(coordinatorDaemonId: string, fingerprint: string): boolean {
+        if (!coordinatorDaemonId || !fingerprint) return false;
+        const row = this.db.prepare(
+            'SELECT 1 FROM mesh_direct_delivered_events WHERE coordinator_daemon_id = ? AND fingerprint = ? AND expires_at > ?'
+        ).get(coordinatorDaemonId, fingerprint, Date.now());
+        return row !== undefined;
+    }
+
+    sweepExpiredDirectDelivered(): void {
+        this.db.prepare('DELETE FROM mesh_direct_delivered_events WHERE expires_at <= ?').run(Date.now());
+    }
+
     private maybeCheckpointWal(): void {
         if (++this.walWriteCounter < MeshRuntimeStore.WAL_CHECK_INTERVAL) return;
         this.walWriteCounter = 0;