@adhdev/daemon-core 0.9.77-rc.4 → 0.9.77-rc.40

package/src/mesh/mesh-events.ts

@@ -3,7 +3,20 @@ import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
 import { LOG } from '../logging/logger.js';
 import { appendLedgerEntry, getSessionRecoveryContext } from './mesh-ledger.js';
 import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
-import { claimNextTask, updateSessionTaskStatus, enqueueTask } from './mesh-work-queue.js';
+import { claimNextTask, updateSessionTaskStatus, enqueueTask, updateTaskStatus } from './mesh-work-queue.js';
+
+// ---------------------------------------------------------------------------
+// Remote Node Idle Session Tracking
+// ---------------------------------------------------------------------------
+// Tracks remote sessions that emitted 'agent:ready' so triggerMeshQueue 
+// can assign tasks to them.
+// ---------------------------------------------------------------------------
+interface RemoteIdleSession {
+    nodeId: string;
+    sessionId: string;
+    providerType: string;
+}
+const remoteIdleSessions = new Map<string, RemoteIdleSession>(); // key: `${nodeId}:${sessionId}`
 
 // ---------------------------------------------------------------------------
 // MCP coordinator pending-event queue
@@ -33,10 +46,19 @@ function readNonEmptyString(value: unknown): string {
     return typeof value === 'string' && value.trim() ? value.trim() : '';
 }
 
+function resolveEventSessionId(event: Record<string, unknown>, fallback?: unknown): string {
+    return readNonEmptyString(event.targetSessionId)
+        || readNonEmptyString(event.sessionId)
+        || readNonEmptyString(event.instanceId)
+        || readNonEmptyString(fallback);
+}
+
 const MESH_COORDINATOR_EVENTS = new Set([
+    'agent:generating_started',
     'agent:generating_completed',
     'agent:waiting_approval',
     'agent:stopped',
+    'agent:ready',
     'monitor:long_generating',
 ]);
 
@@ -60,25 +82,57 @@ function formatCompletionMetadata(event: Record<string, unknown>): string {
     return parts.length > 0 ? ` (${parts.join('; ')})` : '';
 }
 
+function getMeshWithCache(components: DaemonComponents, meshId: string): any | undefined {
+    const localMesh = getMesh(meshId);
+    if (localMesh) return localMesh;
+    return components.router?.getCachedInlineMesh(meshId);
+}
+
+
 export function tryAssignQueueTask(
-    components: { cliManager: any },
+    components: DaemonComponents,
     meshId: string,
     nodeId: string,
     sessionId: string,
     providerType: string
 ): boolean {
     const task = claimNextTask(meshId, nodeId, sessionId);
-    if (!task) return false;
+    if (!task) {
+        return false;
+    }
 
     LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
+
+    // Check if the node is remote
+    const mesh = getMeshWithCache(components, meshId);
+    const node = mesh?.nodes.find((n: any) => n.id === nodeId);
     
+    // If the node is explicitly remote and we have a dispatch mechanism, route via P2P
+    if (node?.daemonId && components.dispatchMeshCommand) {
+        const isLocalNode = components.cliManager.adapters.has(sessionId);
+        if (!isLocalNode) {
+            components.dispatchMeshCommand(node.daemonId, 'agent_command', {
+                targetSessionId: sessionId,
+                cliType: providerType,
+                action: 'send_chat',
+                message: task.message,
+            }).catch((e: any) => {
+                LOG.error('MeshQueue', `Failed to dispatch task via P2P to remote node ${nodeId}: ${e?.message}`);
+                updateTaskStatus(meshId, task.id, 'failed');
+            });
+            return true;
+        }
+    }
+
+    // Local routing
     components.cliManager.handleCliCommand('agent_command', {
         targetSessionId: sessionId,
         cliType: providerType,
         action: 'send_chat',
-        input: task.message,
+        message: task.message,
     }).catch((e: any) => {
-        LOG.error('MeshQueue', `Failed to dispatch task to node ${nodeId}: ${e?.message}`);
+        LOG.error('MeshQueue', `Failed to dispatch task locally to node ${nodeId}: ${e?.message}`);
+        updateTaskStatus(meshId, task.id, 'failed');
     });
 
     return true;
@@ -88,8 +142,8 @@ export function tryAssignQueueTask(
  * Triggers a queue check for all nodes in the mesh.
  * Called when a new task is enqueued, in case nodes are already idle.
  */
-export function triggerMeshQueue(components: { instanceManager: any; cliManager: any }, meshId: string) {
-    const mesh = getMesh(meshId);
+export function triggerMeshQueue(components: DaemonComponents, meshId: string) {
+    const mesh = getMeshWithCache(components, meshId);
     if (!mesh) return;
 
     // Find all CLI instances that belong to this mesh and are idle
@@ -99,13 +153,17 @@ export function triggerMeshQueue(components: { instanceManager: any; cliManager:
         const settings = state.settings as Record<string, unknown> || {};
         
         const instMeshId = readNonEmptyString(settings.meshNodeFor);
-        if (instMeshId !== meshId && !settings.launchedByCoordinator) continue;
+        if (instMeshId !== meshId) continue;
 
         const nodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
         if (!nodeId) continue;
 
-        // Is it idle? (online and waiting for input)
-        if (state.status !== 'idle' && state.status !== 'stopped' && state.activeChat?.status !== 'waiting_input') continue;
+        // Only genuinely idle live sessions can pull work. Restored/stopped
+        // records are kept for transcript/recovery visibility, but assigning
+        // queue items to them strands tasks in assigned/pending without chat.
+        const status = readNonEmptyString(state.status).toLowerCase();
+        if (['stopped', 'failed', 'terminated', 'exited', 'closed'].includes(status)) continue;
+        if (status !== 'idle' && state.activeChat?.status !== 'waiting_input') continue;
 
         const sessionId = state.instanceId;
         const providerType = state.type || readNonEmptyString(settings.providerType);
@@ -115,6 +173,18 @@ export function triggerMeshQueue(components: { instanceManager: any; cliManager:
             tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
         }
     }
+
+    // Also check known idle remote sessions
+    for (const [key, idle] of remoteIdleSessions.entries()) {
+        // Find if this node is in the same mesh
+        const node = mesh.nodes.find((n: any) => n.id === idle.nodeId);
+        if (node) {
+            const assigned = tryAssignQueueTask(components, meshId, idle.nodeId, idle.sessionId, idle.providerType);
+            if (assigned) {
+                remoteIdleSessions.delete(key);
+            }
+        }
+    }
 }
 
 function buildMeshSystemMessage(args: {
@@ -164,18 +234,21 @@ function buildMeshSystemMessage(args: {
 function injectMeshSystemMessage(components: DaemonComponents, args: {
     meshId: string;
     sourceInstanceId?: string;
+    nodeId?: string;
     nodeLabel: string;
     event: string;
     metadataEvent: Record<string, unknown>;
 }) {
     // ── Task Queue & Ledger ──
+    let completedTaskForLedger: { id?: string } | null = null;
     if (args.event === 'agent:generating_completed') {
-        const sessionId = readNonEmptyString(args.metadataEvent.targetSessionId);
-        const nodeId = readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
+        const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
+        const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
         const providerType = readNonEmptyString(args.metadataEvent.providerType);
         
         if (sessionId) {
-            updateSessionTaskStatus(args.meshId, sessionId, 'completed');
+            const completedTask = updateSessionTaskStatus(args.meshId, sessionId, 'completed');
+            completedTaskForLedger = completedTask ? { id: completedTask.id } : null;
             if (nodeId && providerType) {
                 // Short delay to allow completion event to propagate before pulling next
                 setTimeout(() => {
@@ -183,8 +256,56 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
                 }, 500);
             }
         }
+    } else if (args.event === 'agent:ready') {
+        const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
+        const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
+        const providerType = readNonEmptyString(args.metadataEvent.providerType);
+        const completedTask = sessionId
+            ? updateSessionTaskStatus(args.meshId, sessionId, 'completed')
+            : null;
+        if (completedTask) {
+            completedTaskForLedger = { id: completedTask.id };
+            try {
+                appendLedgerEntry(args.meshId, {
+                    kind: 'task_completed',
+                    nodeId: nodeId || undefined,
+                    sessionId,
+                    providerType: providerType || undefined,
+                    payload: {
+                        event: args.event,
+                        nodeLabel: args.nodeLabel,
+                        taskId: completedTask.id,
+                        completedViaReady: true,
+                        providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
+                        finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
+                    },
+                });
+            } catch (e: any) {
+                LOG.warn('MeshLedger', `Failed to record task_completed from ready: ${e?.message || e}`);
+            }
+        }
+        
+        if (sessionId && nodeId && providerType) {
+            remoteIdleSessions.set(`${nodeId}:${sessionId}`, { nodeId, sessionId, providerType });
+            setTimeout(() => {
+                const assigned = tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
+                if (assigned) {
+                    remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
+                }
+            }, 500);
+        }
+    } else if (args.event === 'agent:generating_started') {
+        const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
+        const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
+        if (sessionId && nodeId) {
+            remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
+        }
     } else if (args.event === 'agent:stopped') {
-        const sessionId = readNonEmptyString(args.metadataEvent.targetSessionId);
+        const sessionId = resolveEventSessionId(args.metadataEvent, args.sourceInstanceId);
+        const nodeId = readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
+        if (sessionId && nodeId) {
+            remoteIdleSessions.delete(`${nodeId}:${sessionId}`);
+        }
         if (sessionId) {
             updateSessionTaskStatus(args.meshId, sessionId, 'failed');
         }
@@ -195,13 +316,15 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
         try {
             appendLedgerEntry(args.meshId, {
                 kind: ledgerKind,
-                nodeId: readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
-                sessionId: readNonEmptyString(args.metadataEvent.targetSessionId) || undefined,
+                nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
+                sessionId: resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined,
                 providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
                 payload: {
                     event: args.event,
                     nodeLabel: args.nodeLabel,
+                    taskId: completedTaskForLedger?.id || undefined,
                     providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
+                    finalSummary: readNonEmptyString(args.metadataEvent.finalSummary) || undefined,
                 },
             });
         } catch (e: any) {
@@ -218,8 +341,8 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
             const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
 
             recoveryContext = getSessionRecoveryContext(args.meshId, {
-                sessionId: readNonEmptyString(args.metadataEvent.targetSessionId) || undefined,
-                nodeId: readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
+                sessionId: resolveEventSessionId(args.metadataEvent, args.sourceInstanceId) || undefined,
+                nodeId: readNonEmptyString(args.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
                 maxRetries,
             });
             recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
@@ -327,12 +450,14 @@ export function handleMeshForwardEvent(components: DaemonComponents, payload: Re
     const nodeLabel = nodeId ? `Node '${nodeId}'` : workspace ? `Agent at ${workspace}` : 'Remote agent';
     return injectMeshSystemMessage(components, {
         meshId,
+        nodeId,
         nodeLabel,
         event: eventName,
         metadataEvent: {
-            targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId),
+            targetSessionId: readNonEmptyString(payload.targetSessionId) || readNonEmptyString(payload.sessionId) || readNonEmptyString(payload.instanceId),
             providerType: readNonEmptyString(payload.providerType),
             providerSessionId: readNonEmptyString(payload.providerSessionId),
+            finalSummary: readNonEmptyString(payload.finalSummary) || readNonEmptyString(payload.summary),
         },
     });
 }
@@ -367,13 +492,14 @@ export function setupMeshEventForwarding(components: DaemonComponents) {
         const isMeshDelegate = Boolean(meshIdFromRuntime || settings.launchedByCoordinator);
         if (!isMeshDelegate) return;
 
-        const mesh = meshIdFromRuntime ? getMesh(meshIdFromRuntime) : getMeshByRepo(workspace);
+        const mesh = meshIdFromRuntime ? getMeshWithCache(components, meshIdFromRuntime) : getMeshByRepo(workspace);
         const meshId = meshIdFromRuntime || readNonEmptyString(mesh?.id);
         if (!meshId) return;
 
         // Determine node label. Inline/cloud meshes may be unavailable here, so preserve runtime node id.
         const targetNode = mesh?.nodes?.find((n: any) => n.workspace === workspace);
         const runtimeNodeId = readNonEmptyString(settings.meshNodeId);
+        const resolvedNodeId = targetNode?.id || runtimeNodeId;
         const nodeLabel = targetNode
             ? `Node '${targetNode.id}'`
             : runtimeNodeId
@@ -383,6 +509,7 @@ export function setupMeshEventForwarding(components: DaemonComponents) {
         injectMeshSystemMessage(components, {
             meshId,
             sourceInstanceId: instanceId,
+            nodeId: resolvedNodeId,
             nodeLabel,
             event: event.event,
             metadataEvent: event,

package/src/mesh/mesh-work-queue.ts

@@ -3,7 +3,7 @@ import { join } from 'path';
 import { randomUUID } from 'crypto';
 import { getLedgerDir } from './mesh-ledger.js';
 
-export type MeshTaskStatus = 'pending' | 'assigned' | 'completed' | 'failed';
+export type MeshTaskStatus = 'pending' | 'assigned' | 'completed' | 'failed' | 'cancelled';
 
 export interface MeshWorkQueueEntry {
     id: string;
@@ -12,10 +12,19 @@ export interface MeshWorkQueueEntry {
     status: MeshTaskStatus;
     /** If specified, only this node can claim the task (used by legacy mesh_send_task) */
     targetNodeId?: string;
+    /** If specified, only this runtime session can claim the task */
+    targetSessionId?: string;
     /** The node that actually claimed and is executing the task */
     assignedNodeId?: string;
     /** The session currently executing the task */
     assignedSessionId?: string;
+    /** Human/operator reason for terminal cancellation. */
+    cancelReason?: string;
+    cancelledAt?: string;
+    /** Human/operator reason for manually requeueing a task. */
+    requeueReason?: string;
+    requeuedAt?: string;
+    requeueCount?: number;
     createdAt: string;
     updatedAt: string;
 }
@@ -47,7 +56,7 @@ function writeQueue(meshId: string, queue: MeshWorkQueueEntry[]): void {
 export function enqueueTask(
     meshId: string,
     message: string,
-    opts?: { targetNodeId?: string }
+    opts?: { targetNodeId?: string; targetSessionId?: string }
 ): MeshWorkQueueEntry {
     const queue = readQueue(meshId);
     const entry: MeshWorkQueueEntry = {
@@ -56,6 +65,7 @@ export function enqueueTask(
         message,
         status: 'pending',
         targetNodeId: opts?.targetNodeId,
+        targetSessionId: opts?.targetSessionId,
         createdAt: new Date().toISOString(),
         updatedAt: new Date().toISOString(),
     };
@@ -81,13 +91,25 @@ export function getQueue(meshId: string, opts?: { status?: MeshTaskStatus[] }):
  */
 export function claimNextTask(meshId: string, nodeId: string, sessionId: string): MeshWorkQueueEntry | null {
     const queue = readQueue(meshId);
+
+    // A worker must finish or fail its current queued assignment before it can
+    // claim another one. maxParallelTasks limits total mesh concurrency; it is
+    // not permission for one node/session to accumulate multiple assigned items.
+    const hasActiveAssignment = queue.some(q => q.status === 'assigned' && (
+        q.assignedSessionId === sessionId || q.assignedNodeId === nodeId
+    ));
+    if (hasActiveAssignment) return null;
     
     // Find highest priority task:
-    // 1. Pending tasks explicitly targeted at this node
-    // 2. Pending tasks with no target node
-    let targetIdx = queue.findIndex(q => q.status === 'pending' && q.targetNodeId === nodeId);
+    // 1. Pending tasks explicitly targeted at this runtime session
+    // 2. Pending tasks explicitly targeted at this node (but not another session)
+    // 3. Pending tasks with no target node/session
+    let targetIdx = queue.findIndex(q => q.status === 'pending' && q.targetSessionId === sessionId);
+    if (targetIdx === -1) {
+        targetIdx = queue.findIndex(q => q.status === 'pending' && q.targetNodeId === nodeId && !q.targetSessionId);
+    }
     if (targetIdx === -1) {
-        targetIdx = queue.findIndex(q => q.status === 'pending' && !q.targetNodeId);
+        targetIdx = queue.findIndex(q => q.status === 'pending' && !q.targetNodeId && !q.targetSessionId);
     }
 
     if (targetIdx === -1) return null;
@@ -121,6 +143,65 @@ export function updateTaskStatus(
     return queue[idx];
 }
 
+/**
+ * Mark a queue task as manually cancelled without deleting audit history.
+ */
+export function cancelTask(
+    meshId: string,
+    taskId: string,
+    opts?: { reason?: string },
+): MeshWorkQueueEntry | null {
+    const queue = readQueue(meshId);
+    const idx = queue.findIndex(q => q.id === taskId);
+    if (idx === -1) return null;
+
+    const now = new Date().toISOString();
+    queue[idx].status = 'cancelled';
+    queue[idx].updatedAt = now;
+    queue[idx].cancelledAt = now;
+    if (opts?.reason) queue[idx].cancelReason = opts.reason;
+    writeQueue(meshId, queue);
+    return queue[idx];
+}
+
+/**
+ * Return a queue task to pending for retry. By default, dead session targeting
+ * and assigned ownership are cleared so stale assignments do not strand again.
+ */
+export function requeueTask(
+    meshId: string,
+    taskId: string,
+    opts?: {
+        reason?: string;
+        targetNodeId?: string;
+        targetSessionId?: string;
+        clearTargetNode?: boolean;
+        clearTargetSession?: boolean;
+    },
+): MeshWorkQueueEntry | null {
+    const queue = readQueue(meshId);
+    const idx = queue.findIndex(q => q.id === taskId);
+    if (idx === -1) return null;
+
+    const entry = queue[idx];
+    const now = new Date().toISOString();
+    entry.status = 'pending';
+    delete entry.assignedNodeId;
+    delete entry.assignedSessionId;
+    delete entry.cancelledAt;
+    delete entry.cancelReason;
+    if (opts?.clearTargetNode) delete entry.targetNodeId;
+    if (typeof opts?.targetNodeId === 'string') entry.targetNodeId = opts.targetNodeId;
+    if (opts?.clearTargetSession !== false) delete entry.targetSessionId;
+    if (typeof opts?.targetSessionId === 'string') entry.targetSessionId = opts.targetSessionId;
+    entry.updatedAt = now;
+    entry.requeuedAt = now;
+    entry.requeueCount = (entry.requeueCount || 0) + 1;
+    if (opts?.reason) entry.requeueReason = opts.reason;
+    writeQueue(meshId, queue);
+    return entry;
+}
+
 /**
  * Update the status of the task currently assigned to a specific session.
  */
@@ -148,6 +229,13 @@ export interface MeshWorkQueueStats {
     assigned: number;
     completed: number;
     failed: number;
+    cancelled: number;
+    activeAssignments: Array<{
+        id: string;
+        nodeId?: string;
+        sessionId?: string;
+        message: string;
+    }>;
 }
 
 /**
@@ -160,5 +248,14 @@ export function getMeshQueueStats(meshId: string): MeshWorkQueueStats {
         assigned: queue.filter(q => q.status === 'assigned').length,
         completed: queue.filter(q => q.status === 'completed').length,
         failed: queue.filter(q => q.status === 'failed').length,
+        cancelled: queue.filter(q => q.status === 'cancelled').length,
+        activeAssignments: queue
+            .filter(q => q.status === 'assigned')
+            .map(q => ({
+                id: q.id,
+                nodeId: q.assignedNodeId,
+                sessionId: q.assignedSessionId,
+                message: q.message,
+            })),
     };
 }

package/src/providers/cli-provider-instance.ts

@@ -833,6 +833,8 @@ export class CliProviderInstance implements ProviderInstance {
                         this.completedDebounceTimer = null;
                     }, 3000);
                 }
+            } else if (newStatus === 'idle' && this.lastStatus === 'starting') {
+                this.pushEvent({ event: 'agent:ready', chatTitle, timestamp: now });
             } else if (newStatus === 'stopped') {
                 // Cancel any pending debounce
                 if (this.generatingDebounceTimer) { clearTimeout(this.generatingDebounceTimer); this.generatingDebounceTimer = null; }

package/src/shared-types.ts

@@ -394,6 +394,13 @@ export interface SessionEntry {
         assigned: number;
         completed: number;
         failed: number;
+        cancelled?: number;
+        activeAssignments?: Array<{
+            id: string;
+            nodeId?: string;
+            sessionId?: string;
+            message: string;
+        }>;
     };
 }
 
@@ -439,6 +446,13 @@ export interface CompactSessionEntry {
         assigned: number;
         completed: number;
         failed: number;
+        cancelled?: number;
+        activeAssignments?: Array<{
+            id: string;
+            nodeId?: string;
+            sessionId?: string;
+            message: string;
+        }>;
     };
 }