@adhdev/daemon-core 0.9.77-rc.1 → 0.9.77-rc.11

package/src/commands/stream-commands.ts

@@ -113,11 +113,18 @@ export async function handleOpenPanel(h: CommandHelpers, args: any): Promise<Com
 export async function handlePtyInput(h: CommandHelpers, args: any): Promise<CommandResult> {
     const { cliType, data, targetSessionId } = args || {};
     if (!data) return { success: false, error: 'data required' };
+
+    // Filter out VT100/VT420 Device Attributes responses (e.g. \x1b[?1;2c)
+    // These are echoed by xterm.js in the dashboard in response to \x1b[c queries
+    // and pollute the CLI input buffer.
+    const cleanData = typeof data === 'string' ? data.replace(/\x1b\[\?[0-9;]*c/g, '') : data;
+    if (!cleanData) return { success: true };
+
     const adapter = h.getCliAdapter(targetSessionId || cliType);
     if (!adapter || typeof adapter.writeRaw !== 'function') {
         return { success: false, error: `CLI adapter not found: ${targetSessionId || cliType || 'unknown'}` };
     }
-    await adapter.writeRaw(data);
+    await adapter.writeRaw(cleanData);
     return { success: true };
 }
 

package/src/index.ts

@@ -149,6 +149,17 @@ export type { CoordinatorPromptContext } from './mesh/coordinator-prompt.js';
 export { syncMeshes } from './mesh/mesh-sync.js';
 export type { MeshSyncTransport, MeshSyncResult, RemoteMeshRecord } from './mesh/mesh-sync.js';
 
+// ── Mesh Task Ledger ──
+export { appendLedgerEntry, readLedgerEntries, getLedgerSummary, getLedgerDir, getSessionRecoveryContext } from './mesh/mesh-ledger.js';
+export type { MeshLedgerEntry, MeshLedgerKind, MeshLedgerSummary, ReadLedgerOptions, SessionRecoveryContext } from './mesh/mesh-ledger.js';
+
+// ── Mesh Work Queue (GUPP) ──
+export { enqueueTask, getQueue, claimNextTask, updateTaskStatus, updateSessionTaskStatus } from './mesh/mesh-work-queue.js';
+export type { MeshWorkQueueEntry, MeshTaskStatus } from './mesh/mesh-work-queue.js';
+
+// ── Mesh Events ──
+export { triggerMeshQueue } from './mesh/mesh-events.js';
+
 // ── State Store ──
 export { loadState, saveState, resetState } from './config/state-store.js';
 export type { DaemonState } from './config/state-store.js';

package/src/mesh/coordinator-prompt.ts

@@ -133,6 +133,7 @@ const TOOLS_SECTION = `## Available Tools
 | \`mesh_launch_session\` | Start a new agent session on a node |
 | \`mesh_send_task\` | Send a task (natural language) to a running agent |
 | \`mesh_read_chat\` | Read an agent's recent messages to check progress |
+| \`mesh_task_history\` | Read the task ledger — dispatches, completions, failures. Use to understand what has been done before deciding next steps |
 | \`mesh_git_status\` | Check git status on a specific node |
 | \`mesh_checkpoint\` | Create a git checkpoint on a node |
 | \`mesh_approve\` | Approve/reject a pending agent action |
@@ -145,18 +146,31 @@ Before doing any coordinator work, confirm that the actual callable tool list in
 
 const WORKFLOW_SECTION = `## Orchestration Workflow
 
-1. **Assess** — Call \`mesh_status\` to see which nodes are healthy and available.
-2. **Plan** — Decompose the user's request into independent tasks for parallel execution, or sequential tasks when dependencies exist.
-3. **Delegate** — For each task:
-   a. Pick the best node (consider: health, dirty state, current workload).
-   b. If you need branch isolation for parallel work, call \`mesh_clone_node\` to create a worktree node first.
-   c. If no session exists, call \`mesh_launch_session\` to start one.
-   d. Call \`mesh_send_task\` with a **complete, self-contained** instruction that includes all context the agent needs (file paths, line numbers, what to change, why). Do not send partial instructions expecting future follow-up.
-4. **Monitor** — Prefer event-driven completion/status notifications. Do **not** poll \`mesh_read_chat\` repeatedly just because the delegated session has not produced a final assistant message yet; tool/terminal activity means work may still be in progress. Do not call \`mesh_read_chat\` again within a few seconds for the same generating session; wait for the completion callback/status event instead unless you are debugging a real stall. Use at most one compact \`mesh_read_chat\` check after a completion/approval signal, an explicit user status request, or a real timeout/stall. Handle approvals via \`mesh_approve\`.
+1. **Assess** — Call \`mesh_status\` to see which nodes are healthy and available. Check \`mesh_task_history\` to understand what has already been done in this mesh — previous delegations, completions, and failures.
+2. **Plan** — Decompose the user's request into independent tasks for parallel execution, or sequential tasks when dependencies exist. If \`mesh_task_history\` shows a recent failure for a task, decide whether to retry or reassign.
+3. **Queue / Delegate** — The Mesh uses an autonomous pull-based Work Queue:
+   a. **General Tasks**: Enqueue tasks using \`mesh_enqueue_task\`. Idle node agents will automatically pull tasks from the queue and begin working.
+   b. **Node Preparation**: Call \`mesh_launch_session\` to ensure enough agent sessions are active to handle the queue. If you need branch isolation for parallel work, call \`mesh_clone_node\` to create a worktree node first.
+   c. **Targeted Tasks**: Use \`mesh_send_task\` only when you need to bypass the queue and force a specific node to execute a task immediately.
+   d. Always provide a **complete, self-contained** instruction that includes all context the agent needs (file paths, line numbers, what to change, why). Do not send partial instructions expecting future follow-up.
+4. **Monitor** — Prefer event-driven completion/status notifications. Do **not** poll \`mesh_read_chat\` repeatedly. Use \`mesh_view_queue\` to see the status of all pending, assigned, completed, and failed tasks. Do not call \`mesh_read_chat\` again within a few seconds for the same generating session. Use at most one compact \`mesh_read_chat\` check after a completion/approval signal. Handle approvals via \`mesh_approve\`.
 5. **Verify** — When a task reports completion or git work is visible, call \`mesh_git_status\` to verify changes were made.
 6. **Checkpoint** — Call \`mesh_checkpoint\` to save the work.
 7. **Clean up** — Remove worktree nodes via \`mesh_remove_node\` after their work is merged or no longer needed.
-8. **Report** — Summarize what was done, what changed, and any issues.`;
+8. **Report** — Summarize what was done, what changed, and any issues.
+
+## Failure Recovery
+
+When a node agent stops unexpectedly, the daemon automatically enriches the system message with **Recovery Context** that includes:
+- The number of consecutive failures on that node
+- The original task message (if recorded in the ledger)
+- A recommendation: **retry**, **reassign**, or **escalate**
+
+Follow these recovery rules:
+1. **If "Retry recommended"**: Re-launch the session on the same node (\`mesh_launch_session\`), then resend the original task (\`mesh_send_task\`). The system message includes the original task text.
+2. **If "Max retries exceeded"**: Do NOT retry on the same node. Either reassign the task to a different node, or inform the user that the task requires manual intervention.
+3. **If no recovery context**: The stop may be intentional (normal completion). Use \`mesh_read_chat\` once to verify, then move on.
+4. **Always record what happened**: After handling a failure, briefly note the outcome in your report to the user.`;
 
 function buildRulesSection(coordinatorCliType?: string): string {
     const coordinatorNote = coordinatorCliType
@@ -166,12 +180,13 @@ function buildRulesSection(coordinatorCliType?: string): string {
     return `## Rules
 
 - **Minimize coordinator context.** The coordinator's job is routing, not implementing. Do not read source files, run commands, or analyze code directly — delegate all of that to node agents. Your context should stay lean.
-- **Delegate analysis too.** If you need to understand a bug or explore the codebase, send that investigation as a task to a node. Do not do it yourself.
+- **Delegate analysis too.** If you need to understand a bug or explore the codebase, send that investigation as a task to the queue or a node. Do not do it yourself.
 - **Respect explicit provider requests.** If the user names an agent/provider, pass the matching provider type to \`mesh_launch_session\`: Hermes → \`hermes-cli\`, Claude Code/Claude → \`claude-cli\`, Codex → \`codex-cli\`, Gemini → \`gemini-cli\`. Never substitute \`claude-cli\` just because the coordinator itself is Claude Code.
-- **Front-load the task message.** When calling \`mesh_send_task\`, include everything the agent needs: what files to touch, what the problem is, what the fix should look like. The agent won't ask follow-up questions.
+- **Front-load the task message.** When calling \`mesh_enqueue_task\` or \`mesh_send_task\`, include everything the agent needs: what files to touch, what the problem is, what the fix should look like. The agent won't ask follow-up questions.
 - **Don't inspect code.** Treat delegated agent summaries as self-reports, not verification. Verify side effects via \`mesh_git_status\` (including related repo freshness when configured), not by reading source files.
 - **Don't over-parallelize.** Start with 1-2 concurrent tasks. Scale up if they succeed. Never launch a duplicate session or second worker solely because \`mesh_read_chat\` has no final assistant message while the delegated session is still showing tool/terminal activity.
-- **Handle failures gracefully.** If a task fails, read the chat to understand why, then retry or reassign.
+- **Handle failures with context.** If a task fails, check \`mesh_task_history\` first to see if this task was attempted before and how it failed. Read the chat to understand why, then decide: retry on the same node, reassign to a different node, or escalate to the user.
+- **Check history before starting.** At the beginning of a coordination session, call \`mesh_task_history\` to understand what was previously delegated and its outcomes. This prevents duplicate work and informs recovery decisions.
 - **Keep the user informed.** Report progress after each delegation round — one or two sentences, not a narration.
 - **Respect node capabilities.** Don't send build tasks to read-only nodes. Don't push from nodes that aren't allowed to.
 - **Never fabricate tool results.** Always call the actual tool; never pretend you did.

package/src/mesh/mesh-events.ts

@@ -1,6 +1,9 @@
 import type { DaemonComponents } from '../boot/daemon-lifecycle.js';
 import { getMesh, getMeshByRepo } from '../config/mesh-config.js';
 import { LOG } from '../logging/logger.js';
+import { appendLedgerEntry, getSessionRecoveryContext } from './mesh-ledger.js';
+import type { MeshLedgerKind, SessionRecoveryContext } from './mesh-ledger.js';
+import { claimNextTask, updateSessionTaskStatus, enqueueTask } from './mesh-work-queue.js';
 
 // ---------------------------------------------------------------------------
 // MCP coordinator pending-event queue
@@ -37,6 +40,13 @@ const MESH_COORDINATOR_EVENTS = new Set([
     'monitor:long_generating',
 ]);
 
+const EVENT_TO_LEDGER_KIND: Record<string, MeshLedgerKind> = {
+    'agent:generating_completed': 'task_completed',
+    'agent:waiting_approval': 'task_approval_needed',
+    'agent:stopped': 'task_failed',
+    'monitor:long_generating': 'task_stalled',
+};
+
 function isMeshCoordinatorEvent(eventName: unknown): eventName is string {
     return typeof eventName === 'string' && MESH_COORDINATOR_EVENTS.has(eventName);
 }
@@ -50,10 +60,68 @@ function formatCompletionMetadata(event: Record<string, unknown>): string {
     return parts.length > 0 ? ` (${parts.join('; ')})` : '';
 }
 
+export function tryAssignQueueTask(
+    components: { cliManager: any },
+    meshId: string,
+    nodeId: string,
+    sessionId: string,
+    providerType: string
+): boolean {
+    const task = claimNextTask(meshId, nodeId, sessionId);
+    if (!task) return false;
+
+    LOG.info('MeshQueue', `Node ${nodeId} (${sessionId}) pulled task ${task.id}`);
+    
+    components.cliManager.handleCliCommand('agent_command', {
+        targetSessionId: sessionId,
+        cliType: providerType,
+        action: 'send_chat',
+        message: task.message,
+    }).catch((e: any) => {
+        LOG.error('MeshQueue', `Failed to dispatch task to node ${nodeId}: ${e?.message}`);
+    });
+
+    return true;
+}
+
+/**
+ * Triggers a queue check for all nodes in the mesh.
+ * Called when a new task is enqueued, in case nodes are already idle.
+ */
+export function triggerMeshQueue(components: { instanceManager: any; cliManager: any }, meshId: string) {
+    const mesh = getMesh(meshId);
+    if (!mesh) return;
+
+    // Find all CLI instances that belong to this mesh and are idle
+    const cliInstances = components.instanceManager.getByCategory('cli');
+    for (const inst of cliInstances) {
+        const state = inst.getState();
+        const settings = state.settings as Record<string, unknown> || {};
+        
+        const instMeshId = readNonEmptyString(settings.meshNodeFor);
+        if (instMeshId !== meshId && !settings.launchedByCoordinator) continue;
+
+        const nodeId = readNonEmptyString(settings.meshNodeId) || readNonEmptyString(settings.nodeId);
+        if (!nodeId) continue;
+
+        // Is it idle? (online and waiting for input)
+        if (state.status !== 'idle' && state.status !== 'stopped' && state.activeChat?.status !== 'waiting_input') continue;
+
+        const sessionId = state.instanceId;
+        const providerType = state.type || readNonEmptyString(settings.providerType);
+        
+        if (providerType) {
+            // Try to assign a task to this idle node
+            tryAssignQueueTask(components, meshId, nodeId, sessionId, providerType);
+        }
+    }
+}
+
 function buildMeshSystemMessage(args: {
     event: string;
     nodeLabel: string;
     metadataEvent: Record<string, unknown>;
+    recoveryContext?: SessionRecoveryContext | null;
 }): string {
     const metadata = formatCompletionMetadata(args.metadataEvent);
     if (args.event === 'agent:generating_completed') {
@@ -63,6 +131,28 @@ function buildMeshSystemMessage(args: {
         return `[System] ${args.nodeLabel} is waiting for approval to proceed${metadata}. You may use mesh_read_chat and mesh_approve to handle it.`;
     }
     if (args.event === 'agent:stopped') {
+        const rc = args.recoveryContext;
+        if (rc && rc.consecutiveNodeFailures > 0) {
+            const parts = [
+                `[System] ${args.nodeLabel} has stopped unexpectedly${metadata}.`,
+                `\n\n**Recovery Context:**`,
+                `- Consecutive failures on this node: ${rc.consecutiveNodeFailures}`,
+                rc.taskAttemptCount > 0 ? `- This task has been attempted ${rc.taskAttemptCount} time(s)` : '',
+                `- Recommendation: ${rc.advice}`,
+            ];
+            if (rc.retryRecommended && rc.lastTaskMessage) {
+                parts.push(
+                    `\n\n**Original task to retry:**`,
+                    `> ${rc.lastTaskMessage.length > 300 ? rc.lastTaskMessage.slice(0, 300) + '...' : rc.lastTaskMessage}`,
+                    `\nTo retry: call \`mesh_launch_session\` for this node, then \`mesh_send_task\` with the original task.`,
+                );
+            } else if (!rc.retryRecommended) {
+                parts.push(
+                    `\nDo NOT retry on this node. Consider reassigning to a different node or asking the user for guidance.`,
+                );
+            }
+            return parts.filter(Boolean).join('\n');
+        }
         return `[System] ${args.nodeLabel} has stopped${metadata}. Use mesh_read_chat once if you need to inspect its last output.`;
     }
     if (args.event === 'monitor:long_generating') {
@@ -78,6 +168,111 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
     event: string;
     metadataEvent: Record<string, unknown>;
 }) {
+    // ── Task Queue & Ledger ──
+    if (args.event === 'agent:generating_completed') {
+        const sessionId = readNonEmptyString(args.metadataEvent.targetSessionId);
+        const nodeId = readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId);
+        const providerType = readNonEmptyString(args.metadataEvent.providerType);
+        
+        if (sessionId) {
+            updateSessionTaskStatus(args.meshId, sessionId, 'completed');
+            if (nodeId && providerType) {
+                // Short delay to allow completion event to propagate before pulling next
+                setTimeout(() => {
+                    tryAssignQueueTask(components, args.meshId, nodeId, sessionId, providerType);
+                }, 500);
+            }
+        }
+    } else if (args.event === 'agent:stopped') {
+        const sessionId = readNonEmptyString(args.metadataEvent.targetSessionId);
+        if (sessionId) {
+            updateSessionTaskStatus(args.meshId, sessionId, 'failed');
+        }
+    }
+
+    const ledgerKind = EVENT_TO_LEDGER_KIND[args.event];
+    if (ledgerKind) {
+        try {
+            appendLedgerEntry(args.meshId, {
+                kind: ledgerKind,
+                nodeId: readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
+                sessionId: readNonEmptyString(args.metadataEvent.targetSessionId) || undefined,
+                providerType: readNonEmptyString(args.metadataEvent.providerType) || undefined,
+                payload: {
+                    event: args.event,
+                    nodeLabel: args.nodeLabel,
+                    providerSessionId: readNonEmptyString(args.metadataEvent.providerSessionId) || undefined,
+                },
+            });
+        } catch (e: any) {
+            LOG.warn('MeshLedger', `Failed to record ${ledgerKind}: ${e?.message || e}`);
+        }
+    }
+
+    // ── Recovery Context: enrich agent:stopped with retry intelligence ──
+    let recoveryContext: SessionRecoveryContext | null = null;
+    if (args.event === 'agent:stopped') {
+        try {
+            // Resolve maxTaskRetries from mesh policy
+            const mesh = getMesh(args.meshId);
+            const maxRetries = mesh?.policy?.maxTaskRetries ?? 1;
+
+            recoveryContext = getSessionRecoveryContext(args.meshId, {
+                sessionId: readNonEmptyString(args.metadataEvent.targetSessionId) || undefined,
+                nodeId: readNonEmptyString(args.metadataEvent.nodeId) || readNonEmptyString(args.metadataEvent.meshNodeId) || undefined,
+                maxRetries,
+            });
+            recoveryContext.failedProviderType = readNonEmptyString(args.metadataEvent.providerType) || null;
+
+            // Record recovery_attempted if retry is recommended
+            if (recoveryContext.retryRecommended && recoveryContext.consecutiveNodeFailures > 0) {
+                appendLedgerEntry(args.meshId, {
+                    kind: 'recovery_attempted',
+                    nodeId: recoveryContext.failedNodeId || undefined,
+                    sessionId: recoveryContext.failedSessionId || undefined,
+                    providerType: recoveryContext.failedProviderType || undefined,
+                    payload: {
+                        consecutiveFailures: recoveryContext.consecutiveNodeFailures,
+                        taskAttemptCount: recoveryContext.taskAttemptCount,
+                        retryRecommended: recoveryContext.retryRecommended,
+                        advice: recoveryContext.advice,
+                    },
+                });
+
+                // Auto-Recovery (Phase 5): Automatically re-enqueue the task and re-launch the session
+                if (recoveryContext.lastTaskMessage && recoveryContext.failedNodeId && recoveryContext.failedProviderType) {
+                    const autoNodeId = recoveryContext.failedNodeId;
+                    try {
+                        const task = enqueueTask(args.meshId, recoveryContext.lastTaskMessage, {
+                            targetNodeId: autoNodeId
+                        });
+                        LOG.info('MeshRecovery', `Auto-requeued failed task: ${task.id} for node ${autoNodeId}`);
+
+                        const node = mesh?.nodes.find(n => n.id === autoNodeId);
+                        if (node) {
+                            components.cliManager.handleCliCommand('launch_cli', {
+                                cliType: recoveryContext.failedProviderType,
+                                dir: node.workspace,
+                                settings: {
+                                    meshNodeFor: args.meshId,
+                                    meshNodeId: node.id,
+                                    spawnedSessionVisibility: mesh?.policy?.spawnedSessionVisibility || 'hidden',
+                                    launchedByCoordinator: true,
+                                }
+                            }).catch((e: any) => LOG.error('MeshRecovery', `Failed to auto-relaunch session for ${node.id}: ${e?.message}`));
+                        }
+                    } catch (e: any) {
+                        LOG.warn('MeshRecovery', `Failed to execute auto-recovery: ${e?.message}`);
+                    }
+                }
+            }
+
+            LOG.info('MeshRecovery', `Recovery context for ${args.nodeLabel}: ${recoveryContext.advice}`);
+        } catch (e: any) {
+            LOG.warn('MeshRecovery', `Failed to build recovery context: ${e?.message || e}`);
+        }
+    }
+
     const coordinatorInstances = components.instanceManager.getByCategory('cli').filter((inst) => {
         const instState = inst.getState();
         if (instState.settings?.meshCoordinatorFor !== args.meshId) return false;
@@ -92,7 +287,10 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
                 event: args.event,
                 meshId: args.meshId,
                 nodeLabel: args.nodeLabel,
-                metadataEvent: args.metadataEvent,
+                metadataEvent: {
+                    ...args.metadataEvent,
+                    ...(recoveryContext ? { recoveryContext } : {}),
+                },
                 queuedAt: Date.now(),
             });
             LOG.info('MeshEvents', `Queued ${args.event} for MCP coordinator (mesh ${args.meshId})`);
@@ -104,6 +302,7 @@ function injectMeshSystemMessage(components: DaemonComponents, args: {
         event: args.event,
         nodeLabel: args.nodeLabel,
         metadataEvent: args.metadataEvent,
+        recoveryContext,
     });
     if (!messageText) return { success: false, error: 'unsupported mesh event' };