openclaw-node-harness 2.0.2 → 2.0.4

package/bin/mesh-task-daemon.js

@@ -131,6 +131,36 @@ async function handleSubmit(msg) {
   respond(msg, task);
 }
 
+/**
+ * Abort any collab session tied to a task that is being terminated.
+ * Shared by handleFail, handleRelease, handleCancel.
+ *
+ * NOT called from handleComplete — that path goes through evaluateRound
+ * which already calls collabStore.markCompleted() on the session.
+ *
+ * markAborted() is idempotent: no-op if session is already completed/aborted.
+ * This makes double-abort safe (e.g. stall detection → release race).
+ */
+async function cleanupTaskCollabSession(task, reason) {
+  if (!task.collab_session_id || !collabStore) return;
+  try {
+    // markAborted returns null if session doesn't exist or is already completed/aborted.
+    // Non-null means we actually transitioned the session to aborted.
+    const session = await collabStore.markAborted(task.collab_session_id, reason);
+    if (session) {
+      await collabStore.appendAudit(task.collab_session_id, 'session_aborted', { reason });
+      publishCollabEvent('aborted', session);
+      log(`COLLAB ABORTED ${task.collab_session_id}: ${reason}`);
+    }
+    // Clean up audit error rate-limit counter
+    // NOTE: sessions expiring via KV TTL bypass this — residual Map entry is negligible
+    // for a homelab mesh but worth noting.
+    collabStore.clearAuditErrorCount(task.collab_session_id);
+  } catch (err) {
+    log(`COLLAB CLEANUP WARN: could not abort session ${task.collab_session_id}: ${err.message}`);
+  }
+}
+
 /**
  * mesh.tasks.claim — Agent requests the next available task.
  * Expects: { node_id }
@@ -203,6 +233,14 @@ async function handleComplete(msg) {
   log(`COMPLETE ${task_id} in ${elapsed}m: ${result?.summary || 'no summary'}`);
   publishEvent('completed', task);
 
+  // NOTE: no cleanupTaskCollabSession here — collab tasks complete via
+  // evaluateRound → markCompleted on the session, then store.markCompleted
+  // on the parent task. Calling cleanupTaskCollabSession would markAborted
+  // on an already-completed session. Clean up audit counter only.
+  if (task.collab_session_id && collabStore) {
+    collabStore.clearAuditErrorCount(task.collab_session_id);
+  }
+
   // Check if this task belongs to a plan
   await checkPlanProgress(task_id, 'completed');
 
@@ -222,6 +260,7 @@ async function handleFail(msg) {
 
   log(`FAIL ${task_id}: ${reason}`);
   publishEvent('failed', task);
+  await cleanupTaskCollabSession(task, `Parent task ${task_id} failed: ${reason}`);
 
   // Check if this task belongs to a plan
   await checkPlanProgress(task_id, 'failed');
@@ -302,6 +341,7 @@ async function handleRelease(msg) {
 
   log(`RELEASED ${task_id}: ${reason || 'no reason'} (needs human triage)`);
   publishEvent('released', task);
+  await cleanupTaskCollabSession(task, `Parent task ${task_id} released: ${reason || 'human triage'}`);
   respond(msg, task);
 }
 
@@ -323,6 +363,7 @@ async function handleCancel(msg) {
 
   log(`CANCEL ${task_id}: ${reason || 'no reason'}`);
   publishEvent('cancelled', task);
+  await cleanupTaskCollabSession(task, `Parent task ${task_id} cancelled: ${reason || 'no reason'}`);
   respond(msg, task);
 }
 
@@ -358,6 +399,35 @@ async function detectStalls() {
       }
     }
 
+    // Mark stalled node as dead in any collab sessions it belongs to.
+    // This unblocks isRoundComplete() which otherwise waits forever for
+    // a reflection from a crashed node.
+    // Uses findActiveSessionsByNode() — O(sessions) single pass instead of
+    // the previous O(sessions × nodes) list-then-find pattern.
+    if (task.owner && collabStore) {
+      try {
+        const sessions = await collabStore.findActiveSessionsByNode(task.owner);
+        for (const session of sessions) {
+          const node = session.nodes.find(n => n.node_id === task.owner);
+          if (node && node.status !== 'dead') {
+            await collabStore.setNodeStatus(session.session_id, task.owner, 'dead');
+            log(`STALL → COLLAB: marked ${task.owner} as dead in session ${session.session_id}`);
+            await collabStore.appendAudit(session.session_id, 'node_marked_dead', {
+              node_id: task.owner, reason: `Stall detected: no heartbeat for ${silentMin}m`,
+            });
+
+            // Re-check if the round is now complete (dead nodes excluded)
+            const updated = await collabStore.get(session.session_id);
+            if (updated && collabStore.isRoundComplete(updated)) {
+              await evaluateRound(session.session_id);
+            }
+          }
+        }
+      } catch (err) {
+        log(`STALL → COLLAB ERROR: ${err.message}`);
+      }
+    }
+
     const releasedTask = await store.markReleased(
       task.task_id,
       `Stall detected: no agent heartbeat for ${silentMin}m, alive check failed`,
@@ -526,6 +596,26 @@ async function handleCollabFind(msg) {
   respond(msg, session);
 }
 
+/**
+ * mesh.collab.recruiting — List all sessions currently recruiting nodes.
+ * Used by agents to discover collab sessions they should join.
+ * Returns: array of { session_id, task_id, mode, min_nodes, max_nodes, current_nodes, recruiting_deadline }
+ */
+async function handleCollabRecruiting(msg) {
+  const recruiting = await collabStore.list({ status: COLLAB_STATUS.RECRUITING });
+  const summaries = recruiting.map(s => ({
+    session_id: s.session_id,
+    task_id: s.task_id,
+    mode: s.mode,
+    min_nodes: s.min_nodes,
+    max_nodes: s.max_nodes,
+    current_nodes: s.nodes.length,
+    node_ids: s.nodes.map(n => n.node_id || n.id),
+    recruiting_deadline: s.recruiting_deadline,
+  }));
+  respond(msg, summaries);
+}
+
 /**
  * mesh.collab.reflect — Node submits a reflection for the current round.
  * Expects: { session_id, node_id, summary, learnings, artifacts, confidence, vote }
@@ -546,8 +636,20 @@ async function handleCollabReflect(msg) {
   });
   publishCollabEvent('reflection_received', session);
 
-  // Check if all reflections are in → evaluate convergence
-  if (collabStore.isRoundComplete(session)) {
+  // Sequential mode: advance turn, notify next node or evaluate round
+  // Parallel mode: check if all reflections are in → evaluate convergence
+  // NOTE: Node.js single-threaded event loop prevents concurrent execution of this
+  // handler — no mutex needed. advanceTurn() is safe without CAS here.
+  if (session.mode === 'sequential') {
+    const nextNodeId = await collabStore.advanceTurn(session_id);
+    if (nextNodeId) {
+      // Notify only the next-turn node with accumulated intra-round intel
+      await notifySequentialTurn(session_id, nextNodeId);
+    } else {
+      // All turns done → evaluate round
+      await evaluateRound(session_id);
+    }
+  } else if (collabStore.isRoundComplete(session)) {
     await evaluateRound(session_id);
   }
 
@@ -657,8 +759,14 @@ async function startCollabRound(sessionId) {
   const scopeStrategy = session.scope_strategy || 'shared';
   const nodeScopes = computeNodeScopes(session.nodes, taskScope, scopeStrategy);
 
-  // Notify each node with their enforced scope
-  for (const node of session.nodes) {
+  // Sequential mode: only notify the current_turn node.
+  // Other nodes get notified via notifySequentialTurn() as turns advance.
+  // Parallel mode: notify all nodes at once.
+  const nodesToNotify = session.mode === 'sequential' && session.current_turn
+    ? session.nodes.filter(n => n.node_id === session.current_turn)
+    : session.nodes;
+
+  for (const node of nodesToNotify) {
     const effectiveScope = nodeScopes[node.node_id] || node.scope;
     nc.publish(`mesh.collab.${sessionId}.node.${node.node_id}.round`, sc.encode(JSON.stringify({
       session_id: sessionId,
@@ -674,6 +782,57 @@ async function startCollabRound(sessionId) {
   }
 }
 
+/**
+ * Notify the next node in a sequential turn.
+ * Includes intra-round reflections so far as additional shared intel.
+ */
+async function notifySequentialTurn(sessionId, nextNodeId) {
+  const session = await collabStore.get(sessionId);
+  if (!session) return;
+
+  const currentRound = session.rounds[session.rounds.length - 1];
+  if (!currentRound) return;
+
+  // Compile intra-round intel from reflections already submitted this round
+  const intraLines = [`=== INTRA-ROUND ${currentRound.round_number} (turns so far) ===\n`];
+  for (const r of currentRound.reflections) {
+    intraLines.push(`## Turn: ${r.node_id}${r.parse_failed ? ' [PARSE FAILED]' : ''}`);
+    if (r.summary) intraLines.push(`Summary: ${r.summary}`);
+    if (r.learnings) intraLines.push(`Learnings: ${r.learnings}`);
+    if (r.artifacts.length > 0) intraLines.push(`Artifacts: ${r.artifacts.join(', ')}`);
+    intraLines.push(`Confidence: ${r.confidence} | Vote: ${r.vote}`);
+    intraLines.push('');
+  }
+  const intraRoundIntel = intraLines.join('\n');
+  const combinedIntel = currentRound.shared_intel
+    ? currentRound.shared_intel + '\n\n' + intraRoundIntel
+    : intraRoundIntel;
+
+  const parentTask = await store.get(session.task_id);
+  const taskScope = parentTask?.scope || [];
+  const scopeStrategy = session.scope_strategy || 'shared';
+  const nodeScopes = computeNodeScopes(session.nodes, taskScope, scopeStrategy);
+  const nextNode = session.nodes.find(n => n.node_id === nextNodeId);
+
+  nc.publish(`mesh.collab.${sessionId}.node.${nextNodeId}.round`, sc.encode(JSON.stringify({
+    session_id: sessionId,
+    task_id: session.task_id,
+    round_number: currentRound.round_number,
+    shared_intel: combinedIntel,
+    my_scope: nodeScopes[nextNodeId] || nextNode?.scope || ['*'],
+    my_role: nextNode?.role || 'worker',
+    mode: 'sequential',
+    current_turn: nextNodeId,
+    scope_strategy: scopeStrategy,
+  })));
+
+  log(`COLLAB SEQ ${sessionId} R${currentRound.round_number}: Turn advanced to ${nextNodeId}`);
+  await collabStore.appendAudit(sessionId, 'turn_advanced', {
+    round: currentRound.round_number, next_node: nextNodeId,
+    reflections_so_far: currentRound.reflections.length,
+  });
+}
+
 /**
  * Evaluate the current round: check convergence, advance or complete.
  */
@@ -702,10 +861,11 @@ async function evaluateRound(sessionId) {
     await collabStore.markConverged(sessionId);
     publishCollabEvent('converged', session);
 
-    // Collect artifacts from all reflections
+    // Re-fetch after markConverged to ensure fresh state
+    const freshSession = await collabStore.get(sessionId);
     const allArtifacts = [];
     const contributions = {};
-    for (const round of session.rounds) {
+    for (const round of freshSession.rounds) {
       for (const r of round.reflections) {
         allArtifacts.push(...r.artifacts);
         contributions[r.node_id] = r.summary;
@@ -714,20 +874,20 @@ async function evaluateRound(sessionId) {
 
     await collabStore.markCompleted(sessionId, {
       artifacts: [...new Set(allArtifacts)],
-      summary: `Converged after ${session.current_round} rounds with ${session.nodes.length} nodes`,
+      summary: `Converged after ${freshSession.current_round} rounds with ${freshSession.nodes.length} nodes`,
       node_contributions: contributions,
     });
     await collabStore.appendAudit(sessionId, 'session_completed', {
-      outcome: 'converged', rounds: session.current_round,
+      outcome: 'converged', rounds: freshSession.current_round,
       artifacts: [...new Set(allArtifacts)].length,
-      node_count: session.nodes.length, recruited_count: session.recruited_count,
+      node_count: freshSession.nodes.length, recruited_count: freshSession.recruited_count,
     });
 
     // Complete the parent task
-    const updatedSession = await collabStore.get(sessionId);
-    await store.markCompleted(session.task_id, updatedSession.result);
-    publishEvent('completed', await store.get(session.task_id));
-    publishCollabEvent('completed', updatedSession);
+    const completedSession = await collabStore.get(sessionId);
+    await store.markCompleted(freshSession.task_id, completedSession.result);
+    publishEvent('completed', await store.get(freshSession.task_id));
+    publishCollabEvent('completed', completedSession);
 
   } else if (maxReached) {
     log(`COLLAB MAX ROUNDS ${sessionId}: ${session.current_round}/${session.max_rounds}. Completing with current artifacts.`);
@@ -955,6 +1115,19 @@ async function advancePlanWave(planId) {
   const waveNum = ready[0].wave;
   log(`PLAN WAVE ${planId} W${waveNum}: dispatching ${ready.length} subtasks`);
 
+  // Inherit routing fields from parent task so subtasks use the same LLM/node preferences.
+  // CONSTRAINT: Subtasks cannot override routing independently — they always inherit from the
+  // parent task. If per-subtask routing is needed, extend the subtask schema in mesh-plans.js
+  // (e.g. subtask.llm_provider) and merge here with subtask fields taking priority.
+  const parentTask = await store.get(plan.parent_task_id);
+  const inheritedRouting = {};
+  if (parentTask) {
+    if (parentTask.llm_provider) inheritedRouting.llm_provider = parentTask.llm_provider;
+    if (parentTask.llm_model) inheritedRouting.llm_model = parentTask.llm_model;
+    if (parentTask.preferred_nodes) inheritedRouting.preferred_nodes = parentTask.preferred_nodes;
+    if (parentTask.exclude_nodes) inheritedRouting.exclude_nodes = parentTask.exclude_nodes;
+  }
+
   for (const st of ready) {
     st.status = SUBTASK_STATUS.QUEUED;
 
@@ -962,7 +1135,7 @@ async function advancePlanWave(planId) {
     switch (st.delegation.mode) {
       case 'solo_mesh':
       case 'collab_mesh': {
-        // Submit as mesh task
+        // Submit as mesh task — inherit routing fields from parent task
         const meshTask = createTask({
           task_id: st.subtask_id,
           title: st.title,
@@ -973,6 +1146,7 @@ async function advancePlanWave(planId) {
           success_criteria: st.success_criteria,
           tags: ['plan', planId],
           collaboration: st.delegation.collaboration || undefined,
+          ...inheritedRouting,
         });
         await store.put(meshTask);
         st.mesh_task_id = meshTask.task_id;
@@ -1098,6 +1272,7 @@ async function main() {
     'mesh.collab.status':   handleCollabStatus,
     'mesh.collab.find':     handleCollabFind,
     'mesh.collab.reflect':  handleCollabReflect,
+    'mesh.collab.recruiting': handleCollabRecruiting,
     // Plan handlers
     'mesh.plans.create':          handlePlanCreate,
     'mesh.plans.get':             handlePlanGet,
@@ -1116,7 +1291,7 @@ async function main() {
         try {
           await handler(msg);
         } catch (err) {
-          log(`ERROR handling ${subject}: ${err.message}`);
+          log(`ERROR handling ${subject}: ${err.message}\n${err.stack}`);
           try { respondError(msg, err.message); } catch {}
         }
       }

package/bin/mesh.js

@@ -29,23 +29,44 @@ const path = require('path');
 const os = require('os');
 
 // ─── Config ──────────────────────────────────────────
-// NATS URL resolved via shared lib (env var → openclaw.env → .mesh-config → localhost fallback)
-const { NATS_URL, natsConnectOpts } = require('../lib/nats-resolve');
+// ── NATS URL resolution: env var → ~/.openclaw/openclaw.env → fallback IP ──
+const NATS_FALLBACK = 'nats://100.91.131.61:4222';
+function resolveNatsUrl() {
+  if (process.env.OPENCLAW_NATS) return process.env.OPENCLAW_NATS;
+  try {
+    const envFile = path.join(os.homedir(), '.openclaw', 'openclaw.env');
+    if (fs.existsSync(envFile)) {
+      const content = fs.readFileSync(envFile, 'utf8');
+      const match = content.match(/^\s*OPENCLAW_NATS\s*=\s*(.+)/m);
+      if (match && match[1].trim()) return match[1].trim();
+    }
+  } catch {}
+  return NATS_FALLBACK;
+}
+const NATS_URL = resolveNatsUrl();
 const SHARED_DIR = path.join(os.homedir(), 'openclaw', 'shared');
 const LOCAL_NODE = os.hostname().toLowerCase().replace(/[^a-z0-9-]/g, '-');
 const sc = StringCodec();
 
 // ─── Known nodes (for --node shortcuts) ──────────────
-// Load from ~/.openclaw/mesh-aliases.json if it exists, otherwise empty.
-let NODE_ALIASES = {};
-try {
-  const aliasFile = path.join(os.homedir(), '.openclaw', 'mesh-aliases.json');
-  if (fs.existsSync(aliasFile)) {
-    NODE_ALIASES = JSON.parse(fs.readFileSync(aliasFile, 'utf8'));
-  }
-} catch {
-  // File missing or malformed — proceed with no aliases
+const NODE_ALIASES_DEFAULTS = {
+  'ubuntu': 'calos-vmware-virtual-platform',
+  'linux': 'calos-vmware-virtual-platform',
+  'mac': 'moltymacs-virtual-machine-local',
+  'macos': 'moltymacs-virtual-machine-local',
+};
+
+function loadNodeAliases() {
+  const aliasPath = path.join(os.homedir(), '.openclaw', 'mesh-aliases.json');
+  try {
+    if (fs.existsSync(aliasPath)) {
+      const custom = JSON.parse(fs.readFileSync(aliasPath, 'utf8'));
+      return { ...NODE_ALIASES_DEFAULTS, ...custom };
+    }
+  } catch {}
+  return NODE_ALIASES_DEFAULTS;
 }
+const NODE_ALIASES = loadNodeAliases();
 
 /**
  * Resolve a node name — accepts aliases, full IDs, or "self"/"local"
@@ -98,7 +119,7 @@ function checkExecSafety(command) {
  */
 async function natsConnect() {
   try {
-    return await connect(natsConnectOpts({ timeout: 5000 }));
+    return await connect({ servers: NATS_URL, timeout: 5000 });
   } catch (err) {
     console.error(`Error: Cannot connect to NATS at ${NATS_URL}`);
     console.error(`Is the NATS server running? Is Tailscale connected?`);
@@ -140,21 +161,15 @@ async function collectHeartbeats(nc, waitMs = 3000) {
     uptime: os.uptime(),
   };
 
-  // Force-unsubscribe after deadline to prevent hanging if no messages arrive
-  const timer = setTimeout(() => sub.unsubscribe(), waitMs);
-
   // Listen for heartbeats for a few seconds
   const deadline = Date.now() + waitMs;
   for await (const msg of sub) {
-    try {
-      const s = JSON.parse(sc.decode(msg.data));
-      if (s.node !== LOCAL_NODE) {
-        nodes[s.node] = s;
-      }
-    } catch {}
+    const s = JSON.parse(sc.decode(msg.data));
+    if (s.node !== LOCAL_NODE) {
+      nodes[s.node] = s;
+    }
     if (Date.now() >= deadline) break;
   }
-  clearTimeout(timer);
   sub.unsubscribe();
   return nodes;
 }
@@ -382,6 +397,10 @@ async function cmdSubmit(args) {
       success_criteria: task.success_criteria || [],
       scope: task.scope || [],
       priority: task.auto_priority || 0,
+      llm_provider: task.provider || task.llm_provider || null,
+      llm_model: task.model || task.llm_model || null,
+      preferred_nodes: task.preferred_nodes || [],
+      exclude_nodes: task.exclude_nodes || [],
     });
     console.log(`Submitted: ${result.data.task_id} [${result.data.status}]`);
     // Mark as 'submitted' — NOT 'running'. The card reflects actual mesh state.
@@ -436,12 +455,18 @@ async function cmdSubmit(args) {
     scope: task.scope || [],
     priority: task.priority || 0,
     tags: task.tags || [],
+    llm_provider: task.provider || task.llm_provider || null,
+    llm_model: task.model || task.llm_model || null,
+    preferred_nodes: task.preferred_nodes || [],
+    exclude_nodes: task.exclude_nodes || [],
+    collaboration: task.collaboration || undefined,
   });
 
   console.log(`Submitted: ${result.data.task_id} "${result.data.title}"`);
   console.log(`  Status:  ${result.data.status}`);
   console.log(`  Budget:  ${result.data.budget_minutes}m`);
   console.log(`  Metric:  ${result.data.metric || 'none'}`);
+  if (result.data.llm_provider) console.log(`  Provider: ${result.data.llm_provider}`);
   await nc.close();
 }
 
@@ -575,6 +600,124 @@ async function cmdRepair(args) {
   }
 }
 
+/**
+ * mesh deploy [--force] [--component <name>] [--node <name>] — trigger fleet deploy.
+ *
+ * Publishes mesh.deploy.trigger to NATS. All nodes with mesh-deploy-listener
+ * will pull from git and self-deploy. Polls MESH_DEPLOY_RESULTS for status.
+ */
+async function cmdDeploy(args) {
+  const { execSync } = require('child_process');
+  // Prefer openclaw-node (git repo) over openclaw (runtime)
+  const defaultRepo = fs.existsSync(path.join(os.homedir(), 'openclaw-node', '.git'))
+    ? path.join(os.homedir(), 'openclaw-node')
+    : path.join(os.homedir(), 'openclaw');
+  const repoDir = process.env.OPENCLAW_REPO_DIR || defaultRepo;
+  const force = args.includes('--force');
+
+  // Parse --component flags
+  const components = [];
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--component' && args[i + 1]) {
+      components.push(args[i + 1]);
+      i++;
+    }
+  }
+
+  // Parse --node flags (target specific nodes, default: all)
+  const targetNodes = [];
+  for (let i = 0; i < args.length; i++) {
+    if (args[i] === '--node' && args[i + 1]) {
+      targetNodes.push(resolveNode(args[i + 1]));
+      i++;
+    }
+  }
+
+  // Get current SHA and branch
+  let sha, branch;
+  try {
+    sha = execSync('git rev-parse --short HEAD', { cwd: repoDir, encoding: 'utf8' }).trim();
+    branch = execSync('git rev-parse --abbrev-ref HEAD', { cwd: repoDir, encoding: 'utf8' }).trim();
+  } catch {
+    console.error(`Error: Cannot read git state from ${repoDir}`);
+    process.exit(1);
+  }
+
+  console.log(`Deploying ${sha} (${branch})${force ? ' [FORCE]' : ''}`);
+  if (components.length > 0) console.log(`  Components: ${components.join(', ')}`);
+  if (targetNodes.length > 0) console.log(`  Targets: ${targetNodes.join(', ')}`);
+  else console.log('  Targets: all nodes');
+
+  const nc = await natsConnect();
+
+  const trigger = {
+    sha,
+    branch,
+    components: components.length > 0 ? components : ['all'],
+    nodes: targetNodes.length > 0 ? targetNodes : ['all'],
+    force,
+    initiator: LOCAL_NODE,
+    timestamp: new Date().toISOString(),
+  };
+
+  // Write "latest" marker so offline nodes can catch up
+  try {
+    const js = nc.jetstream();
+    const resultsKv = await js.views.kv('MESH_DEPLOY_RESULTS', { history: 5, ttl: 7 * 24 * 60 * 60 * 1000 });
+    await resultsKv.put('latest', sc.encode(JSON.stringify({ sha, branch })));
+  } catch {}
+
+  // Publish trigger
+  nc.publish('mesh.deploy.trigger', sc.encode(JSON.stringify(trigger)));
+  await nc.flush();
+  console.log('Deploy trigger sent.\n');
+
+  // Poll for results (10s timeout)
+  console.log('Waiting for node responses...');
+  const deadline = Date.now() + 15000;
+  const seen = new Set();
+
+  try {
+    const js = nc.jetstream();
+    const resultsKv = await js.views.kv('MESH_DEPLOY_RESULTS');
+
+    while (Date.now() < deadline) {
+      // Check all nodes
+      const allAliasNodes = [...new Set(Object.values(NODE_ALIASES))];
+      const checkNodes = targetNodes.length > 0 ? targetNodes : allAliasNodes;
+
+      for (const nodeId of checkNodes) {
+        if (seen.has(nodeId)) continue;
+        const key = `${sha}-${nodeId}`;
+        try {
+          const entry = await resultsKv.get(key);
+          if (entry && entry.value) {
+            const result = JSON.parse(sc.decode(entry.value));
+            if (result.status === 'success' || result.status === 'failed' || result.status === 'skipped') {
+              const icon = result.status === 'success' ? '\x1b[32m✓\x1b[0m' : result.status === 'skipped' ? '\x1b[33m-\x1b[0m' : '\x1b[31m✗\x1b[0m';
+              console.log(`  ${icon} ${nodeId}: ${result.status} (${result.durationSeconds || 0}s)`);
+              if (result.errors && result.errors.length > 0) {
+                for (const e of result.errors) console.log(`    Error: ${e}`);
+              }
+              seen.add(nodeId);
+            }
+          }
+        } catch {}
+      }
+
+      if (seen.size >= checkNodes.length) break;
+      await new Promise(r => setTimeout(r, 2000));
+    }
+  } catch {}
+
+  if (seen.size === 0) {
+    console.log('  (no responses yet — nodes may still be deploying)');
+  }
+
+  console.log('');
+  await nc.close();
+}
+
 /**
  * mesh help — show usage.
  */
@@ -602,6 +745,10 @@ function cmdHelp() {
     '  mesh health --json                      Health check (JSON output)',
     '  mesh repair                             Self-repair this node',
     '  mesh repair --all                       Self-repair ALL nodes',
+    '  mesh deploy                             Deploy to all nodes',
+    '  mesh deploy --force                     Force deploy (skip cache)',
+    '  mesh deploy --node ubuntu               Deploy to specific node',
+    '  mesh deploy --component mesh-daemons    Deploy specific component',
     '',
     'NODE ALIASES:',
     '  ubuntu, linux   = Ubuntu VM (calos-vmware-virtual-platform)',
@@ -632,6 +779,7 @@ async function main() {
     case 'tasks':     return cmdTasks(args);
     case 'health':    return cmdHealth(args);
     case 'repair':    return cmdRepair(args);
+    case 'deploy':    return cmdDeploy(args);
     case 'help':
     case '--help':
     case '-h':        return cmdHelp();