openclaw-node-harness 2.0.3 → 2.1.0

package/mission-control/src/app/api/mesh/events/route.ts

@@ -1,10 +1,34 @@
-import { getNats, sc } from "@/lib/nats";
+import { getNats, getTasksKv, sc } from "@/lib/nats";
 
 export const dynamic = "force-dynamic";
 
 /**
- * SSE endpoint that subscribes to mesh.events.> and pushes to connected browsers.
- * Falls back gracefully if NATS is unavailable.
+ * MANUAL VERIFICATION PROTOCOL (SSE dual-iterator cleanup):
+ *
+ * The zombie watcher leak is the highest-risk bug in this route.
+ * It cannot be unit tested (no real HTTP server in Vitest).
+ *
+ * To verify after deployment:
+ * 1. Open MC on worker node in browser
+ * 2. Open DevTools > Network tab > filter "events" (SSE stream)
+ * 3. Trigger a task state change on the lead (create/complete a task)
+ * 4. Verify the SSE event arrives in the stream ("kv.task.updated" event type)
+ * 5. Close the browser tab
+ * 6. Check NATS connections: curl http://localhost:8222/connz | jq '.connections | length'
+ *    -> Connection count should drop by 1 (the SSE watcher + sub are cleaned up)
+ * 7. Repeat with 3 tabs open -> close all -> verify count drops by 3
+ *
+ * If connection count doesn't drop: the cancel() handler isn't calling watcher.stop()
+ * or sub.unsubscribe(). Check the dual-iterator lifecycle in this file.
+ */
+
+/**
+ * SSE endpoint: dual-iterator (NATS pub/sub + KV watcher).
+ *
+ * Iterator 1: NATS subscription on mesh.events.> (task lifecycle events)
+ * Iterator 2: KV watcher on MESH_TASKS (real-time task state changes)
+ *
+ * Both feed into a single SSE stream. On client disconnect, both are cleaned up.
  */
 export async function GET() {
   const nc = await getNats();
@@ -13,16 +37,17 @@ export async function GET() {
   }
 
   const sub = nc.subscribe("mesh.events.>");
+  const kv = await getTasksKv();
+  let watcher: any = null;
   let closed = false;
 
   const stream = new ReadableStream({
     async start(controller) {
       const encoder = new TextEncoder();
 
-      // Send initial keepalive
       controller.enqueue(encoder.encode(": connected\n\n"));
 
-      // Keepalive every 30s to prevent proxy/browser timeout
+      // Keepalive every 30s
       const keepalive = setInterval(() => {
         if (!closed) {
           try {
@@ -33,26 +58,77 @@ export async function GET() {
         }
       }, 30000);
 
-      try {
-        for await (const msg of sub) {
-          if (closed) break;
-          try {
-            const data = sc.decode(msg.data);
-            const eventType = msg.subject.replace("mesh.events.", "");
-            controller.enqueue(
-              encoder.encode(`event: ${eventType}\ndata: ${data}\n\n`)
-            );
-          } catch {
-            // skip malformed messages
+      // Iterator 1: NATS pub/sub events
+      (async () => {
+        try {
+          for await (const msg of sub) {
+            if (closed) break;
+            try {
+              const data = sc.decode(msg.data);
+              const eventType = msg.subject.replace("mesh.events.", "");
+              controller.enqueue(
+                encoder.encode(`event: ${eventType}\ndata: ${data}\n\n`)
+              );
+            } catch {
+              // skip malformed
+            }
           }
+        } catch {
+          // subscription ended
+        }
+      })();
+
+      // Iterator 2: KV watcher (task state changes)
+      if (kv) {
+        try {
+          watcher = await kv.watch();
+          (async () => {
+            try {
+              for await (const entry of watcher) {
+                if (closed) break;
+                try {
+                  const task = entry.value
+                    ? JSON.parse(sc.decode(entry.value))
+                    : null;
+                  const payload = JSON.stringify({
+                    key: entry.key,
+                    operation: entry.value ? "PUT" : "DEL",
+                    task,
+                    revision: entry.revision,
+                  });
+                  controller.enqueue(
+                    encoder.encode(`event: kv.task.updated\ndata: ${payload}\n\n`)
+                  );
+                } catch {
+                  // skip malformed
+                }
+              }
+            } catch {
+              // watcher ended
+            }
+          })();
+        } catch {
+          // KV watch failed — continue with pub/sub only
         }
-      } finally {
-        clearInterval(keepalive);
       }
+
+      // Wait for cancel (client disconnect)
+      await new Promise<void>((resolve) => {
+        const check = setInterval(() => {
+          if (closed) {
+            clearInterval(check);
+            clearInterval(keepalive);
+            resolve();
+          }
+        }, 1000);
+      });
     },
     cancel() {
       closed = true;
       sub.unsubscribe();
+      if (watcher && typeof watcher.stop === "function") {
+        watcher.stop();
+      }
     },
   });
 
@@ -61,7 +137,7 @@ export async function GET() {
       "Content-Type": "text/event-stream",
       "Cache-Control": "no-cache, no-transform",
       Connection: "keep-alive",
-      "X-Accel-Buffering": "no", // disable nginx buffering
+      "X-Accel-Buffering": "no",
     },
   });
 }

package/mission-control/src/app/api/mesh/identity/route.ts

@@ -0,0 +1,11 @@
+import { NODE_ID, NODE_ROLE, NODE_PLATFORM } from "@/lib/config";
+
+export const dynamic = "force-dynamic";
+
+export async function GET() {
+  return Response.json({
+    nodeId: NODE_ID,
+    role: NODE_ROLE,
+    platform: NODE_PLATFORM,
+  });
+}

package/mission-control/src/app/api/mesh/tasks/[id]/route.ts

@@ -0,0 +1,92 @@
+import { getTasksKv, sc } from "@/lib/nats";
+import { NODE_ID, NODE_ROLE } from "@/lib/config";
+
+export const dynamic = "force-dynamic";
+
+/**
+ * GET /api/mesh/tasks/:id — Get a single task from KV.
+ */
+export async function GET(
+  _req: Request,
+  { params }: { params: Promise<{ id: string }> }
+) {
+  const { id } = await params;
+  const kv = await getTasksKv();
+  if (!kv) {
+    return Response.json({ error: "NATS unavailable" }, { status: 503 });
+  }
+
+  const entry = await kv.get(id);
+  if (!entry?.value) {
+    return Response.json({ error: "not found" }, { status: 404 });
+  }
+
+  const task = JSON.parse(sc.decode(entry.value));
+  return Response.json({ task, revision: entry.revision });
+}
+
+/**
+ * PATCH /api/mesh/tasks/:id — Update a task with CAS.
+ *
+ * Authority rules:
+ * - Lead can update any task
+ * - Workers can only update tasks they originated
+ * - Revision must match (CAS) to prevent stale writes
+ */
+export async function PATCH(
+  req: Request,
+  { params }: { params: Promise<{ id: string }> }
+) {
+  const { id } = await params;
+  const kv = await getTasksKv();
+  if (!kv) {
+    return Response.json({ error: "NATS unavailable" }, { status: 503 });
+  }
+
+  const body = await req.json();
+  const { revision, ...updates } = body;
+
+  if (!revision) {
+    return Response.json({ error: "revision is required for CAS update" }, { status: 400 });
+  }
+
+  // Read current state
+  const entry = await kv.get(id);
+  if (!entry?.value) {
+    return Response.json({ error: "not found" }, { status: 404 });
+  }
+
+  const current = JSON.parse(sc.decode(entry.value));
+
+  // Authority check
+  if (NODE_ROLE !== "lead" && current.origin !== NODE_ID) {
+    return Response.json(
+      { error: "workers can only update tasks they originated" },
+      { status: 403 }
+    );
+  }
+
+  // Merge updates
+  const updated = { ...current, ...updates };
+
+  // CAS write
+  try {
+    await kv.update(id, sc.encode(JSON.stringify(updated)), revision);
+  } catch (err: any) {
+    // Revision mismatch — return current state so client can retry
+    const freshEntry = await kv.get(id);
+    const freshTask = freshEntry?.value
+      ? JSON.parse(sc.decode(freshEntry.value))
+      : null;
+    return Response.json(
+      {
+        error: `revision mismatch: expected ${revision}, got ${entry.revision}`,
+        currentTask: freshTask,
+        currentRevision: freshEntry?.revision,
+      },
+      { status: 409 }
+    );
+  }
+
+  return Response.json({ ok: true, task: updated });
+}

package/mission-control/src/app/api/mesh/tasks/route.ts

@@ -0,0 +1,91 @@
+import { getTasksKv, sc } from "@/lib/nats";
+import { NODE_ID, NODE_ROLE } from "@/lib/config";
+import crypto from "crypto";
+
+export const dynamic = "force-dynamic";
+
+/**
+ * GET /api/mesh/tasks — List all tasks from NATS KV.
+ * Available on all nodes (read from shared KV).
+ */
+export async function GET() {
+  const kv = await getTasksKv();
+  if (!kv) {
+    return Response.json({ tasks: [], natsAvailable: false });
+  }
+
+  const tasks: any[] = [];
+  const keys = await kv.keys();
+  for await (const key of keys) {
+    const entry = await kv.get(key);
+    if (!entry?.value) continue;
+    try {
+      tasks.push(JSON.parse(sc.decode(entry.value)));
+    } catch {
+      // skip malformed
+    }
+  }
+
+  tasks.sort((a, b) => {
+    if ((b.priority || 0) !== (a.priority || 0)) return (b.priority || 0) - (a.priority || 0);
+    return new Date(a.created_at || 0).getTime() - new Date(b.created_at || 0).getTime();
+  });
+
+  return Response.json({ tasks, natsAvailable: true });
+}
+
+/**
+ * POST /api/mesh/tasks — Propose a new task.
+ *
+ * On lead: creates directly with status "queued".
+ * On worker: creates with status "proposed" — daemon validates within 30s.
+ */
+export async function POST(req: Request) {
+  const kv = await getTasksKv();
+  if (!kv) {
+    return Response.json({ error: "NATS unavailable" }, { status: 503 });
+  }
+
+  const body = await req.json();
+  if (!body.title) {
+    return Response.json({ error: "title is required" }, { status: 400 });
+  }
+
+  const suffix = crypto.randomBytes(3).toString("hex");
+  const now = new Date();
+  const dateStr =
+    now.getFullYear().toString() +
+    (now.getMonth() + 1).toString().padStart(2, "0") +
+    now.getDate().toString().padStart(2, "0");
+  const taskId = body.task_id || `T-${dateStr}-${suffix}`;
+
+  const status = NODE_ROLE === "lead" ? "queued" : "proposed";
+
+  const task = {
+    task_id: taskId,
+    title: body.title,
+    description: body.description || "",
+    status,
+    origin: NODE_ID,
+    owner: null,
+    priority: body.priority || 0,
+    budget_minutes: body.budget_minutes || 30,
+    metric: body.metric || null,
+    success_criteria: body.success_criteria || [],
+    scope: body.scope || [],
+    tags: body.tags || [],
+    preferred_nodes: body.preferred_nodes || [],
+    exclude_nodes: body.exclude_nodes || [],
+    created_at: now.toISOString(),
+    claimed_at: null,
+    started_at: null,
+    completed_at: null,
+    last_activity: null,
+    result: null,
+    attempts: [],
+  };
+
+  await kv.put(taskId, sc.encode(JSON.stringify(task)));
+
+  return Response.json({ ok: true, task }, { status: 201 });
+}

package/mission-control/src/app/api/tasks/[id]/handoff/route.ts

@@ -39,7 +39,7 @@ export async function POST(
       return NextResponse.json({ error: "Task not found" }, { status: 404 });
     }
 
-    const fromSoul = task.soulId || "daedalus";
+    const fromSoul = task.soulId || "main-agent";
 
     // Create handoff document in ClawVault
     await fs.mkdir(HANDOFFS_DIR, { recursive: true });

package/mission-control/src/app/api/tasks/[id]/route.ts

@@ -6,6 +6,8 @@ import { statusToKanban, kanbanToStatus } from "@/lib/parsers/task-markdown";
 import { syncTasksToMarkdown } from "@/lib/sync/tasks";
 import { logActivity } from "@/lib/activity";
 import { gatewayNotify } from "@/lib/gateway-notify";
+import { AGENT_NAME, HUMAN_NAME } from "@/lib/config";
+import { getNats, sc } from "@/lib/nats";
 
 /**
  * Push a notification message to the OpenClaw TUI via gateway chat.send + abort.
@@ -17,6 +19,22 @@ function notifyAgent(text: string) {
   });
 }
 
+/**
+ * Cancel a running mesh task via NATS. Best-effort — ignores failures.
+ */
+async function cancelMeshTask(taskId: string, execution: string | null, status: string) {
+  if (execution === "mesh" && status !== "done" && status !== "cancelled") {
+    const nc = await getNats();
+    if (nc) {
+      nc.request(
+        "mesh.tasks.cancel",
+        sc.encode(JSON.stringify({ task_id: taskId })),
+        { timeout: 5000 }
+      ).catch(() => {});
+    }
+  }
+}
+
 /**
  * DELETE /api/tasks/[id]
  * Delete a task (and its children) by ID.
@@ -58,6 +76,9 @@ export async function DELETE(
     collectDescendants(id);
     idsToDelete.push(id); // add the root task last
 
+    // Cancel mesh task if running
+    await cancelMeshTask(id, existing.execution, existing.status);
+
     // Delete all collected tasks + their dependency edges
     for (const delId of idsToDelete) {
       db.delete(dependencies)
@@ -109,6 +130,22 @@ export async function PATCH(
     const body = await request.json();
     const now = new Date().toISOString();
 
+    // Transition guard: block execution mode changes on in-flight mesh tasks
+    if (body.execution !== undefined && body.execution !== existing.execution) {
+      if (existing.meshTaskId) {
+        return NextResponse.json(
+          { error: "Cannot change execution mode after task has been submitted to mesh" },
+          { status: 400 }
+        );
+      }
+      if (existing.status !== "queued" && existing.status !== "ready") {
+        return NextResponse.json(
+          { error: `Cannot change execution mode while task is ${existing.status}` },
+          { status: 400 }
+        );
+      }
+    }
+
     // Build the update set from provided fields
     const update: Record<string, unknown> = { updatedAt: now };
 
@@ -130,6 +167,17 @@ export async function PATCH(
         update.status = kanbanToStatus(body.kanbanColumn);
       }
     }
+
+    // Done-gate: only the human operator can mark tasks as done (via force_done flag).
+    // Without force_done, redirect done→review so tasks land in waiting-user.
+    const targetStatus = update.status as string | undefined;
+    const targetColumn = update.kanbanColumn as string | undefined;
+    if (!body.force_done) {
+      if (targetStatus === "done" || targetColumn === "done") {
+        update.status = "waiting-user";
+        update.kanbanColumn = "review";
+      }
+    }
     if (body.owner !== undefined) {
       update.owner = body.owner || null;
     }
@@ -203,18 +251,56 @@ export async function PATCH(
     if (body.acknowledged_at !== undefined) {
       update.acknowledgedAt = body.acknowledged_at || null;
     }
+    // Mesh execution fields
+    if (body.execution !== undefined) {
+      update.execution = body.execution || null;
+    }
+    if (body.collaboration !== undefined) {
+      update.collaboration = body.collaboration ? JSON.stringify(body.collaboration) : null;
+    }
+    if (body.preferred_nodes !== undefined) {
+      update.preferredNodes = body.preferred_nodes?.length ? JSON.stringify(body.preferred_nodes) : null;
+    }
+    if (body.exclude_nodes !== undefined) {
+      update.excludeNodes = body.exclude_nodes?.length ? JSON.stringify(body.exclude_nodes) : null;
+    }
+    if (body.cluster_id !== undefined) {
+      update.clusterId = body.cluster_id || null;
+    }
+    if (body.metric !== undefined) {
+      update.metric = body.metric || null;
+    }
+    if (body.budget_minutes !== undefined) {
+      update.budgetMinutes = body.budget_minutes;
+    }
+    if (body.scope !== undefined) {
+      update.scope = body.scope?.length ? JSON.stringify(body.scope) : null;
+    }
 
     // Auto-set owner when task moves to running manually (no explicit owner change)
     const effectiveStatus = (update.status as string) ?? existing.status;
     if (effectiveStatus === "running" && body.owner === undefined && !existing.owner) {
-      update.owner = "Gui";
+      update.owner = HUMAN_NAME;
     }
 
     db.update(tasks).set(update).where(eq(tasks.id, id)).run();
 
+    // Cancel mesh task if status changed to cancelled
+    if (update.status === "cancelled") {
+      await cancelMeshTask(id, existing.execution, existing.status);
+    }
+
     // Sync back to markdown
     syncTasksToMarkdown(db);
 
+    // Wake bridge when execution is set to mesh (for immediate pickup)
+    const effectiveExecution = (update.execution as string) ?? existing.execution;
+    const effectiveApprovalNum = (update.needsApproval as number) ?? existing.needsApproval;
+    if (effectiveExecution === "mesh" && effectiveApprovalNum === 0) {
+      const nc = await getNats();
+      if (nc) nc.publish("mesh.bridge.wake", sc.encode(""));
+    }
+
     // Log activity
     const movedTo = body.kanban_column || body.kanbanColumn;
     const parts: string[] = [];
@@ -229,14 +315,14 @@ export async function PATCH(
       id
     );
 
-    // Push notification to TUI only for Daedalus autostart tasks
+    // Push notification to TUI only for agent autostart tasks
     if (movedTo || body.status) {
       const effectiveOwner = (update.owner as string) ?? existing.owner;
       const effectiveApproval = (update.needsApproval as number) ?? existing.needsApproval;
-      const isDaedalus = effectiveOwner === "Daedalus";
+      const isAgent = effectiveOwner === AGENT_NAME;
       const isAutostart = effectiveApproval === 0;
 
-      if (isDaedalus && isAutostart) {
+      if (isAgent && isAutostart) {
         const fromCol = existing.kanbanColumn || existing.status;
         const toCol = movedTo || body.status;
         notifyAgent(`MC-Kanban: "${existing.title}" has been moved from ${fromCol} to ${toCol}`);

package/mission-control/src/app/api/tasks/route.ts

@@ -7,7 +7,9 @@ import { statusToKanban } from "@/lib/parsers/task-markdown";
 import { syncTasksFromMarkdownIfChanged, syncTasksToMarkdown } from "@/lib/sync/tasks";
 import { logActivity } from "@/lib/activity";
 import { schedulerTick } from "@/lib/scheduler";
-import { WORKSPACE_ROOT } from "@/lib/config";
+import { generateTaskId } from "@/lib/task-id";
+import { getNats, sc } from "@/lib/nats";
+import { WORKSPACE_ROOT, AGENT_NAME } from "@/lib/config";
 import path from "path";
 
 /**
@@ -67,7 +69,7 @@ export async function GET(request: NextRequest) {
           title: liveWork.title,
           status: "running",
           kanbanColumn: "in_progress",
-          owner: "daedalus",
+          owner: AGENT_NAME.toLowerCase(),
           nextAction: liveWork.nextAction || null,
           updatedAt: new Date().toISOString(),
           createdAt: new Date().toISOString(),
@@ -186,6 +188,15 @@ export async function POST(request: NextRequest) {
         isRecurring: body.is_recurring ? 1 : 0,
         capacityClass: body.capacity_class || "normal",
         autoPriority: body.auto_priority || 0,
+        // Execution fields
+        execution: body.execution || null,
+        collaboration: body.collaboration ? JSON.stringify(body.collaboration) : null,
+        preferredNodes: body.preferred_nodes?.length ? JSON.stringify(body.preferred_nodes) : null,
+        excludeNodes: body.exclude_nodes?.length ? JSON.stringify(body.exclude_nodes) : null,
+        clusterId: body.cluster_id || null,
+        metric: body.metric || null,
+        budgetMinutes: body.budget_minutes || 30,
+        scope: body.scope?.length ? JSON.stringify(body.scope) : null,
         updatedAt: now.toISOString(),
         createdAt: now.toISOString(),
       })
@@ -200,6 +211,13 @@ export async function POST(request: NextRequest) {
     // Immediately run scheduler tick so auto-start tasks dispatch without waiting for poll
     schedulerTick();
 
+    // Wake bridge for mesh tasks
+    if (body.execution === "mesh") {
+      getNats().then((nc) => {
+        if (nc) nc.publish("mesh.bridge.wake", sc.encode(""));
+      }).catch(() => {});
+    }
+
     const created = db
       .select()
       .from(tasks)
@@ -223,31 +241,4 @@ export async function POST(request: NextRequest) {
       { status: 500 }
     );
   }
-}
-
-/**
- * Generate a task ID in the format T-YYYYMMDD-NNN.
- * NNN is a zero-padded sequence number based on existing tasks for that date.
- */
-function generateTaskId(db: ReturnType<typeof getDb>, date: Date): string {
-  const dateStr =
-    date.getFullYear().toString() +
-    (date.getMonth() + 1).toString().padStart(2, "0") +
-    date.getDate().toString().padStart(2, "0");
-
-  const prefix = `T-${dateStr}-`;
-
-  // Find highest existing sequence number for this date prefix
-  const existing = db
-    .select({ id: tasks.id })
-    .from(tasks)
-    .all()
-    .filter((t) => t.id.startsWith(prefix))
-    .map((t) => {
-      const seq = parseInt(t.id.slice(prefix.length), 10);
-      return isNaN(seq) ? 0 : seq;
-    });
-
-  const nextSeq = existing.length > 0 ? Math.max(...existing) + 1 : 1;
-  return `${prefix}${nextSeq.toString().padStart(3, "0")}`;
-}
+}