clawmatrix 0.1.15 → 0.1.17

package/src/handoff.ts

@@ -5,16 +5,25 @@ import type {
   HandoffRequest,
   HandoffResponse,
   HandoffStreamChunk,
+  HandoffCancel,
+  HandoffStatusQuery,
+  HandoffStatusResponse,
+  HandoffInputRequired,
+  HandoffInput,
+  HandoffStatus,
 } from "./types.ts";
 
 const DEFAULT_HANDOFF_TIMEOUT = 600_000; // 10 minutes (resets on each stream chunk)
 const MAX_RETRIES = 2;
+const INPUT_REQUIRED_TTL = 1_800_000; // 30 minutes — max time to wait for user reply
+const STALE_CLEANUP_INTERVAL = 120_000; // check every 2 minutes
 
 interface PendingHandoff {
   resolve: (result: HandoffResponse["payload"]) => void;
   reject: (error: Error) => void;
   timer: ReturnType<typeof setTimeout>;
   target: string;
+  targetNodeId: string;
   retriesLeft: number;
   task: string;
   context?: string;
@@ -22,14 +31,65 @@ interface PendingHandoff {
   onStream?: (delta: string) => void;
 }
 
+interface ActiveHandoff {
+  proc: { kill: () => void; exited: Promise<number> } | null;
+  status: HandoffStatus;
+  sessionId: string;
+  agent: string;
+  startedAt: number;
+  inputRequiredAt: number | null; // when status changed to input_required
+  from: string; // requesting nodeId
+}
+
 export class HandoffManager {
   private config: ClawMatrixConfig;
   private peerManager: PeerManager;
   private pending = new Map<string, PendingHandoff>();
+  private active = new Map<string, ActiveHandoff>();
+  private inputRequiredTargets = new Map<string, string>(); // handoffId → targetNodeId
+  private staleCleanupTimer: ReturnType<typeof setInterval> | null = null;
 
   constructor(config: ClawMatrixConfig, peerManager: PeerManager) {
     this.config = config;
     this.peerManager = peerManager;
+
+    // Periodically clean up stale input_required entries
+    this.staleCleanupTimer = setInterval(() => this.cleanupStale(), STALE_CLEANUP_INTERVAL);
+  }
+
+  /** Remove stale input_required and canceled entries that exceeded TTL. */
+  private cleanupStale() {
+    const now = Date.now();
+    for (const [id, entry] of this.active) {
+      if (entry.status === "input_required" && entry.inputRequiredAt && now - entry.inputRequiredAt > INPUT_REQUIRED_TTL) {
+        this.active.delete(id);
+        // Notify the requester that the handoff timed out
+        this.peerManager.sendTo(entry.from, {
+          type: "handoff_res",
+          id,
+          from: this.config.nodeId,
+          to: entry.from,
+          timestamp: now,
+          payload: {
+            success: false,
+            nodeId: this.config.nodeId,
+            agent: entry.agent,
+            error: "Handoff timed out waiting for input",
+          },
+        } satisfies HandoffResponse);
+      }
+      // Clean up canceled entries that were never cleaned by runAgentTurn
+      // (e.g. cancel arrived during input_required when no process was running)
+      if (entry.status === "canceled") {
+        this.active.delete(id);
+      }
+    }
+    // Clean stale inputRequiredTargets (requester side)
+    for (const [handoffId] of this.inputRequiredTargets) {
+      if (!this.pending.has(handoffId)) {
+        this.inputRequiredTargets.delete(handoffId);
+      }
+    }
   }
 
   /** Send a handoff request and wait for the response. */
@@ -65,7 +125,7 @@ export class HandoffManager {
     return new Promise<HandoffResponse["payload"]>((resolve, reject) => {
       const timer = this.createTimeout(id, targetNodeId, target, task, context, retriesLeft, resolve, reject, onStream);
 
-      this.pending.set(id, { resolve, reject, timer, target, retriesLeft, task, context, accumulated: "", onStream });
+      this.pending.set(id, { resolve, reject, timer, target, targetNodeId, retriesLeft, task, context, accumulated: "", onStream });
 
       const frame: HandoffRequest = {
         type: "handoff_req",
@@ -200,26 +260,94 @@ export class HandoffManager {
       return;
     }
 
-    try {
-      // Execute via openclaw agent subprocess
-      const message = payload.context
-        ? `${payload.task}\n\nContext:\n${payload.context}`
-        : payload.task;
+    const sessionId = `handoff-${id}`;
+    const message = payload.context
+      ? `${payload.task}\n\nContext:\n${payload.context}`
+      : payload.task;
+
+    const activeEntry: ActiveHandoff = {
+      proc: null,
+      status: "working",
+      sessionId,
+      agent: agent.id,
+      startedAt: Date.now(),
+      inputRequiredAt: null,
+      from,
+    };
+    this.active.set(id, activeEntry);
+
+    await this.runAgentTurn(id, agent.id, message, sessionId, from, activeEntry);
+  }
+
+  /** Handle incoming handoff_input (resume agent with user reply). */
+  async handleInput(frame: HandoffInput): Promise<void> {
+    const entry = this.active.get(frame.id);
+    if (!entry || entry.status !== "input_required") return;
+
+    // Verify the input comes from the original requester
+    if (frame.from !== entry.from) return;
+
+    entry.status = "working";
+    await this.runAgentTurn(frame.id, entry.agent, frame.payload.message, entry.sessionId, entry.from, entry);
+  }
 
+  /** Shared logic: spawn agent subprocess, stream output, handle input_required or completion. */
+  private async runAgentTurn(
+    id: string,
+    agent: string,
+    message: string,
+    sessionId: string,
+    from: string,
+    activeEntry: ActiveHandoff,
+  ): Promise<void> {
+    try {
       const proc = spawnProcess(
-        ["openclaw", "agent", "--agent", agent.id, "--message", message],
+        ["openclaw", "agent", "--agent", agent, "--message", message, "--session-id", sessionId],
         { stdout: "pipe", stderr: "pipe" },
       );
+      activeEntry.proc = proc;
+
+      // If cancel arrived between active.set() and proc assignment, kill immediately
+      if (activeEntry.status === "canceled") {
+        proc.kill();
+        this.active.delete(id);
+        return;
+      }
 
-      // Stream stdout chunks back to the caller
-      const fullOutput = await this.streamStdout(proc.stdout, id, from);
-      const exitCode = await proc.exited;
+      // Consume stderr concurrently to prevent pipe buffer deadlock
+      const stderrPromise = proc.stderr ? drainStream(proc.stderr) : Promise.resolve("");
+      const { output, inputRequired } = await this.streamStdout(proc.stdout, id, from);
+      const [exitCode, stderr] = await Promise.all([proc.exited, stderrPromise]);
+
+      if (activeEntry.status === "canceled") {
+        this.active.delete(id);
+        return;
+      }
+
+      if (inputRequired) {
+        const marker = "[INPUT_REQUIRED]";
+        const question = output.slice(output.indexOf(marker) + marker.length).trim();
+        activeEntry.status = "input_required";
+        activeEntry.inputRequiredAt = Date.now();
+
+        this.peerManager.sendTo(from, {
+          type: "handoff_input_required",
+          id,
+          from: this.config.nodeId,
+          to: from,
+          timestamp: Date.now(),
+          payload: { message: question },
+        } satisfies HandoffInputRequired);
+        return;
+      }
 
       if (exitCode !== 0) {
-        const stderr = proc.stderr ? await new Response(proc.stderr).text() : "";
         throw new Error(`Agent exited with code ${exitCode}: ${stderr}`);
       }
 
+      activeEntry.status = "completed";
+      this.active.delete(id);
+
       this.peerManager.sendTo(from, {
         type: "handoff_res",
         id,
@@ -229,11 +357,18 @@ export class HandoffManager {
         payload: {
           success: true,
           nodeId: this.config.nodeId,
-          agent: agent.id,
-          result: fullOutput.trim(),
+          agent,
+          result: output.trim(),
         },
       } satisfies HandoffResponse);
     } catch (err) {
+      if (activeEntry.status === "canceled") {
+        this.active.delete(id);
+        return;
+      }
+      activeEntry.status = "failed";
+      this.active.delete(id);
+
       this.peerManager.sendTo(from, {
         type: "handoff_res",
         id,
@@ -243,7 +378,7 @@ export class HandoffManager {
         payload: {
           success: false,
           nodeId: this.config.nodeId,
-          agent: agent.id,
+          agent,
           error: err instanceof Error ? err.message : String(err),
         },
       } satisfies HandoffResponse);
@@ -255,8 +390,8 @@ export class HandoffManager {
     stdout: ReadableStream | null,
     handoffId: string,
     to: string,
-  ): Promise<string> {
-    if (!stdout) return "";
+  ): Promise<{ output: string; inputRequired: boolean }> {
+    if (!stdout) return { output: "", inputRequired: false };
 
     const reader = stdout.getReader();
     const decoder = new TextDecoder();
@@ -283,6 +418,11 @@ export class HandoffManager {
       reader.releaseLock();
     }
 
+    const INPUT_MARKER = "[INPUT_REQUIRED]";
+    if (full.indexOf(INPUT_MARKER) !== -1) {
+      return { output: full, inputRequired: true };
+    }
+
     // Send final done marker
     this.peerManager.sendTo(to, {
       type: "handoff_stream",
@@ -293,15 +433,169 @@ export class HandoffManager {
       payload: { delta: "", done: true },
     } satisfies HandoffStreamChunk);
 
-    return full;
+    return { output: full, inputRequired: false };
+  }
+
+  /** Handle incoming input_required from remote (requester side). */
+  handleInputRequired(frame: HandoffInputRequired) {
+    const pending = this.pending.get(frame.id);
+    if (!pending) return;
+
+    clearTimeout(pending.timer);
+    this.pending.delete(frame.id);
+
+    // Cache target for sendHandoffInput
+    this.inputRequiredTargets.set(frame.id, frame.from);
+
+    pending.resolve({
+      success: true,
+      nodeId: frame.from,
+      result: frame.payload.message,
+      inputRequired: true,
+      handoffId: frame.id,
+    });
+  }
+
+  /** Send a reply to a remote agent that requested more input (requester side). */
+  sendHandoffInput(handoffId: string, message: string): Promise<HandoffResponse["payload"]> {
+    const targetNodeId = this.inputRequiredTargets.get(handoffId);
+    if (!targetNodeId) throw new Error(`No pending input_required for handoff ${handoffId}`);
+
+    this.inputRequiredTargets.delete(handoffId);
+
+    return new Promise<HandoffResponse["payload"]>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(handoffId);
+        reject(new Error("Handoff reply timed out"));
+      }, this.config.handoffTimeout ?? DEFAULT_HANDOFF_TIMEOUT);
+
+      this.pending.set(handoffId, {
+        resolve, reject, timer,
+        target: "", targetNodeId,
+        retriesLeft: 0, task: message, accumulated: "",
+      });
+
+      this.peerManager.sendTo(targetNodeId, {
+        type: "handoff_input",
+        id: handoffId,
+        from: this.config.nodeId,
+        to: targetNodeId,
+        timestamp: Date.now(),
+        payload: { message },
+      } as HandoffInput);
+    });
+  }
+
+  /** Handle incoming cancel request (receiver side — kill subprocess). */
+  handleCancel(frame: HandoffCancel) {
+    const entry = this.active.get(frame.id);
+    if (!entry || (entry.status !== "working" && entry.status !== "input_required")) return;
+
+    // Verify the cancel comes from the original requester
+    if (frame.from !== entry.from) return;
+
+    const wasInputRequired = entry.status === "input_required";
+    entry.status = "canceled";
+    entry.proc?.kill();
+
+    // If canceled during input_required, no runAgentTurn is running to clean up
+    if (wasInputRequired) {
+      this.active.delete(frame.id);
+    }
+
+    this.peerManager.sendTo(entry.from, {
+      type: "handoff_res",
+      id: frame.id,
+      from: this.config.nodeId,
+      to: entry.from,
+      timestamp: Date.now(),
+      payload: {
+        success: false,
+        nodeId: this.config.nodeId,
+        agent: entry.agent,
+        error: "canceled",
+      },
+    } satisfies HandoffResponse);
+  }
+
+  /** Handle incoming status query (receiver side — report current state). */
+  handleStatusQuery(frame: HandoffStatusQuery) {
+    const entry = this.active.get(frame.id);
+    if (!entry) return;
+
+    this.peerManager.sendTo(frame.from, {
+      type: "handoff_status_res",
+      id: frame.id,
+      from: this.config.nodeId,
+      to: frame.from,
+      timestamp: Date.now(),
+      payload: {
+        status: entry.status,
+        nodeId: this.config.nodeId,
+        agent: entry.agent,
+        elapsedMs: Date.now() - entry.startedAt,
+      },
+    } satisfies HandoffStatusResponse);
+  }
+
+  /** Cancel an outgoing handoff (requester side — send cancel to remote). */
+  cancelHandoff(handoffId: string): boolean {
+    const pending = this.pending.get(handoffId);
+    if (!pending) return false;
+
+    clearTimeout(pending.timer);
+    this.pending.delete(handoffId);
+    pending.reject(new Error("Handoff canceled by requester"));
+
+    // Send cancel frame to the remote node
+    this.peerManager.sendTo(pending.targetNodeId, {
+      type: "handoff_cancel",
+      id: handoffId,
+      from: this.config.nodeId,
+      to: pending.targetNodeId,
+      timestamp: Date.now(),
+    } as HandoffCancel);
+
+    return true;
   }
 
   /** Clean up on shutdown. */
   destroy() {
+    if (this.staleCleanupTimer) {
+      clearInterval(this.staleCleanupTimer);
+      this.staleCleanupTimer = null;
+    }
+
     for (const [, pending] of this.pending) {
       clearTimeout(pending.timer);
       pending.reject(new Error("Shutting down"));
     }
     this.pending.clear();
+
+    // Kill all active tasks (working and input_required)
+    for (const [, entry] of this.active) {
+      if (entry.status === "working" || entry.status === "input_required") {
+        entry.proc?.kill();
+      }
+    }
+    this.active.clear();
+    this.inputRequiredTargets.clear();
+  }
+}
+
+/** Drain a ReadableStream into a string (for stderr consumption). */
+async function drainStream(stream: ReadableStream): Promise<string> {
+  const reader = stream.getReader();
+  const decoder = new TextDecoder();
+  let result = "";
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      result += decoder.decode(value, { stream: true });
+    }
+  } finally {
+    reader.releaseLock();
   }
+  return result;
 }

package/src/http-utils.ts

@@ -0,0 +1,35 @@
+import type { IncomingMessage } from "node:http";
+
+const MAX_BODY_SIZE = 1024 * 1024; // 1 MB
+
+/** Read HTTP request body with size limit (default 1 MB). */
+export function readBody(req: IncomingMessage, maxBytes = MAX_BODY_SIZE): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const chunks: Buffer[] = [];
+    let size = 0;
+    let settled = false;
+    req.on("data", (chunk: Buffer) => {
+      if (settled) return;
+      size += chunk.length;
+      if (size > maxBytes) {
+        settled = true;
+        req.destroy();
+        reject(new Error("Request body too large"));
+        return;
+      }
+      chunks.push(chunk);
+    });
+    req.on("end", () => {
+      if (!settled) {
+        settled = true;
+        resolve(Buffer.concat(chunks).toString());
+      }
+    });
+    req.on("error", (err) => {
+      if (!settled) {
+        settled = true;
+        reject(err);
+      }
+    });
+  });
+}

package/src/index.ts

@@ -2,12 +2,14 @@ import type { OpenClawPluginApi, GatewayRequestHandlerOptions } from "openclaw/p
 import { ClawMatrixConfigSchema, parseConfig } from "./config.ts";
 import { createClusterService, getClusterRuntime } from "./cluster-service.ts";
 import { createClusterHandoffTool } from "./tools/cluster-handoff.ts";
+import { createClusterHandoffReplyTool } from "./tools/cluster-handoff-reply.ts";
 import { createClusterSendTool } from "./tools/cluster-send.ts";
 import { createClusterPeersTool } from "./tools/cluster-peers.ts";
 import { createClusterExecTool } from "./tools/cluster-exec.ts";
 import { createClusterReadTool } from "./tools/cluster-read.ts";
 import { createClusterWriteTool } from "./tools/cluster-write.ts";
 import { createClusterToolTool } from "./tools/cluster-tool.ts";
+import { createClusterEventsTool } from "./tools/cluster-events.ts";
 import { registerClusterCli } from "./cli.ts";
 
 const plugin = {
@@ -99,12 +101,14 @@ const plugin = {
 
     // Agent tools
     api.registerTool(createClusterHandoffTool(), { optional: true });
+    api.registerTool(createClusterHandoffReplyTool(), { optional: true });
     api.registerTool(createClusterSendTool(), { optional: true });
     api.registerTool(createClusterPeersTool(), { optional: true });
     api.registerTool(createClusterExecTool(), { optional: true });
     api.registerTool(createClusterReadTool(), { optional: true });
     api.registerTool(createClusterWriteTool(), { optional: true });
     api.registerTool(createClusterToolTool(), { optional: true });
+    api.registerTool(createClusterEventsTool(), { optional: true });
 
     // Gateway methods (queried by CLI via `openclaw gateway call`)
     api.registerGatewayMethod(
@@ -160,56 +164,63 @@ const plugin = {
     // CLI subcommand
     api.registerCli(registerClusterCli, { commands: ["clawmatrix"] });
 
-    // Inject cluster context into agent prompts
+    // Inject cluster context into agent prompts.
+    //
+    // Minimal-injection strategy:
+    //   prependSystemContext (cached) = static identity + guidance + peer count.
+    //     Content is stable across turns → prompt caching works.
+    //     Peer count is included so the agent knows the cluster exists;
+    //     detailed topology is on-demand via cluster_peers tool.
+    //   prependContext (per-turn) = only pending event notifications.
+    //     Events need proactive push so the agent can react without being asked.
+    //     Everything else (peer details, satellites) is pull-based via tools.
+
+    let cachedPeerCount = -1;
+    let cachedSystemContext = "";
+
     api.on("before_prompt_build", () => {
       try {
         const runtime = getClusterRuntime();
-        const peers = runtime.peerManager.router.getAllPeers();
-        if (peers.length === 0) return;
-
-        const lines = [
-          `[ClawMatrix Cluster]`,
-          `You are on node "${config.nodeId}"${config.tags.length ? ` (tags: ${config.tags.join(", ")})` : ""}.`,
-        ];
+        const peerCount = runtime.peerManager.router.getAllPeers().length;
 
-        if (config.agents.length > 0) {
-          const localAgent = config.agents[0]!;
-          lines.push(`Your role: ${localAgent.description}`);
+        // Rebuild system context only when peer count changes
+        if (peerCount !== cachedPeerCount) {
+          cachedPeerCount = peerCount;
+          const lines: string[] = [];
+          if (peerCount === 0) {
+            lines.push("[ClawMatrix] No peers online. Use cluster_peers to check cluster status.");
+          } else {
+            lines.push(
+              `[ClawMatrix Cluster] node="${config.nodeId}"${config.tags.length ? ` tags=${config.tags.join(",")}` : ""}`,
+              ...(config.agents.length > 0 ? [`Role: ${config.agents[0]!.description}`] : []),
+              `${peerCount} remote peer(s) online. Use cluster_peers to see topology, agents, and models.`,
+              "Prefer cluster_exec/read/write for simple ops; cluster_handoff for complex multi-step tasks.",
+              "IMPORTANT: Always tell user which remote node you're targeting before calling cluster tools.",
+            );
+          }
+          cachedSystemContext = lines.join("\n");
         }
 
-        lines.push("", "Remote nodes in the cluster:");
-        for (const peer of peers) {
-          const status = peer.connection?.isOpen ? "connected" : "via relay";
-          const tags = peer.tags.length ? ` [${peer.tags.join(", ")}]` : "";
-          lines.push(`  - ${peer.nodeId} (${status})${tags}`);
-          for (const agent of peer.agents) {
-            lines.push(`    agent "${agent.id}": ${agent.description}`);
-          }
-          if (peer.models.length > 0) {
-            lines.push(`    models: ${peer.models.map((m) => m.id).join(", ")}`);
+        // Per-turn: only push pending events (agent must react proactively)
+        const pendingEvents = runtime.webHandler?.getUnconsumedEvents(5) ?? [];
+        let prependContext: string | undefined;
+        if (pendingEvents.length > 0) {
+          const evtLines = ["Pending events (use cluster_events to query details or consume):"];
+          for (const evt of pendingEvents) {
+            const age = Math.floor((Date.now() - evt.ts) / 1000);
+            const dataStr = Object.entries(evt.data)
+              .map(([k, v]) => `${k}:${typeof v === "string" ? v : JSON.stringify(v)}`)
+              .join(",");
+            const truncated = dataStr.length > 120 ? dataStr.slice(0, 120) + "…" : dataStr;
+            evtLines.push(`  [${evt.type}] ${evt.source} (${age}s,id:${evt.id}): ${truncated}`);
           }
+          prependContext = evtLines.join("\n");
         }
 
-        lines.push(
-          "",
-          "When a task involves resources on a remote node (files, commands, services), " +
-            "use cluster tools to operate there directly:",
-          "  - cluster_exec / cluster_read / cluster_write — run commands, read/write files on a remote node",
-          "  - cluster_tool — invoke any OpenClaw tool on a remote node",
-          "  - cluster_handoff — delegate a complex task to a remote agent for autonomous execution",
-          "  - cluster_send — send a one-way message to a remote agent",
-          "  - cluster_peers — inspect cluster topology",
-          "Use the node's description and tags to decide which node to target. " +
-            "For simple operations, prefer cluster_exec/read/write; for complex multi-step tasks, prefer cluster_handoff.",
-          "",
-          "IMPORTANT: Before calling any cluster tool or using a remote model, you MUST explicitly tell the user " +
-            "which remote node you are targeting and what you are about to do. For example:",
-          '  "I\'m going to run this command on remote node «coder» ..."',
-          '  "I\'m delegating this task to agent «reviewer» on node «server-b» ..."',
-          "This ensures the user always knows when operations leave the local node.",
-        );
-
-        return { prependSystemContext: lines.join("\n") };
+        return {
+          prependSystemContext: cachedSystemContext,
+          ...(prependContext ? { prependContext } : {}),
+        };
       } catch {
         return;
       }