clawmatrix 0.1.6 → 0.1.11

package/src/model-proxy.ts

@@ -7,6 +7,7 @@ import type {
   ModelResponse,
   ModelStreamChunk,
 } from "./types.ts";
+import { debug } from "./debug.ts";
 
 const MODEL_TIMEOUT = 120_000; // 2 minutes
 
@@ -35,21 +36,37 @@ export class ModelProxy {
   /** Start the local HTTP proxy server for OpenAI-compatible requests. */
   start() {
     this.httpServer = createServer(async (req, res) => {
-      const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`);
-
-      if (url.pathname === "/v1/chat/completions" && req.method === "POST") {
-        const body = await this.readBody(req);
-        const response = await this.handleChatCompletion(body);
-        this.sendResponse(res, response);
-      } else if (url.pathname === "/v1/models" && req.method === "GET") {
-        const response = this.handleListModels();
-        this.sendResponse(res, response);
-      } else {
-        res.writeHead(404, { "Content-Type": "text/plain" });
-        res.end("Not Found");
+      try {
+        const url = new URL(req.url ?? "/", `http://${req.headers.host ?? "localhost"}`);
+
+        const p = url.pathname.replace(/^\/v1/, "");
+        debug("proxy", `${req.method} ${url.pathname} → ${p}`);
+
+        if (p === "/chat/completions" && req.method === "POST") {
+          const body = await this.readBody(req);
+          const response = await this.handleChatCompletion(body);
+          debug("proxy", `response status=${response.status}`);
+          this.sendResponse(res, response);
+        } else if (p === "/models" && req.method === "GET") {
+          const response = this.handleListModels();
+          this.sendResponse(res, response);
+        } else {
+          res.writeHead(404, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ error: { message: `No handler for ${req.method} ${url.pathname}` } }));
+        }
+      } catch {
+        if (!res.headersSent) {
+          res.writeHead(500, { "Content-Type": "application/json" });
+        }
+        res.end(JSON.stringify({ error: { message: "Internal proxy error" } }));
       }
     });
 
+    this.httpServer.on("error", (err) => {
+      // Log but don't crash — port conflict or other server error
+      console.error(`[clawmatrix] Model proxy server error: ${err.message}`);
+    });
+
     this.httpServer.listen(this.config.proxyPort, "127.0.0.1");
   }
 
@@ -84,12 +101,16 @@ export class ModelProxy {
       const pump = (): void => {
         reader.read().then(({ done, value }) => {
           if (done) {
+            reader.releaseLock();
             res.end();
             return;
           }
           res.write(value);
           pump();
-        }).catch(() => res.end());
+        }).catch(() => {
+          reader.releaseLock();
+          res.end();
+        });
       };
       pump();
     }
@@ -115,20 +136,46 @@ export class ModelProxy {
       };
     }
 
-    const modelId = body.model;
-    // Check proxyModels for a nodeId routing hint
-    const proxyModel = this.config.proxyModels.find((m) => m.id === modelId);
-    const route = proxyModel?.nodeId
-      ? this.peerManager.router.getRoute(proxyModel.nodeId)
+    const rawModelId = body.model;
+    // Parse "nodeId/model" format: first segment is nodeId, rest is model ID.
+    // OpenClaw sends "providerId/modelId" where providerId = nodeId, so this
+    // naturally handles both OpenClaw calls and direct curl calls.
+    // If no "/" present, treat entire string as model ID and auto-resolve.
+    let nodeId: string | undefined;
+    let modelId: string;
+    const slashIdx = rawModelId.indexOf("/");
+    if (slashIdx > 0) {
+      nodeId = rawModelId.slice(0, slashIdx);
+      modelId = rawModelId.slice(slashIdx + 1);
+    } else {
+      modelId = rawModelId;
+    }
+    debug("proxy", `model raw="${rawModelId}" nodeId=${nodeId ?? "auto"} modelId="${modelId}" stream=${body.stream ?? false}`);
+    const proxyModel = this.config.proxyModels.find((m) => m.id === modelId && (!nodeId || m.nodeId === nodeId))
+      ?? this.config.proxyModels.find((m) => m.id === modelId);
+    const route = nodeId
+      ? this.peerManager.router.getRoute(nodeId)
       : this.peerManager.router.resolveModel(modelId);
+    debug("proxy", `proxyModel=${proxyModel?.id ?? "none"} route=${route?.nodeId ?? "none"} reachable=${route ? this.peerManager.canReach(route.nodeId) : false}`);
     if (!route) {
       return {
         status: 404,
         headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ error: { message: `Model "${modelId}" not found in cluster` } }),
+        body: JSON.stringify({ error: { message: `Model "${modelId}" not found in cluster (proxyModels: [${this.config.proxyModels.map(m => m.id).join(", ")}])` } }),
       };
     }
 
+    // Inject model identity so the LLM knows what it is
+    const messages = body.messages;
+    if (proxyModel?.description) {
+      const first = messages[0] as { role?: string; content?: string } | undefined;
+      if (first?.role === "system" && typeof first.content === "string") {
+        first.content = `[Model: ${proxyModel.description}]\n${first.content}`;
+      } else {
+        messages.unshift({ role: "system", content: `[Model: ${proxyModel.description}]` });
+      }
+    }
+
     const stream = body.stream ?? false;
     const requestId = crypto.randomUUID();
 
@@ -140,13 +187,22 @@ export class ModelProxy {
       timestamp: Date.now(),
       payload: {
         model: modelId,
-        messages: body.messages,
+        messages,
         temperature: body.temperature,
         maxTokens: body.max_tokens,
         stream,
       },
     };
 
+    // Pre-check reachability before starting a stream (avoids silent empty response)
+    if (!this.peerManager.canReach(route.nodeId)) {
+      return {
+        status: 502,
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ error: { message: `Cannot reach model node "${route.nodeId}"` } }),
+      };
+    }
+
     if (stream) {
       return this.handleStreamRequest(requestId, route.nodeId, frame);
     } else {
@@ -167,8 +223,13 @@ export class ModelProxy {
           this.pending.delete(requestId);
           this.peerManager.router.markFailed(requestId);
           try {
+            const errorChunk = {
+              id: `chatcmpl-${requestId}`,
+              object: "chat.completion.chunk",
+              choices: [{ index: 0, delta: { content: "\n\n[ClawMatrix] Error: model request timed out" }, finish_reason: "stop" }],
+            };
             controller.enqueue(
-              encoder.encode(`data: ${JSON.stringify({ error: "timeout" })}\n\n`),
+              encoder.encode(`data: ${JSON.stringify(errorChunk)}\n\n`),
             );
             controller.enqueue(encoder.encode("data: [DONE]\n\n"));
             controller.close();
@@ -190,10 +251,13 @@ export class ModelProxy {
         if (!sent) {
           this.pending.delete(requestId);
           clearTimeout(timer);
+          const errChunk = {
+            id: `chatcmpl-${requestId}`,
+            object: "chat.completion.chunk",
+            choices: [{ index: 0, delta: { content: `[ClawMatrix] Cannot reach model node "${targetNodeId}"` }, finish_reason: "stop" }],
+          };
           controller.enqueue(
-            encoder.encode(
-              `data: ${JSON.stringify({ error: "Cannot reach model node" })}\n\n`,
-            ),
+            encoder.encode(`data: ${JSON.stringify(errChunk)}\n\n`),
           );
           controller.enqueue(encoder.encode("data: [DONE]\n\n"));
           controller.close();
@@ -306,7 +370,31 @@ export class ModelProxy {
   handleModelResponse(frame: ModelResponse) {
     if (this.peerManager.router.isFailed(frame.id)) return;
     const pending = this.pending.get(frame.id);
-    if (!pending || pending.stream) return;
+    if (!pending) return;
+
+    // For stream requests, handle error responses (the remote node couldn't
+    // process the request and sent model_res instead of model_stream).
+    if (pending.stream) {
+      if (!frame.payload.success && pending.controller && pending.encoder) {
+        clearTimeout(pending.timer);
+        this.pending.delete(frame.id);
+        try {
+          const errChunk = {
+            id: `chatcmpl-${frame.id}`,
+            object: "chat.completion.chunk",
+            choices: [{ index: 0, delta: { content: `[ClawMatrix] Remote error: ${frame.payload.error}` }, finish_reason: "stop" }],
+          };
+          pending.controller.enqueue(
+            pending.encoder.encode(`data: ${JSON.stringify(errChunk)}\n\n`),
+          );
+          pending.controller.enqueue(pending.encoder.encode("data: [DONE]\n\n"));
+          pending.controller.close();
+        } catch {
+          // controller may already be closed
+        }
+      }
+      return;
+    }
 
     clearTimeout(pending.timer);
     this.pending.delete(frame.id);
@@ -314,27 +402,28 @@ export class ModelProxy {
   }
 
   handleModelStream(frame: ModelStreamChunk) {
+    debug("stream", `id=${frame.id} done=${frame.payload.done} delta=${JSON.stringify(frame.payload.delta?.slice?.(0, 50) ?? frame.payload.delta)} failed=${this.peerManager.router.isFailed(frame.id)} hasPending=${this.pending.has(frame.id)}`);
     if (this.peerManager.router.isFailed(frame.id)) return;
     const pending = this.pending.get(frame.id);
     if (!pending?.stream || !pending.controller || !pending.encoder) return;
 
     try {
       if (frame.payload.done) {
+        const finalChunk: Record<string, unknown> = {
+          id: `chatcmpl-${frame.id}`,
+          object: "chat.completion.chunk",
+          choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
+        };
         if (frame.payload.usage) {
-          const usageChunk = {
-            id: `chatcmpl-${frame.id}`,
-            object: "chat.completion.chunk",
-            choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
-            usage: {
-              prompt_tokens: frame.payload.usage.inputTokens,
-              completion_tokens: frame.payload.usage.outputTokens,
-              total_tokens: frame.payload.usage.inputTokens + frame.payload.usage.outputTokens,
-            },
+          finalChunk.usage = {
+            prompt_tokens: frame.payload.usage.inputTokens,
+            completion_tokens: frame.payload.usage.outputTokens,
+            total_tokens: frame.payload.usage.inputTokens + frame.payload.usage.outputTokens,
           };
-          pending.controller.enqueue(
-            pending.encoder.encode(`data: ${JSON.stringify(usageChunk)}\n\n`),
-          );
         }
+        pending.controller.enqueue(
+          pending.encoder.encode(`data: ${JSON.stringify(finalChunk)}\n\n`),
+        );
         pending.controller.enqueue(pending.encoder.encode("data: [DONE]\n\n"));
         pending.controller.close();
         clearTimeout(pending.timer);
@@ -364,6 +453,7 @@ export class ModelProxy {
   /** Handle model_req locally: forward to OpenClaw's configured model provider. */
   async handleModelRequest(frame: ModelRequest): Promise<void> {
     const { id, from, payload } = frame;
+    debug("model_req", `handling model="${payload.model}" from=${from} stream=${payload.stream}`);
 
     const model = this.config.models.find((m) => m.id === payload.model);
     if (!model) {
@@ -392,63 +482,96 @@ export class ModelProxy {
           temperature: payload.temperature,
           max_tokens: payload.maxTokens,
           stream: payload.stream,
+          ...(payload.stream ? { stream_options: { include_usage: true } } : {}),
         }),
       });
 
+      if (!response.ok) {
+        const errBody = await response.text();
+        throw new Error(`Upstream ${response.status}: ${errBody.slice(0, 200)}`);
+      }
+
       if (payload.stream) {
         const reader = response.body?.getReader();
         if (!reader) throw new Error("No response body");
 
-        const decoder = new TextDecoder();
-        let buffer = "";
-
-        while (true) {
-          const { done, value } = await reader.read();
-          if (done) break;
-
-          buffer += decoder.decode(value, { stream: true });
-          const lines = buffer.split("\n");
-          buffer = lines.pop()!;
-
-          for (const line of lines) {
-            if (!line.startsWith("data: ")) continue;
-            const data = line.slice(6).trim();
-            if (data === "[DONE]") {
-              this.peerManager.sendTo(from, {
-                type: "model_stream",
-                id,
-                from: this.config.nodeId,
-                to: from,
-                timestamp: Date.now(),
-                payload: { delta: "", done: true },
-              } satisfies ModelStreamChunk);
-              break;
-            }
+        try {
+          const decoder = new TextDecoder();
+          let buffer = "";
+          let lastUsage: { inputTokens: number; outputTokens: number } | undefined;
+          let streamDone = false;
+
+          while (!streamDone) {
+            const { done, value } = await reader.read();
+            if (done) break;
+
+            buffer += decoder.decode(value, { stream: true });
+            const lines = buffer.split("\n");
+            buffer = lines.pop()!;
 
-            try {
-              const parsed = JSON.parse(data);
-              const delta = parsed.choices?.[0]?.delta?.content ?? "";
-              if (delta) {
+            for (const line of lines) {
+              if (!line.startsWith("data: ")) continue;
+              const data = line.slice(6).trim();
+              if (data === "[DONE]") {
                 this.peerManager.sendTo(from, {
                   type: "model_stream",
                   id,
                   from: this.config.nodeId,
                   to: from,
                   timestamp: Date.now(),
-                  payload: { delta, done: false },
+                  payload: { delta: "", done: true, usage: lastUsage },
                 } satisfies ModelStreamChunk);
+                streamDone = true;
+                break;
+              }
+
+              try {
+                const parsed = JSON.parse(data);
+                if (parsed.usage) {
+                  lastUsage = {
+                    inputTokens: parsed.usage.prompt_tokens,
+                    outputTokens: parsed.usage.completion_tokens,
+                  };
+                }
+                const d = parsed.choices?.[0]?.delta;
+                const delta = d?.content || d?.reasoning_content || "";
+                if (delta) {
+                  this.peerManager.sendTo(from, {
+                    type: "model_stream",
+                    id,
+                    from: this.config.nodeId,
+                    to: from,
+                    timestamp: Date.now(),
+                    payload: { delta, done: false },
+                  } satisfies ModelStreamChunk);
+                }
+              } catch {
+                // skip malformed chunks
               }
-            } catch {
-              // skip malformed chunks
             }
           }
+          // If the upstream closed without sending [DONE], send a completion
+          // frame so the requesting side doesn't hang until MODEL_TIMEOUT.
+          if (!streamDone) {
+            this.peerManager.sendTo(from, {
+              type: "model_stream",
+              id,
+              from: this.config.nodeId,
+              to: from,
+              timestamp: Date.now(),
+              payload: { delta: "", done: true, usage: lastUsage },
+            } satisfies ModelStreamChunk);
+          }
+        } finally {
+          reader.releaseLock();
         }
       } else {
         const result = (await response.json()) as {
-          choices?: { message?: { content?: string } }[];
+          choices?: { message?: { content?: string; reasoning_content?: string } }[];
           usage?: { prompt_tokens: number; completion_tokens: number };
         };
-        const content = result.choices?.[0]?.message?.content ?? "";
+        const msg = result.choices?.[0]?.message;
+        const content = msg?.content || msg?.reasoning_content || "";
         const usage = result.usage;
 
         this.peerManager.sendTo(from, {

package/src/peer-manager.ts

@@ -33,6 +33,7 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
   private stopped = false;
   /** Map from ws WebSocket to Connection for inbound connections. */
   private inboundConnections = new Map<WsWebSocket, Connection>();
+  private gossipDebounceTimer: ReturnType<typeof setTimeout> | null = null;
 
   constructor(config: ClawMatrixConfig) {
     super();
@@ -62,6 +63,10 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
 
   async stop() {
     this.stopped = true;
+    if (this.gossipDebounceTimer) {
+      clearTimeout(this.gossipDebounceTimer);
+      this.gossipDebounceTimer = null;
+    }
     for (const timer of this.reconnectTimers.values()) {
       clearTimeout(timer);
     }
@@ -86,6 +91,8 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       this.httpServer.close();
       this.httpServer = null;
     }
+
+    this.router.destroy();
   }
 
   // ── Inbound WS server (node:http + ws) ──────────────────────────
@@ -119,6 +126,10 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       });
     });
 
+    this.httpServer.on("error", (err) => {
+      console.error(`[clawmatrix] WS server error on port ${port}: ${err.message}`);
+    });
+
     this.httpServer.listen(port, hostname);
   }
 
@@ -319,13 +330,17 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       }
     }
 
-    // If we learned new info, re-sync with other peers
+    // If we learned new info, re-sync with other peers (debounced to avoid storms)
     if (changed) {
-      for (const conn of this.router.getDirectConnections()) {
-        if (conn !== from && conn.isOpen) {
-          this.sendPeerSync(conn);
+      if (this.gossipDebounceTimer) clearTimeout(this.gossipDebounceTimer);
+      this.gossipDebounceTimer = setTimeout(() => {
+        this.gossipDebounceTimer = null;
+        for (const conn of this.router.getDirectConnections()) {
+          if (conn !== from && conn.isOpen) {
+            this.sendPeerSync(conn);
+          }
         }
-      }
+      }, 100);
     }
   }
 
@@ -334,6 +349,18 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
     return this.router.sendTo(nodeId, frame);
   }
 
+  /** Check if a node is reachable (direct or via relay) without sending. */
+  canReach(nodeId: string): boolean {
+    const route = this.router.getRoute(nodeId);
+    if (!route) return false;
+    if (route.connection?.isOpen) return true;
+    if (route.reachableVia) {
+      const relay = this.router.getRoute(route.reachableVia);
+      return !!relay?.connection?.isOpen;
+    }
+    return false;
+  }
+
   broadcast(frame: ClusterFrame | AnyClusterFrame) {
     this.router.broadcast(frame);
   }

package/src/router.ts

@@ -4,6 +4,7 @@ import type { Connection } from "./connection.ts";
 const DEFAULT_TTL = 3;
 const SEEN_FRAME_TTL = 60_000; // 60s dedup window
 const MAX_SEEN_FRAMES = 10_000;
+const PRUNE_INTERVAL = 30_000; // periodic cleanup every 30s
 
 export interface RouteEntry {
   nodeId: string;
@@ -24,6 +25,7 @@ export class Router {
   private routes = new Map<string, RouteEntry>();
   private connections = new Map<string, Connection>(); // nodeId → direct connection
   private seenFrames = new Map<string, number>(); // frameId → timestamp
+  private pruneTimer: ReturnType<typeof setInterval> | null = null;
 
   constructor(
     nodeId: string,
@@ -33,6 +35,17 @@ export class Router {
     this.localAgents = localCapabilities?.agents ?? [];
     this.localModels = localCapabilities?.models ?? [];
     this.localTags = localCapabilities?.tags ?? [];
+
+    this.pruneTimer = setInterval(() => this.pruneSeenFrames(true), PRUNE_INTERVAL);
+  }
+
+  /** Stop periodic cleanup. Call on shutdown. */
+  destroy() {
+    if (this.pruneTimer) {
+      clearInterval(this.pruneTimer);
+      this.pruneTimer = null;
+    }
+    this.seenFrames.clear();
   }
 
   // ── Route table management ─────────────────────────────────────
@@ -120,6 +133,12 @@ export class Router {
       }
     }
 
+    // Fallback: if no agent ID or tag matched, try matching by nodeId
+    if (candidates.length === 0 && !isTagQuery) {
+      const byNode = this.routes.get(target);
+      if (byNode) candidates.push(byNode);
+    }
+
     if (candidates.length === 0) return undefined;
 
     // Sort: direct connections first, then by latency
@@ -205,7 +224,9 @@ export class Router {
   tryRelay(frame: ClusterFrame): boolean {
     if (!frame.to || frame.to === this.nodeId) return false;
 
-    const ttl = (frame.ttl ?? DEFAULT_TTL) - 1;
+    const rawTtl = frame.ttl ?? DEFAULT_TTL;
+    if (typeof rawTtl !== "number" || !Number.isFinite(rawTtl) || rawTtl < 1) return false;
+    const ttl = rawTtl - 1;
     if (ttl <= 0) return false;
 
     const relayed = this.sendTo(frame.to, { ...frame, ttl });
@@ -232,8 +253,8 @@ export class Router {
     return this.seenFrames.has(`failed:${requestId}`);
   }
 
-  private pruneSeenFrames() {
-    if (this.seenFrames.size <= MAX_SEEN_FRAMES) return;
+  private pruneSeenFrames(force = false) {
+    if (!force && this.seenFrames.size <= MAX_SEEN_FRAMES) return;
     const now = Date.now();
     for (const [id, ts] of this.seenFrames) {
       if (now - ts > SEEN_FRAME_TTL) {

package/src/tool-proxy.ts

@@ -1,6 +1,7 @@
 import type { PeerManager } from "./peer-manager.ts";
 import type { ClawMatrixConfig, ToolProxyConfig } from "./config.ts";
 import type { ToolProxyRequest, ToolProxyResponse } from "./types.ts";
+import { isLocalTool, executeLocally } from "./local-tools.ts";
 
 const TOOL_TIMEOUT = 30_000;
 
@@ -113,7 +114,9 @@ export class ToolProxy {
     }
 
     try {
-      const result = await this.executeViaGateway(payload.tool, payload.params);
+      const result = isLocalTool(payload.tool)
+        ? await executeLocally(payload.tool, payload.params)
+        : await this.executeViaGateway(payload.tool, payload.params);
       this.sendResponse(id, from, { success: true, result });
     } catch (err) {
       this.sendResponse(id, from, {

package/src/types.ts

@@ -113,6 +113,15 @@ export interface HandoffRequest extends ClusterFrame {
   };
 }
 
+export interface HandoffStreamChunk extends ClusterFrame {
+  type: "handoff_stream";
+  id: string;
+  payload: {
+    delta: string;
+    done: boolean;
+  };
+}
+
 export interface HandoffResponse extends ClusterFrame {
   type: "handoff_res";
   id: string;
@@ -161,10 +170,26 @@ export interface AgentInfo {
   tags: string[];
 }
 
+export interface ModelCompatInfo {
+  supportsTools?: boolean;
+  supportsStore?: boolean;
+  supportsDeveloperRole?: boolean;
+  supportsReasoningEffort?: boolean;
+  supportsUsageInStreaming?: boolean;
+  supportsStrictMode?: boolean;
+  maxTokensField?: "max_completion_tokens" | "max_tokens";
+  thinkingFormat?: "openai" | "zai" | "qwen";
+  requiresToolResultName?: boolean;
+  requiresAssistantAfterToolResult?: boolean;
+  requiresThinkingAsText?: boolean;
+  requiresMistralToolIds?: boolean;
+}
+
 export interface ModelInfo {
   id: string;
   provider: string;
   description?: string;
+  compat?: ModelCompatInfo;
 }
 
 export interface PeerInfo {
@@ -197,6 +222,7 @@ export type AnyClusterFrame =
   | ModelResponse
   | ModelStreamChunk
   | HandoffRequest
+  | HandoffStreamChunk
   | HandoffResponse
   | SendMessage
   | ToolProxyRequest