clawmatrix 0.1.21 → 0.1.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "clawmatrix",
-  "version": "0.1.21",
+  "version": "0.1.23",
   "description": "Decentralized mesh cluster plugin for OpenClaw — inter-gateway communication, model proxy, task handoff, and tool proxy.",
   "type": "module",
   "license": "MIT",

package/src/connection.ts

@@ -278,6 +278,8 @@ export class Connection extends EventEmitter<ConnectionEvents> {
       const interval = HEARTBEAT_BASE + Math.random() * HEARTBEAT_JITTER;
       this.heartbeatTimer = setTimeout(() => {
         if (this.closed) return;
+        // Increment before checking: this ping is about to be sent and
+        // counts as outstanding until a pong arrives.
         this.missedPongs++;
         if (this.missedPongs >= HEARTBEAT_TIMEOUT_COUNT) {
           this.close(4002, "heartbeat timeout");

package/src/handoff.ts

@@ -203,7 +203,7 @@ export class HandoffManager {
     clearTimeout(pending.timer);
     pending.timer = this.createTimeout(
       frame.id,
-      frame.from,
+      pending.targetNodeId,
       pending.target,
       pending.task,
       pending.context,

package/src/index.ts

@@ -1,5 +1,5 @@
-import type { OpenClawPluginApi, GatewayRequestHandlerOptions } from "openclaw/plugin-sdk";
-import { ClawMatrixConfigSchema, parseConfig } from "./config.ts";
+import type { OpenClawPluginApi, OpenClawConfig, GatewayRequestHandlerOptions } from "openclaw/plugin-sdk";
+import { ClawMatrixConfigSchema, parseConfig, type ClawMatrixConfig } from "./config.ts";
 import { createClusterService, getClusterRuntime } from "./cluster-service.ts";
 import { createClusterHandoffTool } from "./tools/cluster-handoff.ts";
 import { createClusterHandoffReplyTool } from "./tools/cluster-handoff-reply.ts";
@@ -12,6 +12,59 @@ import { createClusterToolTool } from "./tools/cluster-tool.ts";
 import { createClusterEventsTool } from "./tools/cluster-events.ts";
 import { registerClusterCli } from "./cli.ts";
 
+/**
+ * Auto-discover models from OpenClaw's models.providers config.
+ * Iterates all configured providers and their models, building the
+ * full model list with endpoint info (baseUrl, apiKey, api type, etc.).
+ * Excludes clawmatrix proxy providers (remote models from peer nodes).
+ */
+type ProviderEntry = { baseUrl?: string; apiKey?: string; api?: string; models?: Array<Record<string, unknown>> };
+
+function discoverModels(
+  openclawConfig: OpenClawConfig,
+  config: ClawMatrixConfig,
+): ClawMatrixConfig["models"] {
+  const cfg = openclawConfig as Record<string, unknown>;
+  const providers = (cfg.models as { providers?: Record<string, ProviderEntry> } | undefined)?.providers;
+  if (!providers || typeof providers !== "object") return [];
+
+  // Collect proxyModel node IDs to exclude clawmatrix-registered providers
+  const proxyNodeIds = new Set(config.proxyModels.map((m) => m.nodeId));
+
+  const result: ClawMatrixConfig["models"] = [];
+  for (const [providerId, providerConfig] of Object.entries(providers)) {
+    // Skip clawmatrix proxy providers (remote models from peer nodes)
+    if (proxyNodeIds.has(providerId)) continue;
+
+    const models = providerConfig?.models;
+    if (!Array.isArray(models)) continue;
+
+    for (const m of models) {
+      if (!m.id || typeof m.id !== "string") continue;
+      result.push({
+        id: m.id,
+        provider: providerId,
+        description: m.name as string | undefined,
+        baseUrl: providerConfig?.baseUrl,
+        apiKey: typeof providerConfig?.apiKey === "string" ? providerConfig.apiKey : undefined,
+        api: (m.api ?? providerConfig?.api) as ClawMatrixConfig["models"][0]["api"],
+        contextWindow: m.contextWindow as number | undefined,
+        maxTokens: m.maxTokens as number | undefined,
+        reasoning: m.reasoning as boolean | undefined,
+        input: m.input as ("text" | "image")[] | undefined,
+        cost: m.cost as { input: number; output: number; cacheRead: number; cacheWrite: number } | undefined,
+        compat: m.compat as ClawMatrixConfig["models"][0]["compat"],
+      });
+    }
+  }
+
+  if (result.length > 0) {
+    console.debug(`[clawmatrix] Auto-discovered ${result.length} model(s) from models.providers: ${result.map((m) => `${m.provider}/${m.id}`).join(", ")}`);
+  }
+
+  return result;
+}
+
 const plugin = {
   id: "clawmatrix",
   name: "ClawMatrix",
@@ -36,6 +89,11 @@ const plugin = {
       return;
     }
 
+    // Auto-discover models from agents.defaults.models if no explicit models configured
+    if (config.models.length === 0) {
+      config = { ...config, models: discoverModels(api.config, config) };
+    }
+
     // Background service: manages mesh connections, WS listener, heartbeat
     api.registerService(createClusterService(config, api.config, api.runtime.version));
 
@@ -60,26 +118,42 @@ const plugin = {
       const models = ((cfg).models ??= {}) as Record<string, unknown>;
       const providers = (models.providers ??= {}) as Record<string, unknown>;
       for (const [nodeId, nodeModels] of Object.entries(modelsByNode)) {
-        if (!providers[nodeId]) {
-          const api = nodeApiType[nodeId] ?? "openai-completions";
-          providers[nodeId] = { baseUrl, apiKey: "sk-clawmatrix-proxy", api, models: nodeModels };
+        const apiType = nodeApiType[nodeId] ?? "openai-completions";
+        const existing = providers[nodeId] as Record<string, unknown> | undefined;
+        if (existing) {
+          // Provider already exists (e.g. from models.json or config reload) —
+          // ensure the dummy apiKey is always present so auth resolution succeeds.
+          if (!existing.apiKey) existing.apiKey = "sk-clawmatrix-proxy";
+          if (!existing.baseUrl) existing.baseUrl = baseUrl;
+        } else {
+          providers[nodeId] = { baseUrl, apiKey: "sk-clawmatrix-proxy", api: apiType, models: nodeModels };
         }
       }
     };
 
-    patchProviders(api.config as Record<string, unknown>);
-
-    // Also patch the runtime config snapshot (loadConfig returns it by reference).
-    // activateSecretsRuntimeSnapshot clones the config, so api.config and the
-    // snapshot returned by loadConfig() are separate objects — patch both.
-    try {
-      const snapshot = api.runtime.config.loadConfig();
-      if (snapshot && snapshot !== api.config) {
-        patchProviders(snapshot as Record<string, unknown>);
+    // Patch all known config objects.
+    // activateSecretsRuntimeSnapshot clones config on startup & hot-reload,
+    // so our injected providers get lost. There is no plugin-facing "config_reload"
+    // event, so we re-patch periodically and skip when the reference hasn't changed.
+    let lastSnapshotRef: unknown = null;
+    const patchAllConfigs = () => {
+      patchProviders(api.config as Record<string, unknown>);
+      try {
+        const snapshot = api.runtime.config.loadConfig();
+        if (snapshot && snapshot !== lastSnapshotRef) {
+          lastSnapshotRef = snapshot;
+          patchProviders(snapshot as Record<string, unknown>);
+        }
+      } catch {
+        // Best-effort
       }
-    } catch {
-      // Best-effort; api.config patch is the fallback
-    }
+    };
+
+    patchAllConfigs();
+
+    const repatchTimer = setInterval(patchAllConfigs, 10_000);
+    repatchTimer.unref?.();
+    api.on("dispose", () => clearInterval(repatchTimer));
 
     for (const [nodeId, models] of Object.entries(modelsByNode)) {
       api.registerProvider({
@@ -165,6 +239,11 @@ const plugin = {
       },
     );
 
+    // Log model selection on each LLM call (fire-and-forget)
+    api.on("llm_input", (event) => {
+      api.logger.debug(`[clawmatrix] llm_input: provider=${event.provider} model=${event.model}`);
+    });
+
     // CLI subcommand
     api.registerCli(registerClusterCli, { commands: ["clawmatrix"] });
 

package/src/knowledge-sync.ts

@@ -31,13 +31,27 @@ export interface KnowledgeSyncOptions {
 
 const TAG = "knowledge";
 
+async function streamToString(stream: ReadableStream | null): Promise<string> {
+  if (!stream) return "";
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    chunks.push(value);
+  }
+  return Buffer.concat(chunks).toString("utf-8");
+}
+
 export class KnowledgeSync {
   private doc: Automerge.Doc<KnowledgeDoc>;
   private syncStates = new Map<string, Automerge.SyncState>();
   private watcher: FSWatcher | null = null;
   private debounceTimer: ReturnType<typeof setTimeout> | null = null;
-  /** Paths currently being written by exportToFs — suppressed from fs watcher. */
-  private writingPaths = new Set<string>();
+  /** Paths recently written by exportToFs with their expected content — used to
+   *  suppress watcher-triggered syncs for our own writes. Entries are cleared
+   *  once handleLocalChanges confirms the file content matches. */
+  private writtenByExport = new Map<string, string>();
   private opts: KnowledgeSyncOptions;
   private ig: Ignore = ignore();
 
@@ -78,8 +92,6 @@ export class KnowledgeSync {
     // Start watching for file changes
     this.watcher = watch(this.opts.workspacePath, { recursive: true }, (_event, filename) => {
       if (!filename) return;
-      // Ignore files currently being written by export
-      if (this.writingPaths.has(filename)) return;
       // Ignore hidden files
       if (filename.startsWith(".")) return;
       // Ignore gitignored files
@@ -184,12 +196,22 @@ export class KnowledgeSync {
     const currentFiles = await this.readWorkspaceFiles();
     const docFiles = this.doc.files ?? {};
 
+    // Clear export markers whose content matches the current file (our write landed).
+    // If the content differs, a real local edit happened after export — treat it as modified.
+    for (const [relPath, expectedContent] of this.writtenByExport) {
+      if (currentFiles[relPath] === expectedContent) {
+        this.writtenByExport.delete(relPath);
+      }
+    }
+
     // Collect changed files for logging
     const added: string[] = [];
     const modified: string[] = [];
     const deleted: string[] = [];
 
     for (const [relPath, content] of Object.entries(currentFiles)) {
+      // Skip files that were just written by exportToFs and haven't been edited since
+      if (this.writtenByExport.has(relPath)) continue;
       if (!(relPath in docFiles)) {
         added.push(relPath);
       } else if (docFiles[relPath] !== content) {
@@ -308,28 +330,21 @@ export class KnowledgeSync {
       // Don't export files that would be gitignored
       if (this.isIgnored(relPath)) continue;
       if (currentFiles[relPath] !== content) {
-        this.writingPaths.add(relPath);
+        this.writtenByExport.set(relPath, content);
         const absPath = path.join(this.opts.workspacePath, relPath);
         await mkdir(path.dirname(absPath), { recursive: true });
         await writeFile(absPath, content, "utf-8");
-        setTimeout(() => this.writingPaths.delete(relPath), 500);
         written++;
       }
     }
 
-    let removed = 0;
-    for (const relPath of Object.keys(currentFiles)) {
-      if (!(relPath in docFiles)) {
-        this.writingPaths.add(relPath);
-        const absPath = path.join(this.opts.workspacePath, relPath);
-        await unlink(absPath).catch(() => {});
-        setTimeout(() => this.writingPaths.delete(relPath), 500);
-        removed++;
-      }
-    }
+    // Note: we intentionally do NOT delete local files that are absent from
+    // the doc. A locally created file that hasn't been synced yet would be
+    // lost if we deleted it here. Deletions propagate through the doc via
+    // handleLocalChanges() → Automerge change → broadcastSync() instead.
 
-    if (written > 0 || removed > 0) {
-      debug(TAG, `exported to filesystem: ${written} written, ${removed} removed`);
+    if (written > 0) {
+      debug(TAG, `exported to filesystem: ${written} written`);
     }
   }
 
@@ -356,6 +371,34 @@ export class KnowledgeSync {
       } else {
         debug(TAG, "git repo already initialized");
       }
+
+      // Ensure local git user config exists (so commits don't fail on unconfigured machines)
+      const nameCheck = spawnProcess(["git", "config", "user.name"], {
+        cwd: this.opts.workspacePath,
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      if ((await nameCheck.exited) !== 0) {
+        const setName = spawnProcess(["git", "config", "user.name", "clawmatrix"], {
+          cwd: this.opts.workspacePath,
+          stdout: "pipe",
+          stderr: "pipe",
+        });
+        await setName.exited;
+      }
+      const emailCheck = spawnProcess(["git", "config", "user.email"], {
+        cwd: this.opts.workspacePath,
+        stdout: "pipe",
+        stderr: "pipe",
+      });
+      if ((await emailCheck.exited) !== 0) {
+        const setEmail = spawnProcess(["git", "config", "user.email", "clawmatrix@local"], {
+          cwd: this.opts.workspacePath,
+          stdout: "pipe",
+          stderr: "pipe",
+        });
+        await setEmail.exited;
+      }
     } catch {
       debug(TAG, "git not available, skipping git integration");
     }
@@ -389,8 +432,13 @@ export class KnowledgeSync {
           stderr: "pipe",
         },
       );
-      await commit.exited;
-      debug(TAG, `git commit: ${message}`);
+      const commitCode = await commit.exited;
+      if (commitCode !== 0) {
+        const stderr = await streamToString(commit.stderr);
+        debug(TAG, `git commit failed (exit ${commitCode}): ${stderr}`);
+      } else {
+        debug(TAG, `git commit: ${message}`);
+      }
     } catch (err) {
       debug(TAG, `git commit failed: ${err}`);
     }

package/src/local-tools.ts

@@ -126,13 +126,20 @@ async function executeExec(params: ExecParams): Promise<ToolResult> {
 
 // ── read/write/edit: reuse pi-coding-agent factories ───────────────
 
-const piToolCache = new Map<string, { execute: Function }>();
+/** Cache key includes cwd so tools are recreated if the working directory changes. */
+let piToolCache = new Map<string, { execute: Function }>();
+let piToolCwd = "";
 
 function getPiTool(name: string): { execute: Function } {
+  const cwd = process.cwd();
+  if (cwd !== piToolCwd) {
+    piToolCache = new Map();
+    piToolCwd = cwd;
+  }
+
   let tool = piToolCache.get(name);
   if (tool) return tool;
 
-  const cwd = process.cwd();
   switch (name) {
     case "read":
       tool = createReadTool(cwd);

package/src/model-proxy.ts

@@ -12,6 +12,15 @@ import { debug } from "./debug.ts";
 import { readBody } from "./http-utils.ts";
 
 const MODEL_TIMEOUT = 120_000; // 2 minutes
+
+/** Normalize usage from OpenAI-compatible APIs (supports both field naming conventions). */
+function parseUsage(usage: Record<string, number> | undefined): { inputTokens: number; outputTokens: number } | undefined {
+  if (!usage) return undefined;
+  return {
+    inputTokens: usage.input_tokens ?? usage.prompt_tokens ?? 0,
+    outputTokens: usage.output_tokens ?? usage.completion_tokens ?? 0,
+  };
+}
 const MAX_STREAM_BUFFER = 1_048_576; // 1MB — guard against upstream not sending newlines
 
 type ResponseFormat = "chat" | "responses";
@@ -54,6 +63,11 @@ export class ModelProxy {
   private gatewayInfo: GatewayInfo;
   private openclawConfig: OpenClawConfig;
 
+  /** Cache of models that need a different API format than configured (detected at runtime).
+   *  Entries expire after 10 minutes so upstream upgrades are eventually detected. */
+  private modelApiCache = new Map<string, { api: string; ts: number }>();
+  private static readonly MODEL_API_CACHE_TTL = 600_000; // 10 minutes
+
   constructor(config: ClawMatrixConfig, peerManager: PeerManager, gatewayInfo: GatewayInfo, openclawConfig: OpenClawConfig) {
     this.config = config;
     this.peerManager = peerManager;
@@ -227,9 +241,11 @@ export class ModelProxy {
     }
 
     // 2. Read from OpenClaw's models.providers[provider]
-    const providers = (this.openclawConfig as Record<string, unknown>).models as
-      { providers?: Record<string, { baseUrl?: string; apiKey?: string; api?: string }> } | undefined;
-    const providerConfig = providers?.providers?.[model.provider];
+    const modelsConfig = (this.openclawConfig as Record<string, unknown>).models;
+    const providers = (modelsConfig && typeof modelsConfig === "object")
+      ? (modelsConfig as { providers?: Record<string, { baseUrl?: string; apiKey?: string; api?: string }> }).providers
+      : undefined;
+    const providerConfig = providers?.[model.provider];
     if (providerConfig?.baseUrl) {
       return {
         baseUrl: providerConfig.baseUrl.replace(/\/$/, ""),
@@ -303,8 +319,17 @@ export class ModelProxy {
     this.pending.clear();
     this.streamText.clear();
     this.streamSetupSent.clear();
+    this.modelApiCache.clear();
   }
 
+  /** Clean up all tracking state for a request (pending, streamText, streamSetupSent). */
+  private cleanupRequest(id: string, stableStreamId?: string) {
+    const pending = this.pending.get(id);
+    if (pending) clearTimeout(pending.timer);
+    this.pending.delete(id);
+    this.streamText.delete(id);
+    if (stableStreamId) this.streamSetupSent.delete(stableStreamId);
+  }
 
   private sendResponse(res: import("node:http").ServerResponse, response: ProxyResponse) {
     res.writeHead(response.status, response.headers);
@@ -320,8 +345,12 @@ export class ModelProxy {
             res.end();
             return;
           }
-          res.write(value);
-          pump();
+          const ok = res.write(value);
+          if (ok) {
+            pump();
+          } else {
+            res.once("drain", pump);
+          }
         }).catch(() => {
           reader.releaseLock();
           res.end();
@@ -377,7 +406,7 @@ export class ModelProxy {
   }
 
   private async handleChatCompletion(rawBody: string, _api: string): Promise<ProxyResponse> {
-    let body: { model: string; messages: unknown[]; stream?: boolean; temperature?: number; max_tokens?: number };
+    let body: { model: string; messages: unknown[]; stream?: boolean; temperature?: number; max_tokens?: number; tools?: unknown[]; tool_choice?: unknown };
     try {
       body = JSON.parse(rawBody);
     } catch {
@@ -410,7 +439,7 @@ export class ModelProxy {
       }
       return {
         type: "model_req", id, from: this.config.nodeId, to: candidate.routeNodeId, timestamp: Date.now(),
-        payload: { model: modelId, provider: candidate.proxyModel?.provider, api: candidate.proxyModel?.api, messages, temperature: body.temperature, maxTokens: body.max_tokens, stream },
+        payload: { model: modelId, provider: candidate.proxyModel?.provider, api: candidate.proxyModel?.api, messages, tools: body.tools, toolChoice: body.tool_choice, temperature: body.temperature, maxTokens: body.max_tokens, stream },
       };
     };
     const frame = buildFrame(first, requestId);
@@ -423,7 +452,7 @@ export class ModelProxy {
   }
 
   private async handleResponses(rawBody: string): Promise<ProxyResponse> {
-    let body: { model: string; input: unknown; stream?: boolean; temperature?: number; max_output_tokens?: number; instructions?: string };
+    let body: { model: string; input: unknown; stream?: boolean; temperature?: number; max_output_tokens?: number; instructions?: string; tools?: unknown[]; tool_choice?: unknown };
     try {
       body = JSON.parse(rawBody);
     } catch {
@@ -467,7 +496,7 @@ export class ModelProxy {
       }
       return {
         type: "model_req", id, from: this.config.nodeId, to: candidate.routeNodeId, timestamp: Date.now(),
-        payload: { model: modelId, provider: candidate.proxyModel?.provider, api: candidate.proxyModel?.api, messages: inputItems, inputFormat: "responses", temperature: body.temperature, maxTokens: body.max_output_tokens, stream },
+        payload: { model: modelId, provider: candidate.proxyModel?.provider, api: candidate.proxyModel?.api, messages: inputItems, inputFormat: "responses", tools: body.tools, toolChoice: body.tool_choice, temperature: body.temperature, maxTokens: body.max_output_tokens, stream },
       };
     };
     const frame = buildFrame(first, requestId);
@@ -524,8 +553,7 @@ export class ModelProxy {
     const stableId = streamId ?? requestId;
 
     const timer = setTimeout(() => {
-      this.pending.delete(requestId);
-      this.streamText.delete(requestId);
+      this.cleanupRequest(requestId);
       this.peerManager.router.markFailed(requestId);
       this.tryStreamFailover(stableId, responseFormat, controller, encoder, model, failoverCandidates, buildFrame, `model request to "${targetNodeId}" timed out`);
     }, MODEL_TIMEOUT);
@@ -542,14 +570,14 @@ export class ModelProxy {
 
     // Emit setup events for responses API (only once per stream, keyed by stableId)
     if (responseFormat === "responses" && !this.streamSetupSent.has(stableId)) {
-      this.enqueueResponsesStreamSetup(controller, encoder, stableId, model);
+      const hasTools = Array.isArray(frame.payload.tools) && frame.payload.tools.length > 0;
+      this.enqueueResponsesStreamSetup(controller, encoder, stableId, model, hasTools);
       this.streamSetupSent.add(stableId);
     }
 
     const sent = this.peerManager.sendTo(targetNodeId, frame);
     if (!sent) {
-      this.pending.delete(requestId);
-      clearTimeout(timer);
+      this.cleanupRequest(requestId);
       this.tryStreamFailover(stableId, responseFormat, controller, encoder, model, failoverCandidates, buildFrame, `cannot reach node "${targetNodeId}"`);
     }
   }
@@ -593,18 +621,23 @@ export class ModelProxy {
   }
 
   /** Emit responses API stream setup events (response.created → content_part.added). */
-  private enqueueResponsesStreamSetup(controller: ReadableStreamDefaultController, encoder: TextEncoder, id: string, model: string) {
+  private enqueueResponsesStreamSetup(controller: ReadableStreamDefaultController, encoder: TextEncoder, id: string, model: string, hasTools = false) {
     const respId = `resp_${id}`;
-    const msgId = `msg_${id}`;
     const now = Math.floor(Date.now() / 1000);
     const baseResp = { id: respId, object: "response", created_at: now, status: "in_progress", model, output: [] };
-    const msgItem = { type: "message", id: msgId, role: "assistant", content: [], status: "in_progress" };
-    const textPart = { type: "output_text", text: "" };
 
     controller.enqueue(encoder.encode(`event: response.created\ndata: ${JSON.stringify({ type: "response.created", response: baseResp })}\n\n`));
     controller.enqueue(encoder.encode(`event: response.in_progress\ndata: ${JSON.stringify({ type: "response.in_progress", response: baseResp })}\n\n`));
-    controller.enqueue(encoder.encode(`event: response.output_item.added\ndata: ${JSON.stringify({ type: "response.output_item.added", output_index: 0, item: msgItem })}\n\n`));
-    controller.enqueue(encoder.encode(`event: response.content_part.added\ndata: ${JSON.stringify({ type: "response.content_part.added", item_id: msgId, output_index: 0, content_index: 0, part: textPart })}\n\n`));
+
+    // When tools are present, skip pre-fabricated output_item/content_part events —
+    // the real events (including function_call items) will be forwarded from the remote.
+    if (!hasTools) {
+      const msgId = `msg_${id}`;
+      const msgItem = { type: "message", id: msgId, role: "assistant", content: [], status: "in_progress" };
+      const textPart = { type: "output_text", text: "" };
+      controller.enqueue(encoder.encode(`event: response.output_item.added\ndata: ${JSON.stringify({ type: "response.output_item.added", output_index: 0, item: msgItem })}\n\n`));
+      controller.enqueue(encoder.encode(`event: response.content_part.added\ndata: ${JSON.stringify({ type: "response.content_part.added", item_id: msgId, output_index: 0, content_index: 0, part: textPart })}\n\n`));
+    }
   }
 
   /** Emit responses API stream completion events (output_text.done → response.completed). */
@@ -820,18 +853,17 @@ export class ModelProxy {
     // process the request and sent model_res instead of model_stream).
     if (pending.stream) {
       if (!frame.payload.success && pending.controller && pending.encoder) {
-        clearTimeout(pending.timer);
-        this.pending.delete(frame.id);
-        this.streamText.delete(frame.id);
+        const stableId = pending.stableStreamId ?? frame.id;
         // Try failover if no content has been sent yet
         if (!pending.hasContent && pending.failoverCandidates?.length && pending.buildFrame) {
+          this.cleanupRequest(frame.id);
           this.tryStreamFailover(
-            pending.stableStreamId ?? frame.id, pending.responseFormat, pending.controller, pending.encoder,
+            stableId, pending.responseFormat, pending.controller, pending.encoder,
             pending.model ?? "", pending.failoverCandidates, pending.buildFrame,
             `remote error: ${frame.payload.error}`,
           );
         } else {
-          const stableId = pending.stableStreamId ?? frame.id;
+          this.cleanupRequest(frame.id, stableId);
           try {
             const errMsg = `[ClawMatrix] Remote error: ${frame.payload.error}`;
             if (pending.responseFormat === "responses") {
@@ -843,14 +875,12 @@ export class ModelProxy {
             }
             pending.controller.close();
           } catch { /* controller may already be closed */ }
-          this.streamSetupSent.delete(stableId);
         }
       }
       return;
     }
 
-    clearTimeout(pending.timer);
-    this.pending.delete(frame.id);
+    this.cleanupRequest(frame.id);
     pending.resolve(frame.payload);
   }
 
@@ -870,9 +900,7 @@ export class ModelProxy {
         this.handleModelStreamChat(frame, pending);
       }
     } catch {
-      clearTimeout(pending.timer);
-      this.pending.delete(frame.id);
-      this.streamText.delete(frame.id);
+      this.cleanupRequest(frame.id, pending.stableStreamId);
     }
   }
 
@@ -889,9 +917,7 @@ export class ModelProxy {
       pending.controller!.enqueue(pending.encoder!.encode(`data: ${JSON.stringify(finalChunk)}\n\n`));
       pending.controller!.enqueue(pending.encoder!.encode("data: [DONE]\n\n"));
       pending.controller!.close();
-      clearTimeout(pending.timer);
-      this.pending.delete(frame.id);
-      this.streamSetupSent.delete(stableId);
+      this.cleanupRequest(frame.id, stableId);
     } else {
       // Use full deltaObj when available (carries tool_calls etc.), otherwise simple text delta
       const delta = frame.payload.deltaObj ?? { content: frame.payload.delta };
@@ -907,13 +933,27 @@ export class ModelProxy {
       const stableId = pending.stableStreamId ?? frame.id;
       const fullText = this.streamText.get(frame.id) ?? "";
       this.streamText.delete(frame.id);
-      this.enqueueResponsesStreamDone(pending.controller!, pending.encoder!, stableId, pending.model ?? "", fullText, frame.payload.usage);
+
+      // If the remote forwarded the full response.completed event, emit it directly
+      const doneObj = frame.payload.deltaObj as { event?: string; data?: unknown } | undefined;
+      if (doneObj?.event === "response.completed" && doneObj.data) {
+        pending.controller!.enqueue(pending.encoder!.encode(`event: response.completed\ndata: ${JSON.stringify(doneObj.data)}\n\n`));
+      } else {
+        // Fallback: reconstruct text-only completion
+        this.enqueueResponsesStreamDone(pending.controller!, pending.encoder!, stableId, pending.model ?? "", fullText, frame.payload.usage);
+      }
       pending.controller!.close();
-      clearTimeout(pending.timer);
-      this.pending.delete(frame.id);
-      this.streamSetupSent.delete(stableId);
+      this.cleanupRequest(frame.id, stableId);
     } else {
-      // Accumulate text for done event
+      // Forward structured Responses API events (function_call, output_item, etc.)
+      const obj = frame.payload.deltaObj as { event?: string; data?: unknown } | undefined;
+      if (obj?.event && obj.data) {
+        pending.controller!.enqueue(pending.encoder!.encode(`event: ${obj.event}\ndata: ${JSON.stringify(obj.data)}\n\n`));
+        pending.hasContent = true;
+        return;
+      }
+
+      // Text delta
       this.streamText.set(frame.id, (this.streamText.get(frame.id) ?? "") + (frame.payload.delta ?? ""));
       const respStableId = pending.stableStreamId ?? frame.id;
       const evt = { type: "response.output_text.delta", item_id: `msg_${respStableId}`, output_index: 0, content_index: 0, delta: frame.payload.delta };
@@ -933,14 +973,14 @@ export class ModelProxy {
     } satisfies ModelStreamChunk);
   }
 
-  private sendStreamDone(to: string, id: string, usage?: { inputTokens: number; outputTokens: number }) {
+  private sendStreamDone(to: string, id: string, usage?: { inputTokens: number; outputTokens: number }, deltaObj?: unknown) {
     this.peerManager.sendTo(to, {
       type: "model_stream",
       id,
       from: this.config.nodeId,
       to,
       timestamp: Date.now(),
-      payload: { delta: "", done: true, usage },
+      payload: { delta: "", done: true, usage, ...(deltaObj !== undefined && { deltaObj }) },
     } satisfies ModelStreamChunk);
   }
 
@@ -967,7 +1007,12 @@ export class ModelProxy {
 
     try {
       const endpoint = this.resolveModelEndpoint(model);
-      const isResponsesApi = endpoint.api === "openai-responses" || endpoint.api === "openai-codex-responses";
+      // Use payload.api override from requesting side, or cached API from previous auto-detection
+      const cached = this.modelApiCache.get(model.id);
+      const cachedApi = (cached && Date.now() - cached.ts < ModelProxy.MODEL_API_CACHE_TTL) ? cached.api : undefined;
+      if (cached && !cachedApi) this.modelApiCache.delete(model.id); // expired
+      const effectiveApi = payload.api ?? cachedApi ?? endpoint.api;
+      const isResponsesApi = effectiveApi === "openai-responses" || effectiveApi === "openai-codex-responses";
       const path = isResponsesApi ? "/responses" : "/chat/completions";
       const url = `${endpoint.baseUrl}${path}`;
       const headers: Record<string, string> = { "Content-Type": "application/json" };
@@ -996,6 +1041,8 @@ export class ModelProxy {
           stream: payload.stream,
           temperature: payload.temperature,
           max_output_tokens: payload.maxTokens,
+          ...(payload.tools && { tools: payload.tools }),
+          ...(payload.toolChoice !== undefined && { tool_choice: payload.toolChoice }),
         };
       } else {
         const messages = srcFormat === "chat"
@@ -1008,6 +1055,8 @@ export class ModelProxy {
           max_tokens: payload.maxTokens,
           stream: payload.stream,
           ...(payload.stream ? { stream_options: { include_usage: true } } : {}),
+          ...(payload.tools && { tools: payload.tools }),
+          ...(payload.toolChoice !== undefined && { tool_choice: payload.toolChoice }),
         };
       }
 
@@ -1031,6 +1080,8 @@ export class ModelProxy {
           let buffer = "";
           let lastUsage: { inputTokens: number; outputTokens: number } | undefined;
           let streamDone = false;
+          let contentSent = false;
+          let completedEvent: unknown = undefined;
 
           while (!streamDone) {
             const { done, value } = await reader.read();
@@ -1053,7 +1104,6 @@ export class ModelProxy {
               if (!line.startsWith("data: ")) continue;
               const data = line.slice(6).trim();
               if (data === "[DONE]") {
-                this.sendStreamDone(from, id, lastUsage);
                 streamDone = true;
                 break;
               }
@@ -1067,33 +1117,37 @@ export class ModelProxy {
                     const delta = parsed.delta || "";
                     if (delta) {
                       this.sendStreamDelta(from, id, delta);
+                      contentSent = true;
                     }
+                  } else if (
+                    evtType === "response.output_item.added" ||
+                    evtType === "response.output_item.done" ||
+                    evtType === "response.content_part.added" ||
+                    evtType === "response.content_part.done" ||
+                    evtType === "response.output_text.done" ||
+                    evtType === "response.function_call_arguments.delta" ||
+                    evtType === "response.function_call_arguments.done"
+                  ) {
+                    // Forward structured Responses API events via deltaObj
+                    this.sendStreamDelta(from, id, "", { event: evtType, data: parsed });
+                    contentSent = true;
                   } else if (evtType === "response.completed") {
                     const usage = parsed.response?.usage;
-                    if (usage) {
-                      lastUsage = {
-                        inputTokens: usage.input_tokens ?? usage.prompt_tokens ?? 0,
-                        outputTokens: usage.output_tokens ?? usage.completion_tokens ?? 0,
-                      };
-                    }
-                    this.sendStreamDone(from, id, lastUsage);
+                    lastUsage = parseUsage(usage) ?? lastUsage;
+                    completedEvent = { event: evtType, data: parsed };
                     streamDone = true;
                     break;
                   }
                 } else {
                   // Chat completions format
-                  if (parsed.usage) {
-                    lastUsage = {
-                      inputTokens: parsed.usage.prompt_tokens,
-                      outputTokens: parsed.usage.completion_tokens,
-                    };
-                  }
+                  lastUsage = parseUsage(parsed.usage) ?? lastUsage;
                   const d = parsed.choices?.[0]?.delta;
                   const delta = d?.content || d?.reasoning_content || "";
                   // Pass full delta object when it contains tool_calls or other structured data
                   const hasStructured = d?.tool_calls || d?.refusal != null;
                   if (delta || hasStructured) {
                     this.sendStreamDelta(from, id, delta, hasStructured ? d : undefined);
+                    contentSent = true;
                   }
                 }
               } catch {
@@ -1102,9 +1156,30 @@ export class ModelProxy {
               currentEvent = "";
             }
           }
-          // If the upstream closed without sending [DONE] or response.completed,
-          // send a completion frame so the requesting side doesn't hang.
-          if (!streamDone) {
+
+          // Responses API stream produced no content — fall back to chat completions
+          if (isResponsesApi && !contentSent && !cachedApi) {
+            debug("model_req", `responses API stream produced no content for "${model.id}", retrying with chat completions`);
+            const chatResult = await this.retryWithChatCompletions(endpoint, modelField, payload, headers);
+            if (chatResult) {
+              this.modelApiCache.set(model.id, { api: "openai-completions", ts: Date.now() });
+              debug("model_req", `cached "${model.id}" as openai-completions (stream fallback)`);
+              if (chatResult.content) {
+                this.sendStreamDelta(from, id, chatResult.content);
+              }
+              this.sendStreamDone(from, id, chatResult.usage);
+            } else if (completedEvent) {
+              this.sendStreamDone(from, id, lastUsage, completedEvent);
+            } else {
+              this.sendStreamDone(from, id, lastUsage);
+            }
+          } else if (completedEvent) {
+            this.sendStreamDone(from, id, lastUsage, completedEvent);
+          } else if (!streamDone) {
+            // Upstream closed without sending [DONE] or response.completed
+            this.sendStreamDone(from, id, lastUsage);
+          } else {
+            // Chat completions [DONE] received
             this.sendStreamDone(from, id, lastUsage);
           }
         } finally {
@@ -1112,12 +1187,31 @@ export class ModelProxy {
         }
       } else {
         // Non-streaming response
-        const result = await response.json();
+        const responseText = await response.text();
+        let result: Record<string, unknown>;
+        let chatFallbackResult: Awaited<ReturnType<ModelProxy["retryWithChatCompletions"]>> = null;
+        try {
+          result = JSON.parse(responseText);
+        } catch {
+          // Upstream returned non-JSON (e.g. SSE in non-stream mode) — try chat completions fallback
+          if (!cachedApi && isResponsesApi) {
+            debug("model_req", `responses API returned non-JSON for "${model.id}", retrying with chat completions`);
+            chatFallbackResult = await this.retryWithChatCompletions(endpoint, modelField, payload, headers);
+            if (chatFallbackResult) {
+              this.modelApiCache.set(model.id, { api: "openai-completions", ts: Date.now() });
+              debug("model_req", `cached "${model.id}" as openai-completions (non-JSON fallback)`);
+            }
+          }
+          if (!chatFallbackResult) throw new Error(`Upstream returned non-JSON: ${responseText.slice(0, 100)}`);
+          result = {}; // unused — chatFallbackResult takes precedence
+        }
         let content: string;
         let message: unknown | undefined;
         let usage: { inputTokens: number; outputTokens: number } | undefined;
 
-        if (isResponsesApi) {
+        if (chatFallbackResult) {
+          ({ content, message, usage } = chatFallbackResult);
+        } else if (isResponsesApi) {
           // Responses API: extract text from output[].content[].text
           content = "";
           const output = result.output as { type?: string; content?: { type?: string; text?: string }[] }[] | undefined;
@@ -1130,13 +1224,27 @@ export class ModelProxy {
               }
             }
           }
-          // Carry full output array for structured data (function_call items, etc.)
-          message = result.output;
-          if (result.usage) {
-            usage = {
-              inputTokens: result.usage.input_tokens ?? result.usage.prompt_tokens ?? 0,
-              outputTokens: result.usage.output_tokens ?? result.usage.completion_tokens ?? 0,
-            };
+
+          // Auto-detect: if Responses API returned empty output but produced tokens,
+          // the upstream adapter likely doesn't support Responses API properly.
+          // Retry with chat completions and cache the result.
+          const parsedUsage = parseUsage(result.usage as Record<string, number> | undefined);
+          const hasMessage = Array.isArray(output) && output.some((o: { type?: string }) => o.type === "message");
+          if (!hasMessage && (parsedUsage?.outputTokens ?? 0) > 0 && !cachedApi) {
+            debug("model_req", `responses API returned empty output for "${model.id}" (output_tokens=${parsedUsage!.outputTokens}), retrying with chat completions`);
+            const chatResult = await this.retryWithChatCompletions(endpoint, modelField, payload, headers);
+            if (chatResult) {
+              this.modelApiCache.set(model.id, { api: "openai-completions", ts: Date.now() });
+              debug("model_req", `cached "${model.id}" as openai-completions`);
+              ({ content, message, usage } = chatResult);
+            } else {
+              message = result.output;
+              usage = parsedUsage;
+            }
+          } else {
+            // Carry full output array for structured data (function_call items, etc.)
+            message = result.output;
+            usage = parsedUsage;
           }
         } else {
           // Chat completions format
@@ -1146,12 +1254,7 @@ export class ModelProxy {
           if (msg?.tool_calls || msg?.refusal != null || msg?.function_call) {
             message = msg;
           }
-          if (result.usage) {
-            usage = {
-              inputTokens: result.usage.prompt_tokens,
-              outputTokens: result.usage.completion_tokens,
-            };
-          }
+          usage = parseUsage(result.usage);
         }
 
         this.peerManager.sendTo(from, {
@@ -1182,4 +1285,44 @@ export class ModelProxy {
       } satisfies ModelResponse);
     }
   }
+
+  /** Retry a model request using chat completions format (fallback from Responses API). */
+  private async retryWithChatCompletions(
+    endpoint: { baseUrl: string; apiKey?: string; direct: boolean; api: string },
+    modelField: string,
+    payload: ModelRequest["payload"],
+    headers: Record<string, string>,
+  ): Promise<{ content: string; message?: unknown; usage?: { inputTokens: number; outputTokens: number } } | null> {
+    try {
+      const srcFormat = payload.inputFormat ?? "chat";
+      const messages = srcFormat === "chat"
+        ? payload.messages
+        : ModelProxy.normalizeResponsesInput(payload.messages);
+      const chatBody: Record<string, unknown> = {
+        model: modelField,
+        messages,
+        temperature: payload.temperature,
+        max_tokens: payload.maxTokens,
+        stream: false,
+        ...(payload.tools && { tools: payload.tools }),
+        ...(payload.toolChoice !== undefined && { tool_choice: payload.toolChoice }),
+      };
+      const chatUrl = `${endpoint.baseUrl}/chat/completions`;
+      const chatResp = await fetch(chatUrl, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(chatBody),
+      });
+      if (!chatResp.ok) return null;
+      const chatResult = await chatResp.json();
+      const msg = chatResult.choices?.[0]?.message;
+      const content = msg?.content || msg?.reasoning_content || "";
+      const message = (msg?.tool_calls || msg?.refusal != null || msg?.function_call) ? msg : undefined;
+      const usage = parseUsage(chatResult.usage);
+      return { content, message, usage };
+    } catch (err) {
+      debug("model_req", `retryWithChatCompletions failed for "${modelField}": ${err instanceof Error ? err.message : String(err)}`);
+      return null;
+    }
+  }
 }

package/src/peer-manager.ts

@@ -213,13 +213,16 @@ export class PeerManager extends EventEmitter<PeerManagerEvents> {
       });
     });
 
-    ws.addEventListener("error", () => {
-      this.scheduleReconnect(peer);
-    });
+    let reconnectScheduled = false;
+    const tryReconnect = () => {
+      if (!reconnectScheduled) {
+        reconnectScheduled = true;
+        this.scheduleReconnect(peer);
+      }
+    };
 
-    ws.addEventListener("close", () => {
-      this.scheduleReconnect(peer);
-    });
+    ws.addEventListener("error", tryReconnect);
+    ws.addEventListener("close", tryReconnect);
   }
 
   private scheduleReconnect(peer: PeerConfig) {

package/src/types.ts

@@ -101,6 +101,8 @@ export interface ModelRequest extends ClusterFrame {
     /** Format of `messages`: "chat" = OpenAI chat completions, "responses" = OpenAI Responses API input items.
      *  Defaults to "chat" for backward compatibility. */
     inputFormat?: "chat" | "responses";
+    tools?: unknown[];
+    toolChoice?: unknown;
     temperature?: number;
     maxTokens?: number;
     stream: boolean;