clawmatrix 0.2.9 → 0.3.1

package/src/handoff.ts

@@ -54,16 +54,33 @@ export class HandoffManager {
   // Multi-device sync: track which nodes are watching each handoff session (by sessionId)
   private sessionWatchers = new Map<string, Set<string>>();
 
+  // Pre-built indexes for O(1) agent lookup
+  private agentById = new Map<string, ClawMatrixConfig["agents"][number]>();
+  private agentsByTag = new Map<string, ClawMatrixConfig["agents"][number]>();
+
   constructor(config: ClawMatrixConfig, peerManager: PeerManager, gatewayInfo: GatewayInfo) {
     this.config = config;
     this.peerManager = peerManager;
     this.gatewayInfo = gatewayInfo;
     this.taskActivity = new TaskActivityBroadcaster(config, peerManager);
 
+    // Build agent indexes
+    for (const a of config.agents) {
+      this.agentById.set(a.id, a);
+      for (const t of a.tags) this.agentsByTag.set(t, a);
+    }
+
     // Periodically clean up stale input_required entries
     this.staleCleanupTimer = setInterval(() => this.cleanupStale(), STALE_CLEANUP_INTERVAL);
   }
 
+  private findLocalAgent(target: string): ClawMatrixConfig["agents"][number] | undefined {
+    if (target.startsWith("tags:")) {
+      return this.agentsByTag.get(target.slice(5));
+    }
+    return this.agentById.get(target);
+  }
+
   // ── Multi-device sync helpers ──────────────────────────────────
 
   private addSessionWatcher(sessionId: string, nodeId: string) {
@@ -271,13 +288,7 @@ export class HandoffManager {
     const { id, from, payload } = frame;
 
     // Find matching local agent
-    const agent = this.config.agents.find((a) => {
-      if (payload.target.startsWith("tags:")) {
-        const tag = payload.target.slice(5);
-        return a.tags.includes(tag);
-      }
-      return a.id === payload.target;
-    });
+    const agent = this.findLocalAgent(payload.target);
 
     if (!agent) {
       this.peerManager.sendTo(from, {
@@ -477,6 +488,8 @@ export class HandoffManager {
     const decoder = new TextDecoder();
     const chunks: string[] = [];
     let buffer = "";
+    // Cache active entry lookup outside the hot loop — stable during the stream
+    const activeEntry = this.active.get(handoffId);
 
     try {
       while (true) {
@@ -510,7 +523,6 @@ export class HandoffManager {
 
               // Broadcast progress to mobile nodes (throttled, detail is just
               // a heartbeat — don't send token-level deltas as they're meaningless fragments)
-              const activeEntry = this.active.get(handoffId);
               if (activeEntry) {
                 this.taskActivity.broadcast(
                   handoffId, "handoff", "progress", activeEntry.agent, activeEntry.startedAt,
@@ -605,6 +617,7 @@ export class HandoffManager {
 
     // If canceled during input_required, no runAgentTurn is running to clean up
     if (wasInputRequired) {
+      this.sessionWatchers.delete(entry.sessionId);
       this.active.delete(frame.id);
     }
 

package/src/health-tracker.ts

@@ -132,12 +132,17 @@ export class HealthTracker {
   // ── Event recording ─────────────────────────────────────────
 
   recordEvent(event: HealthEvent) {
+    // Strip undefined values — Automerge rejects them
+    const clean: Record<string, unknown> = {};
+    for (const [k, v] of Object.entries(event)) {
+      if (v !== undefined) clean[k] = v;
+    }
     this.doc = Automerge.change(this.doc, (d) => {
       if (!d.nodes[this.nodeId]) {
         d.nodes[this.nodeId] = { events: [], lastUpdated: 0 };
       }
       const entry = d.nodes[this.nodeId]!;
-      entry.events.push({ ...event });
+      entry.events.push(clean as HealthEvent);
       entry.lastUpdated = Date.now();
     });
     this.scheduleSave();

package/src/index.ts

@@ -16,6 +16,7 @@ import { createClusterDiagnosticTool } from "./tools/cluster-diagnostic.ts";
 import { createClusterAcpTool } from "./tools/cluster-acp.ts";
 import { createClusterTerminalTool } from "./tools/cluster-terminal.ts";
 import { createClusterToolInvokeTool } from "./tools/cluster-tool.ts";
+import { createClusterTransferTool } from "./tools/cluster-transfer.ts";
 import { registerClusterCli } from "./cli.ts";
 import { spawnProcess } from "./compat.ts";
 
@@ -271,7 +272,7 @@ const plugin = {
 
     const repatchTimer = setInterval(patchAllConfigs, 10_000);
     repatchTimer.unref?.();
-    api.on("dispose", () => clearInterval(repatchTimer));
+    api.on("gateway_stop", () => clearInterval(repatchTimer));
 
     for (const [nodeId, models] of Object.entries(modelsByNode)) {
       api.registerProvider({
@@ -302,6 +303,7 @@ const plugin = {
     api.registerTool(createClusterAcpTool(), { optional: true });
     api.registerTool(createClusterTerminalTool(), { optional: true });
     api.registerTool(createClusterToolInvokeTool(), { optional: true });
+    api.registerTool(createClusterTransferTool(), { optional: true });
 
     // Wire up peer approval with OpenClaw channel API
     if (config.peerApproval.enabled) {
@@ -330,20 +332,23 @@ const plugin = {
 
             const proc = spawnProcess(
               ["openclaw", "gateway", "call", "send", "--json", "--params", JSON.stringify(sendParams)],
-              { stdout: "pipe", stderr: "pipe" },
+              { stdout: "ignore", stderr: "pipe" },
             );
-            const code = await proc.exited;
-            if (code !== 0) {
-              const stderrChunks: Uint8Array[] = [];
+            // Collect stderr concurrently with waiting for exit to avoid pipe deadlock
+            const stderrPromise = (async () => {
+              const chunks: Uint8Array[] = [];
               if (proc.stderr) {
                 const reader = proc.stderr.getReader();
                 while (true) {
                   const { done, value } = await reader.read();
                   if (done) break;
-                  stderrChunks.push(value);
+                  chunks.push(value);
                 }
               }
-              const errMsg = Buffer.concat(stderrChunks).toString("utf-8").trim();
+              return Buffer.concat(chunks).toString("utf-8").trim();
+            })();
+            const [code, errMsg] = await Promise.all([proc.exited, stderrPromise]);
+            if (code !== 0) {
               throw new Error(`gateway send failed (exit ${code}): ${errMsg}`);
             }
           });
@@ -550,6 +555,180 @@ const plugin = {
       },
     );
 
+    // ── Tool proxy CLI gateway methods ──────────────────────────────
+
+    api.registerGatewayMethod(
+      "clawmatrix.tools.list",
+      ({ params, respond }: GatewayRequestHandlerOptions) => {
+        try {
+          const runtime = getClusterRuntime();
+          const { node } = (params ?? {}) as { node?: string };
+          const allPeers = runtime.peerManager.router.getAllPeers();
+          const peers = mergeSentinelPeers(allPeers, runtime)
+            .filter((p) => (p as { nodeId: string }).nodeId !== config.nodeId);
+
+          type PeerToolInfo = { nodeId: string; status: string; toolProxy?: import("./types.ts").ToolProxyInfo };
+          const result: PeerToolInfo[] = [];
+
+          for (const peer of peers) {
+            const p = peer as { nodeId: string; connected: boolean; status: string; toolProxy?: import("./types.ts").ToolProxyInfo };
+            if (node && p.nodeId !== node) continue;
+            if (!p.toolProxy?.enabled) continue;
+            result.push({
+              nodeId: p.nodeId,
+              status: p.status,
+              toolProxy: p.toolProxy,
+            });
+          }
+
+          respond(true, result);
+        } catch {
+          respond(false, { error: "ClawMatrix service not running" });
+        }
+      },
+    );
+
+    api.registerGatewayMethod(
+      "clawmatrix.tools.call",
+      async ({ params, respond }: GatewayRequestHandlerOptions) => {
+        try {
+          const runtime = getClusterRuntime();
+          const { node, tool, params: toolParams, timeout } = (params ?? {}) as {
+            node?: string; tool?: string; params?: Record<string, unknown>; timeout?: number;
+          };
+
+          if (!node || !tool) {
+            respond(false, { error: "Missing required params: node, tool" });
+            return;
+          }
+
+          const result = await runtime.toolProxy.invoke(node, tool, toolParams ?? {}, timeout);
+          respond(true, result);
+        } catch (err) {
+          respond(false, { error: err instanceof Error ? err.message : String(err) });
+        }
+      },
+    );
+
+    api.registerGatewayMethod(
+      "clawmatrix.tools.batch",
+      async ({ params, respond }: GatewayRequestHandlerOptions) => {
+        try {
+          const runtime = getClusterRuntime();
+          const { node, items, stopOnError, timeout } = (params ?? {}) as {
+            node?: string;
+            items?: Array<{ tool: string; params?: Record<string, unknown> }>;
+            stopOnError?: boolean;
+            timeout?: number;
+          };
+
+          if (!node || !items || items.length === 0) {
+            respond(false, { error: "Missing required params: node, items" });
+            return;
+          }
+
+          const batchItems = items.map((item) => ({
+            tool: item.tool,
+            params: item.params ?? {},
+          }));
+
+          const results = await runtime.toolProxy.invokeBatch(node, batchItems, { stopOnError, timeout });
+          respond(true, results);
+        } catch (err) {
+          respond(false, { error: err instanceof Error ? err.message : String(err) });
+        }
+      },
+    );
+
+    // ── Models CLI gateway method ──────────────────────────────────
+
+    api.registerGatewayMethod(
+      "clawmatrix.models.list",
+      ({ params, respond }: GatewayRequestHandlerOptions) => {
+        try {
+          const runtime = getClusterRuntime();
+          const { node } = (params ?? {}) as { node?: string };
+          const allProxyModels = runtime.modelProxy.allProxyModels;
+
+          const reachable = new Set(
+            runtime.peerManager.router.getAllPeers()
+              .filter((p) => p.connection?.isOpen || p.reachableVia)
+              .map((p) => p.nodeId),
+          );
+
+          const models = allProxyModels
+            .filter((m) => !node || m.nodeId === node)
+            .map((m) => ({
+              id: m.id,
+              nodeId: m.nodeId,
+              provider: m.provider ?? m.nodeId,
+              ...(m.description && { description: m.description }),
+              ...(m.contextWindow && { contextWindow: m.contextWindow }),
+              ...(m.maxTokens && { maxTokens: m.maxTokens }),
+              ...(m.reasoning !== undefined && { reasoning: m.reasoning }),
+              ...(m.input && { input: m.input }),
+              ...(m.api && { api: m.api }),
+              ...(m.cost && { cost: m.cost }),
+              reachable: reachable.has(m.nodeId),
+            }));
+
+          respond(true, models);
+        } catch {
+          respond(false, { error: "ClawMatrix service not running" });
+        }
+      },
+    );
+
+    // ── Events CLI gateway methods ──────────────────────────────────
+
+    api.registerGatewayMethod(
+      "clawmatrix.events.query",
+      ({ params, respond }: GatewayRequestHandlerOptions) => {
+        try {
+          const runtime = getClusterRuntime();
+          if (!runtime.webHandler) {
+            respond(false, { error: "Events not enabled (web.enabled = false)" });
+            return;
+          }
+          const { type, source, since, unconsumed, limit } = (params ?? {}) as {
+            type?: string; source?: string; since?: number; unconsumed?: boolean; limit?: number;
+          };
+          const events = runtime.webHandler.queryEvents({
+            type,
+            source,
+            since,
+            unconsumed: unconsumed ?? true,
+            limit: limit ?? 20,
+          });
+          respond(true, events);
+        } catch {
+          respond(false, { error: "ClawMatrix service not running" });
+        }
+      },
+    );
+
+    api.registerGatewayMethod(
+      "clawmatrix.events.consume",
+      ({ params, respond }: GatewayRequestHandlerOptions) => {
+        try {
+          const runtime = getClusterRuntime();
+          if (!runtime.webHandler) {
+            respond(false, { error: "Events not enabled (web.enabled = false)" });
+            return;
+          }
+          const { ids } = (params ?? {}) as { ids?: string[] };
+          if (!ids || ids.length === 0) {
+            respond(false, { error: "Missing required param: ids (array of event IDs)" });
+            return;
+          }
+          const consumed = runtime.webHandler.consumeEvents(ids);
+          respond(true, { consumed, ids });
+        } catch {
+          respond(false, { error: "ClawMatrix service not running" });
+        }
+      },
+    );
+
     // Log model selection on each LLM call (fire-and-forget)
     api.on("llm_input", (event) => {
       api.logger.debug(`[clawmatrix] llm_input: provider=${event.provider} model=${event.model}`);
@@ -558,6 +737,10 @@ const plugin = {
     // CLI subcommand
     api.registerCli(registerClusterCli, { commands: ["clawmatrix"] });
 
+    // Auto-install global `clawmatrix` shim next to the `openclaw` binary.
+    // Runs once on plugin load; non-blocking, best-effort.
+    installGlobalCliShim(api.logger);
+
     // Plugin command: /clawmatrix approve|deny|revoke
     // Handles Telegram callback buttons and other chat surfaces.
     // Plugin commands are processed before the agent, so they bypass the LLM.
@@ -646,11 +829,9 @@ const plugin = {
             lines.push("[ClawMatrix] No peers online. Use cluster_peers to check cluster status.");
           } else {
             lines.push(
-              `[ClawMatrix Cluster] YOU ARE node="${config.nodeId}"${config.tags.length ? ` tags=${config.tags.join(",")}` : ""}. This is YOUR identity — never target yourself with cluster tools.`,
+              `[ClawMatrix Cluster] YOU ARE node="${config.nodeId}"${config.tags.length ? ` tags=${config.tags.join(",")}` : ""}. ${peerCount} peer(s) online.`,
               ...(config.agents.length > 0 ? [`Role: ${config.agents[0]!.description}`] : []),
-              `${peerCount} remote peer(s) online. Use cluster_peers to see topology, agents, and models.`,
-              "Prefer cluster_tool for device-specific tools (screenshot, battery, etc.); cluster_exec/read/write for file/shell ops; cluster_handoff for complex multi-step tasks.",
-              "IMPORTANT: Always tell user which remote node you're targeting before calling cluster tools.",
+              "Use cluster_peers to see topology. Always tell user which remote node you're targeting before calling cluster tools.",
             );
           }
           cachedSystemContext = lines.join("\n");
@@ -748,6 +929,9 @@ function mergeSentinelPeers(
       status: effectiveStatus,
       reachableVia: p.reachableVia,
       latencyMs: p.latencyMs,
+      toolProxy: p.toolProxy,
+      acpAgents: p.acpAgents,
+      deviceInfo: p.deviceInfo,
       ...(sentinel ? { sentinel: sentinelOnline ? "online" : "offline" } : {}),
     });
   }
@@ -772,4 +956,54 @@ function mergeSentinelPeers(
   return result;
 }
 
+/** Auto-install a global `clawmatrix` CLI shim next to the `openclaw` binary. */
+function installGlobalCliShim(logger: { info: (msg: string) => void; warn: (msg: string) => void }) {
+  try {
+    const fs = require("node:fs") as typeof import("node:fs");
+    const path = require("node:path") as typeof import("node:path");
+
+    // Find the real directory where `openclaw` lives (resolve symlinks)
+    const openclawBin = process.argv[0]; // node process running openclaw
+    // Walk up to find the openclaw binary: it's in the same bin dir as node,
+    // or we can resolve it from process.env.PATH
+    let binDir: string | null = null;
+    const envPath = process.env.PATH ?? "";
+    for (const dir of envPath.split(path.delimiter)) {
+      const candidate = path.join(dir, "openclaw");
+      try {
+        fs.accessSync(candidate, fs.constants.X_OK);
+        // Resolve symlinks to get the real bin directory
+        const realPath = fs.realpathSync(candidate);
+        binDir = path.dirname(realPath);
+        break;
+      } catch {
+        // Not in this dir
+      }
+    }
+    if (!binDir) return;
+
+    const shimPath = path.join(binDir, "clawmatrix");
+
+    // Skip if shim already exists and is our shim (check marker comment)
+    try {
+      const existing = fs.readFileSync(shimPath, "utf-8");
+      if (existing.includes("clawmatrix-shim")) return; // already installed
+    } catch {
+      // File doesn't exist, proceed to create
+    }
+
+    const shim = [
+      "#!/usr/bin/env sh",
+      "# clawmatrix-shim: auto-installed by clawmatrix plugin",
+      'exec openclaw clawmatrix "$@"',
+      "",
+    ].join("\n");
+
+    fs.writeFileSync(shimPath, shim, { mode: 0o755 });
+    logger.info(`[clawmatrix] Installed global CLI shim: ${shimPath}`);
+  } catch {
+    // Best-effort: don't break plugin loading if shim install fails
+  }
+}
+
 export default plugin;

package/src/knowledge-sync.ts

@@ -1,6 +1,6 @@
 import * as Automerge from "@automerge/automerge";
 import { watch, type FSWatcher } from "node:fs";
-import { readdir, readFile, stat as fsStat, writeFile, mkdir, rename } from "node:fs/promises";
+import { readdir, readFile, stat as fsStat, writeFile, mkdir, rename, unlink } from "node:fs/promises";
 import path from "node:path";
 import ignore, { type Ignore } from "ignore";
 import picomatch from "picomatch";
@@ -128,8 +128,10 @@ export class KnowledgeSync {
   // ── FS / Watcher ───────────────────────────────────────────────
   private watcher: FSWatcher | null = null;
   private debounceTimer: ReturnType<typeof setTimeout> | null = null;
+  private localChangesRunning = false;
+  private localChangesQueued = false;
   /** Paths recently written by exportFileToFs — suppress watcher re-trigger. Stores {content, timestamp}. */
-  private writtenByExport = new Map<string, { content: string; ts: number }>();
+  private writtenByExport = new Map<string, { content: string | null; ts: number }>();
   /** Deferred git commit timer — batches multiple remote syncs into one commit. */
   private gitCommitTimer: ReturnType<typeof setTimeout> | null = null;
   private pendingGitSources = new Set<string>();
@@ -324,10 +326,20 @@ export class KnowledgeSync {
     await this.saveAutomergeDoc(this.registryPath, this.registry);
 
     // Discover new files from registry and initiate their sync
+    const deletedPaths: string[] = [];
     for (const [relPath, meta] of Object.entries(newDoc.files ?? {})) {
       if (meta.deleted) {
-        // Clean up sync states for deleted files
+        // Clean up sync states, in-memory doc, persisted doc, and local file
         this.cleanupDeletedFileSyncStates(relPath);
+        if (this.fileDocs.has(relPath)) {
+          this.fileDocs.delete(relPath);
+          deletedPaths.push(relPath);
+          // Remove persisted automerge doc
+          const docPath = path.join(this.docsDir, docFileName(relPath));
+          await rename(docPath, docPath + ".deleted").catch(() => {});
+          // Delete local file from workspace
+          await this.deleteLocalFile(relPath);
+        }
         continue;
       }
       if (!this.fileDocs.has(relPath)) {
@@ -337,6 +349,12 @@ export class KnowledgeSync {
       this.syncDocWithPeer(peerId, relPath);
     }
 
+    // Commit remote deletions to git
+    if (deletedPaths.length > 0) {
+      debug(TAG, `remote deletion from ${peerId}: ${deletedPaths.join(", ")}`);
+      this.schedulePendingGitCommit(peerId);
+    }
+
     this.sendSyncMessage(peerId, REGISTRY_DOC_ID);
   }
 
@@ -474,6 +492,27 @@ export class KnowledgeSync {
   }
 
   private async handleLocalChanges() {
+    // Mutex: if already running, mark queued and return — the running
+    // invocation will re-run when it finishes to pick up new changes.
+    if (this.localChangesRunning) {
+      this.localChangesQueued = true;
+      return;
+    }
+    this.localChangesRunning = true;
+    try {
+      await this.handleLocalChangesInner();
+    } finally {
+      this.localChangesRunning = false;
+      if (this.localChangesQueued) {
+        this.localChangesQueued = false;
+        this.handleLocalChanges().catch((err) => {
+          debug(TAG, `queued local change handling error: ${err}`);
+        });
+      }
+    }
+  }
+
+  private async handleLocalChangesInner() {
     // Only process files in pendingChanges (incremental)
     const changesToProcess = new Set(this.pendingChanges);
     this.pendingChanges.clear();
@@ -802,7 +841,13 @@ export class KnowledgeSync {
     if (this.isIgnored(relPath)) return;
 
     const content = doc.content ?? "";
-    const absPath = path.join(this.opts.workspacePath, relPath);
+    const absPath = path.resolve(this.opts.workspacePath, relPath);
+
+    // Prevent path traversal (e.g. relPath = "../../etc/passwd")
+    if (!absPath.startsWith(this.opts.workspacePath + path.sep) && absPath !== this.opts.workspacePath) {
+      debug(TAG, `blocked path traversal attempt: ${relPath}`);
+      return;
+    }
 
     let currentContent: string | null = null;
     try {
@@ -817,6 +862,30 @@ export class KnowledgeSync {
     }
   }
 
+  /** Delete a local file from workspace (triggered by remote deletion). */
+  private async deleteLocalFile(relPath: string) {
+    const absPath = path.resolve(this.opts.workspacePath, relPath);
+
+    // Prevent path traversal
+    if (!absPath.startsWith(this.opts.workspacePath + path.sep) && absPath !== this.opts.workspacePath) {
+      debug(TAG, `blocked path traversal on delete: ${relPath}`);
+      return;
+    }
+
+    // Mark as our own deletion so the watcher doesn't re-process it.
+    // handleLocalChangesInner sees currentContent === null === marker.content and skips.
+    this.writtenByExport.set(relPath, { content: null, ts: Date.now() });
+    try {
+      await unlink(absPath);
+      debug(TAG, `deleted local file: ${relPath}`);
+    } catch (err) {
+      // File may not exist locally — that's fine
+      if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
+        debug(TAG, `failed to delete local file ${relPath}: ${err}`);
+      }
+    }
+  }
+
   /** Read all workspace files matching whitelist. */
   private async readWhitelistedFiles(): Promise<Record<string, string>> {
     const files: Record<string, string> = {};
@@ -925,9 +994,7 @@ export class KnowledgeSync {
       }
 
       let doc = Automerge.init<FileDoc>();
-      doc = Automerge.change(doc, (d) => {
-        (d as FileDoc).content = content;
-      });
+      doc = changeFileContent(doc, content);
       this.fileDocs.set(relPath, doc);
 
       this.registry = Automerge.change(this.registry, (d) => {

package/src/model-proxy.ts

@@ -754,7 +754,7 @@ export class ModelProxy {
     let currentId = requestId;
     let currentTarget = targetNodeId;
     let currentFrame = frame;
-    let remaining = failoverCandidates;
+    let failoverIdx = 0; // index into failoverCandidates (avoids slice allocations)
     const maxAttempts = failoverCandidates.length + 1;
 
     for (let attempt = 0; attempt < maxAttempts; attempt++) {
@@ -763,13 +763,13 @@ export class ModelProxy {
 
         if (!result.success) {
           // Upstream error — try failover if available
-          if (remaining.length > 0 && buildFrame) {
-            const next = remaining[0]!;
-            debug("proxy", `failover: remote error "${result.error}" → trying ${next.routeNodeId} (${remaining.length - 1} left)`);
+          if (failoverIdx < failoverCandidates.length && buildFrame) {
+            const next = failoverCandidates[failoverIdx]!;
+            debug("proxy", `failover: remote error "${result.error}" → trying ${next.routeNodeId} (${failoverCandidates.length - failoverIdx - 1} left)`);
+            failoverIdx++;
             currentId = crypto.randomUUID();
             currentFrame = buildFrame(next, currentId);
             currentTarget = next.routeNodeId;
-            remaining = remaining.slice(1);
             continue;
           }
           return {
@@ -782,13 +782,13 @@ export class ModelProxy {
         return this.formatNonStreamResult(result, currentId, currentFrame, responseFormat);
       } catch (err) {
         // Timeout or send failure — try failover
-        if (remaining.length > 0 && buildFrame) {
-          const next = remaining[0]!;
-          debug("proxy", `failover: ${err instanceof Error ? err.message : String(err)} → trying ${next.routeNodeId} (${remaining.length - 1} left)`);
+        if (failoverIdx < failoverCandidates.length && buildFrame) {
+          const next = failoverCandidates[failoverIdx]!;
+          debug("proxy", `failover: ${err instanceof Error ? err.message : String(err)} → trying ${next.routeNodeId} (${failoverCandidates.length - failoverIdx - 1} left)`);
+          failoverIdx++;
           currentId = crypto.randomUUID();
           currentFrame = buildFrame(next, currentId);
           currentTarget = next.routeNodeId;
-          remaining = remaining.slice(1);
           continue;
         }
         return {
@@ -980,6 +980,24 @@ export class ModelProxy {
     const pending = this.pending.get(frame.id);
     if (!pending?.stream || !pending.controller || !pending.encoder) return;
 
+    // Reset activity timer — keeps long-running streams alive and detects
+    // stalled connections within modelTimeout of the last received chunk.
+    clearTimeout(pending.timer);
+    if (!frame.payload.done) {
+      pending.timer = setTimeout(() => {
+        // Capture references before cleanup removes pending from the map
+        const { stableStreamId, responseFormat, controller, encoder, model, failoverCandidates, buildFrame } = pending;
+        this.cleanupRequest(frame.id);
+        this.peerManager.router.markFailed(frame.id);
+        this.tryStreamFailover(
+          stableStreamId ?? frame.id, responseFormat,
+          controller!, encoder!, model ?? "",
+          failoverCandidates ?? [], buildFrame,
+          `stream stalled (no data for ${this.modelTimeout / 1000}s)`,
+        );
+      }, this.modelTimeout);
+    }
+
     try {
       if (pending.responseFormat === "responses") {
         this.handleModelStreamResponses(frame, pending);
@@ -1305,7 +1323,14 @@ export class ModelProxy {
         let chatFallbackResult: Awaited<ReturnType<ModelProxy["retryWithChatCompletions"]>> = null;
         try {
           result = JSON.parse(responseText);
-        } catch {
+          // Detect error objects in 200 OK responses (some APIs return HTTP 200 with error body)
+          if (result.error && typeof result.error === "object" && !result.choices && !result.output) {
+            const errMsg = (result.error as { message?: string }).message ?? JSON.stringify(result.error);
+            throw new Error(`Upstream error (200 OK): ${String(errMsg).slice(0, 200)}`);
+          }
+        } catch (parseErr) {
+          // Re-throw non-parse errors (e.g. upstream error detection above)
+          if (!(parseErr instanceof SyntaxError)) throw parseErr;
           // Upstream returned non-JSON (e.g. SSE in non-stream mode) — try chat completions fallback
           if (!cachedApi && isResponsesApi) {
             debug("model_req", `responses API returned non-JSON for "${model.id}", retrying with chat completions`);