clawmatrix 0.1.20 → 0.1.22

package/src/model-proxy.ts

@@ -22,6 +22,11 @@ interface ProxyResponse {
   body: string | ReadableStream;
 }
 
+interface FailoverCandidate {
+  proxyModel: import("./config.ts").ProxyModel | undefined;
+  routeNodeId: string;
+}
+
 interface PendingModelReq {
   resolve: (value: unknown) => void;
   reject: (error: Error) => void;
@@ -31,6 +36,14 @@ interface PendingModelReq {
   model?: string;
   controller?: ReadableStreamDefaultController;
   encoder?: TextEncoder;
+  /** Whether real content (not just setup events) has been sent to the stream. */
+  hasContent?: boolean;
+  /** Remaining failover candidates (excludes the currently attempted node). */
+  failoverCandidates?: FailoverCandidate[];
+  /** Factory to build a new frame for the next failover candidate. */
+  buildFrame?: (candidate: FailoverCandidate, newId: string) => ModelRequest;
+  /** Stable ID for the entire stream (for setup events & final close). */
+  stableStreamId?: string;
 }
 
 export class ModelProxy {
@@ -80,6 +93,26 @@ export class ModelProxy {
         continue;
       }
 
+      // function_call → merge into assistant message with tool_calls
+      if (obj.type === "function_call") {
+        const toolCall = {
+          id: obj.call_id ?? obj.id,
+          type: "function",
+          function: {
+            name: obj.name,
+            arguments: typeof obj.arguments === "string" ? obj.arguments : JSON.stringify(obj.arguments),
+          },
+        };
+        // Merge consecutive function_calls into one assistant message
+        const last = messages[messages.length - 1] as { role?: string; tool_calls?: unknown[] } | undefined;
+        if (last?.role === "assistant" && Array.isArray(last.tool_calls)) {
+          last.tool_calls.push(toolCall);
+        } else {
+          messages.push({ role: "assistant", content: null, tool_calls: [toolCall] });
+        }
+        continue;
+      }
+
       const role = typeof obj.role === "string" ? obj.role : "user";
 
       // Simple shorthand: {role: "user", content: "hello"}
@@ -133,6 +166,52 @@ export class ModelProxy {
     return messages;
   }
 
+  /**
+   * Convert chat-format messages back to Responses API input items.
+   *
+   * Reverses normalizeResponsesInput:
+   *   - {role: "tool", tool_call_id, content} → {type: "function_call_output", call_id, output}
+   *   - {role: "assistant", tool_calls: [...]} → {type: "function_call", ...} items
+   *   - {role: "developer"|"user"|"assistant"|"system", content} → pass through
+   */
+  private static chatToResponsesInput(messages: unknown[]): unknown[] {
+    const items: unknown[] = [];
+    for (const msg of messages) {
+      if (!msg || typeof msg !== "object") continue;
+      const obj = msg as Record<string, unknown>;
+
+      if (obj.role === "tool") {
+        items.push({
+          type: "function_call_output",
+          call_id: obj.tool_call_id,
+          output: typeof obj.content === "string" ? obj.content : JSON.stringify(obj.content),
+        });
+      } else if (obj.role === "assistant" && Array.isArray(obj.tool_calls)) {
+        // Emit text content first if present
+        if (typeof obj.content === "string" && obj.content) {
+          items.push({ role: "assistant", content: obj.content });
+        }
+        // Convert each tool_call to a function_call item
+        for (const tc of obj.tool_calls) {
+          if (!tc || typeof tc !== "object") continue;
+          const call = tc as Record<string, unknown>;
+          const fn = call.function as Record<string, unknown> | undefined;
+          if (fn) {
+            items.push({
+              type: "function_call",
+              call_id: call.id,
+              name: fn.name,
+              arguments: typeof fn.arguments === "string" ? fn.arguments : JSON.stringify(fn.arguments),
+            });
+          }
+        }
+      } else {
+        items.push(obj);
+      }
+    }
+    return items;
+  }
+
   /** Resolve API endpoint for a model: explicit config > OpenClaw provider > gateway fallback */
   private resolveModelEndpoint(model: { id: string; provider: string; baseUrl?: string; apiKey?: string; api?: string }): { baseUrl: string; apiKey?: string; direct: boolean; api: string } {
     const defaultApi = "openai-completions";
@@ -223,6 +302,7 @@ export class ModelProxy {
     }
     this.pending.clear();
     this.streamText.clear();
+    this.streamSetupSent.clear();
   }
 
 
@@ -253,39 +333,47 @@ export class ModelProxy {
 
   // ── HTTP handlers ──────────────────────────────────────────────
 
-  /** Resolve model ID → proxyModel + route. Shared by chat completions and responses handlers. */
-  private resolveModelRoute(rawModelId: string): {
-    nodeId: string; modelId: string;
-    proxyModel: (typeof this.config.proxyModels)[number] | undefined;
-    routeNodeId: string;
+  /** Resolve model ID → all reachable candidates, ordered for failover. */
+  private resolveModelCandidates(rawModelId: string): {
+    modelId: string;
+    candidates: FailoverCandidate[];
   } | { error: { status: number; message: string } } {
     const slashIdx = rawModelId.indexOf("/");
-    let nodeId: string;
     let modelId: string;
-    let proxyModel: (typeof this.config.proxyModels)[number] | undefined;
+    let matchingModels: (typeof this.config.proxyModels)[number][];
 
     if (slashIdx > 0) {
-      nodeId = rawModelId.slice(0, slashIdx);
+      const nodeId = rawModelId.slice(0, slashIdx);
       modelId = rawModelId.slice(slashIdx + 1);
-      proxyModel = this.config.proxyModels.find((m) => m.id === modelId && m.nodeId === nodeId);
+      // Explicit node/model — only target that specific node, no failover to others
+      matchingModels = this.config.proxyModels.filter((m) => m.id === modelId && m.nodeId === nodeId);
     } else {
       modelId = rawModelId;
-      proxyModel = this.config.proxyModels.find((m) => m.id === modelId);
-      if (!proxyModel) {
-        return { error: { status: 404, message: `Model "${rawModelId}" not found in proxy models` } };
-      }
-      nodeId = proxyModel.nodeId;
+      matchingModels = this.config.proxyModels.filter((m) => m.id === modelId);
     }
 
-    const route = this.peerManager.router.getRoute(nodeId);
-    debug("proxy", `model raw="${rawModelId}" nodeId=${nodeId} modelId="${modelId}" route=${route?.nodeId ?? "none"}`);
-    if (!route) {
-      return { error: { status: 404, message: `Node "${nodeId}" not found in cluster` } };
+    if (matchingModels.length === 0) {
+      return { error: { status: 404, message: `Model "${rawModelId}" not found in proxy models` } };
     }
-    if (!this.peerManager.canReach(route.nodeId)) {
-      return { error: { status: 502, message: `Cannot reach model node "${route.nodeId}"` } };
+
+    // Build candidates from all reachable nodes
+    const candidates: FailoverCandidate[] = [];
+    const seenNodes = new Set<string>();
+    for (const pm of matchingModels) {
+      if (seenNodes.has(pm.nodeId)) continue;
+      seenNodes.add(pm.nodeId);
+      const route = this.peerManager.router.getRoute(pm.nodeId);
+      if (route && this.peerManager.canReach(route.nodeId)) {
+        candidates.push({ proxyModel: pm, routeNodeId: route.nodeId });
+      }
+    }
+
+    debug("proxy", `model raw="${rawModelId}" modelId="${modelId}" candidates=${candidates.map((c) => c.routeNodeId).join(",") || "none"}`);
+    if (candidates.length === 0) {
+      return { error: { status: 502, message: `No reachable node for model "${rawModelId}"` } };
     }
-    return { nodeId, modelId, proxyModel, routeNodeId: route.nodeId };
+
+    return { modelId, candidates };
   }
 
   private async handleChatCompletion(rawBody: string, _api: string): Promise<ProxyResponse> {
@@ -296,35 +384,41 @@ export class ModelProxy {
       return { status: 400, headers: { "Content-Type": "application/json" }, body: JSON.stringify({ error: "Invalid JSON" }) };
     }
 
-    const resolved = this.resolveModelRoute(body.model);
+    const resolved = this.resolveModelCandidates(body.model);
     if ("error" in resolved) {
       return { status: resolved.error.status, headers: { "Content-Type": "application/json" }, body: JSON.stringify({ error: { message: resolved.error.message } }) };
     }
 
-    const { modelId, proxyModel, routeNodeId } = resolved;
-    const messages = body.messages;
-    debug("proxy", `messages count=${messages?.length ?? 0} roles=${(messages ?? []).map((m: unknown) => (m as Record<string, unknown>)?.role).join(",")}`);
-
-    if (proxyModel?.description) {
-      const first = messages[0] as { role?: string; content?: string } | undefined;
-      if (first?.role === "system" && typeof first.content === "string") {
-        first.content = `[Model: ${proxyModel.description}]\n${first.content}`;
-      } else {
-        messages.unshift({ role: "system", content: `[Model: ${proxyModel.description}]` });
-      }
-    }
+    const { modelId, candidates } = resolved;
+    const first = candidates[0]!;
+    const rest = candidates.slice(1);
+    const baseMessages = body.messages;
+    debug("proxy", `messages count=${baseMessages?.length ?? 0} roles=${(baseMessages ?? []).map((m: unknown) => (m as Record<string, unknown>)?.role).join(",")}`);
 
     const stream = body.stream ?? false;
     const requestId = crypto.randomUUID();
-    const frame: ModelRequest = {
-      type: "model_req", id: requestId, from: this.config.nodeId, to: routeNodeId, timestamp: Date.now(),
-      payload: { model: modelId, provider: proxyModel?.provider, api: proxyModel?.api, messages, temperature: body.temperature, maxTokens: body.max_tokens, stream },
+    const buildFrame = (candidate: FailoverCandidate, id: string): ModelRequest => {
+      // Clone messages so each candidate gets its own description prefix
+      const messages = baseMessages.map((m: unknown) => (m && typeof m === "object" ? { ...(m as object) } : m));
+      if (candidate.proxyModel?.description) {
+        const firstMsg = messages[0] as { role?: string; content?: string } | undefined;
+        if (firstMsg?.role === "system" && typeof firstMsg.content === "string") {
+          firstMsg.content = `[Model: ${candidate.proxyModel.description}]\n${firstMsg.content}`;
+        } else {
+          messages.unshift({ role: "system", content: `[Model: ${candidate.proxyModel.description}]` });
+        }
+      }
+      return {
+        type: "model_req", id, from: this.config.nodeId, to: candidate.routeNodeId, timestamp: Date.now(),
+        payload: { model: modelId, provider: candidate.proxyModel?.provider, api: candidate.proxyModel?.api, messages, temperature: body.temperature, maxTokens: body.max_tokens, stream },
+      };
     };
+    const frame = buildFrame(first, requestId);
 
     if (stream) {
-      return this.handleStreamRequest(requestId, routeNodeId, frame, "chat");
+      return this.handleStreamRequest(requestId, first.routeNodeId, frame, "chat", rest, buildFrame);
     } else {
-      return this.handleNonStreamRequest(requestId, routeNodeId, frame, "chat");
+      return this.handleNonStreamRequest(requestId, first.routeNodeId, frame, "chat", rest, buildFrame);
     }
   }
 
@@ -336,46 +430,52 @@ export class ModelProxy {
       return { status: 400, headers: { "Content-Type": "application/json" }, body: JSON.stringify({ error: "Invalid JSON" }) };
     }
 
-    const resolved = this.resolveModelRoute(body.model);
+    const resolved = this.resolveModelCandidates(body.model);
     if ("error" in resolved) {
       return { status: resolved.error.status, headers: { "Content-Type": "application/json" }, body: JSON.stringify({ error: { message: resolved.error.message } }) };
     }
 
-    const { modelId, proxyModel, routeNodeId } = resolved;
+    const { modelId, candidates } = resolved;
+    const first = candidates[0]!;
+    const rest = candidates.slice(1);
 
-    // Normalize responses API input → simple chat messages for WS transport.
-    // Responses API items use {type: "message", role, content: [{type: "input_text", text}]}
-    // but WS protocol carries simple {role, content} chat messages.
-    const messages = ModelProxy.normalizeResponsesInput(body.input);
+    // Pass original Responses API input items through WS without normalizing.
+    // The remote side will convert formats as needed based on its own API type.
+    // Note: requires all cluster nodes to be on the same plugin version (inputFormat field).
+    const baseItems = Array.isArray(body.input) ? [...body.input] : (typeof body.input === "string" ? [{ role: "user", content: body.input }] : []);
 
     // Prepend instructions as system/developer message
     if (body.instructions) {
-      messages.unshift({ role: "developer", content: body.instructions });
-    }
-
-    if (proxyModel?.description) {
-      const first = messages[0] as { role?: string; content?: string } | undefined;
-      if (first?.role === "system" && typeof first.content === "string") {
-        first.content = `[Model: ${proxyModel.description}]\n${first.content}`;
-      } else if (first?.role === "developer" && typeof first.content === "string") {
-        first.content = `[Model: ${proxyModel.description}]\n${first.content}`;
-      } else {
-        messages.unshift({ role: "system", content: `[Model: ${proxyModel.description}]` });
-      }
+      baseItems.unshift({ role: "developer", content: body.instructions });
     }
 
     const stream = body.stream ?? false;
     const requestId = crypto.randomUUID();
-    debug("proxy", `responses: stream=${stream} messages=${messages.length} input_type=${typeof body.input}${Array.isArray(body.input) ? `[${body.input.length}]` : ""}`);
-    const frame: ModelRequest = {
-      type: "model_req", id: requestId, from: this.config.nodeId, to: routeNodeId, timestamp: Date.now(),
-      payload: { model: modelId, provider: proxyModel?.provider, api: proxyModel?.api, messages, temperature: body.temperature, maxTokens: body.max_output_tokens, stream },
+    debug("proxy", `responses: stream=${stream} messages=${baseItems.length} input_type=${typeof body.input}${Array.isArray(body.input) ? `[${body.input.length}]` : ""}`);
+    const buildFrame = (candidate: FailoverCandidate, id: string): ModelRequest => {
+      // Clone items so each candidate gets its own description prefix
+      const inputItems = baseItems.map((item: unknown) => (item && typeof item === "object" ? { ...(item as object) } : item));
+      if (candidate.proxyModel?.description) {
+        const firstItem = inputItems[0] as { role?: string; content?: string } | undefined;
+        if (firstItem?.role === "system" && typeof firstItem.content === "string") {
+          firstItem.content = `[Model: ${candidate.proxyModel.description}]\n${firstItem.content}`;
+        } else if (firstItem?.role === "developer" && typeof firstItem.content === "string") {
+          firstItem.content = `[Model: ${candidate.proxyModel.description}]\n${firstItem.content}`;
+        } else {
+          inputItems.unshift({ role: "system", content: `[Model: ${candidate.proxyModel.description}]` });
+        }
+      }
+      return {
+        type: "model_req", id, from: this.config.nodeId, to: candidate.routeNodeId, timestamp: Date.now(),
+        payload: { model: modelId, provider: candidate.proxyModel?.provider, api: candidate.proxyModel?.api, messages: inputItems, inputFormat: "responses", temperature: body.temperature, maxTokens: body.max_output_tokens, stream },
+      };
     };
+    const frame = buildFrame(first, requestId);
 
     if (stream) {
-      return this.handleStreamRequest(requestId, routeNodeId, frame, "responses");
+      return this.handleStreamRequest(requestId, first.routeNodeId, frame, "responses", rest, buildFrame);
     } else {
-      return this.handleNonStreamRequest(requestId, routeNodeId, frame, "responses");
+      return this.handleNonStreamRequest(requestId, first.routeNodeId, frame, "responses", rest, buildFrame);
     }
   }
 
@@ -384,57 +484,22 @@ export class ModelProxy {
     targetNodeId: string,
     frame: ModelRequest,
     responseFormat: ResponseFormat,
+    failoverCandidates: FailoverCandidate[] = [],
+    buildFrame?: (candidate: FailoverCandidate, newId: string) => ModelRequest,
   ): ProxyResponse & { body: ReadableStream } {
     const encoder = new TextEncoder();
     const model = frame.payload.model;
 
     const readable = new ReadableStream({
       start: (controller) => {
-        const timer = setTimeout(() => {
-          this.pending.delete(requestId);
-          this.streamText.delete(requestId);
-          this.peerManager.router.markFailed(requestId);
-          try {
-            if (responseFormat === "responses") {
-              controller.enqueue(encoder.encode(`event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", delta: "\n\n[ClawMatrix] Error: model request timed out" })}\n\n`));
-              this.enqueueResponsesStreamDone(controller, encoder, requestId, model);
-            } else {
-              controller.enqueue(encoder.encode(`data: ${JSON.stringify({ id: `chatcmpl-${requestId}`, object: "chat.completion.chunk", choices: [{ index: 0, delta: { content: "\n\n[ClawMatrix] Error: model request timed out" }, finish_reason: "stop" }] })}\n\n`));
-              controller.enqueue(encoder.encode("data: [DONE]\n\n"));
-            }
-            controller.close();
-          } catch { /* controller may already be closed */ }
-        }, MODEL_TIMEOUT);
-
-        this.pending.set(requestId, {
-          resolve: () => {}, reject: () => {},
-          timer, stream: true, responseFormat, model,
-          controller, encoder,
-        });
-
-        // Emit setup events for responses API
-        if (responseFormat === "responses") {
-          this.enqueueResponsesStreamSetup(controller, encoder, requestId, model);
-        }
-
-        const sent = this.peerManager.sendTo(targetNodeId, frame);
-        if (!sent) {
-          this.pending.delete(requestId);
-          clearTimeout(timer);
-          try {
-            if (responseFormat === "responses") {
-              controller.enqueue(encoder.encode(`event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", delta: `[ClawMatrix] Cannot reach model node "${targetNodeId}"` })}\n\n`));
-              this.enqueueResponsesStreamDone(controller, encoder, requestId, model);
-            } else {
-              controller.enqueue(encoder.encode(`data: ${JSON.stringify({ id: `chatcmpl-${requestId}`, object: "chat.completion.chunk", choices: [{ index: 0, delta: { content: `[ClawMatrix] Cannot reach model node "${targetNodeId}"` }, finish_reason: "stop" }] })}\n\n`));
-              controller.enqueue(encoder.encode("data: [DONE]\n\n"));
-            }
-            controller.close();
-          } catch { /* controller may already be closed */ }
-        }
+        this.startStreamAttempt(requestId, targetNodeId, frame, responseFormat, controller, encoder, model, failoverCandidates, buildFrame);
       },
     });
 
+    // Emit setup events for responses API (once, before any attempts)
+    // Note: we enqueue after creating the ReadableStream but the controller
+    // is available synchronously in the `start` callback.
+
     return {
       status: 200,
       headers: { "Content-Type": "text/event-stream", "Cache-Control": "no-cache", "Connection": "keep-alive" },
@@ -442,6 +507,91 @@ export class ModelProxy {
     };
   }
 
+  /** Start (or retry) a stream attempt to a specific node. */
+  private startStreamAttempt(
+    requestId: string,
+    targetNodeId: string,
+    frame: ModelRequest,
+    responseFormat: ResponseFormat,
+    controller: ReadableStreamDefaultController,
+    encoder: TextEncoder,
+    model: string,
+    failoverCandidates: FailoverCandidate[],
+    buildFrame?: (candidate: FailoverCandidate, newId: string) => ModelRequest,
+    /** Stable ID for the entire stream (used for setup events & final close). Defaults to requestId on first attempt. */
+    streamId?: string,
+  ) {
+    const stableId = streamId ?? requestId;
+
+    const timer = setTimeout(() => {
+      this.pending.delete(requestId);
+      this.streamText.delete(requestId);
+      this.peerManager.router.markFailed(requestId);
+      this.tryStreamFailover(stableId, responseFormat, controller, encoder, model, failoverCandidates, buildFrame, `model request to "${targetNodeId}" timed out`);
+    }, MODEL_TIMEOUT);
+
+    this.pending.set(requestId, {
+      resolve: () => {}, reject: () => {},
+      timer, stream: true, responseFormat, model,
+      controller, encoder,
+      hasContent: false,
+      failoverCandidates,
+      buildFrame,
+      stableStreamId: stableId,
+    });
+
+    // Emit setup events for responses API (only once per stream, keyed by stableId)
+    if (responseFormat === "responses" && !this.streamSetupSent.has(stableId)) {
+      this.enqueueResponsesStreamSetup(controller, encoder, stableId, model);
+      this.streamSetupSent.add(stableId);
+    }
+
+    const sent = this.peerManager.sendTo(targetNodeId, frame);
+    if (!sent) {
+      this.pending.delete(requestId);
+      clearTimeout(timer);
+      this.tryStreamFailover(stableId, responseFormat, controller, encoder, model, failoverCandidates, buildFrame, `cannot reach node "${targetNodeId}"`);
+    }
+  }
+
+  /** Track which stream requests have already sent responses API setup events. */
+  private streamSetupSent = new Set<string>();
+
+  /** Attempt failover to next candidate, or close stream with error. */
+  private tryStreamFailover(
+    stableStreamId: string,
+    responseFormat: ResponseFormat,
+    controller: ReadableStreamDefaultController,
+    encoder: TextEncoder,
+    model: string,
+    candidates: FailoverCandidate[],
+    buildFrame?: (candidate: FailoverCandidate, newId: string) => ModelRequest,
+    reason?: string,
+  ) {
+    if (candidates.length > 0 && buildFrame) {
+      const next = candidates[0]!;
+      const remaining = candidates.slice(1);
+      const newId = crypto.randomUUID();
+      const newFrame = buildFrame(next, newId);
+      debug("proxy", `failover: ${reason} → trying ${next.routeNodeId} (${remaining.length} left)`);
+      this.startStreamAttempt(newId, next.routeNodeId, newFrame, responseFormat, controller, encoder, model, remaining, buildFrame, stableStreamId);
+    } else {
+      debug("proxy", `failover exhausted: ${reason}`);
+      try {
+        const errMsg = `\n\n[ClawMatrix] Error: ${reason ?? "all candidates failed"}`;
+        if (responseFormat === "responses") {
+          controller.enqueue(encoder.encode(`event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", delta: errMsg })}\n\n`));
+          this.enqueueResponsesStreamDone(controller, encoder, stableStreamId, model, errMsg);
+        } else {
+          controller.enqueue(encoder.encode(`data: ${JSON.stringify({ id: `chatcmpl-${stableStreamId}`, object: "chat.completion.chunk", choices: [{ index: 0, delta: { content: errMsg }, finish_reason: "stop" }] })}\n\n`));
+          controller.enqueue(encoder.encode("data: [DONE]\n\n"));
+        }
+        controller.close();
+      } catch { /* controller may already be closed */ }
+      this.streamSetupSent.delete(stableStreamId);
+    }
+  }
+
   /** Emit responses API stream setup events (response.created → content_part.added). */
   private enqueueResponsesStreamSetup(controller: ReadableStreamDefaultController, encoder: TextEncoder, id: string, model: string) {
     const respId = `resp_${id}`;
@@ -478,98 +628,148 @@ export class ModelProxy {
     targetNodeId: string,
     frame: ModelRequest,
     responseFormat: ResponseFormat,
+    failoverCandidates: FailoverCandidate[] = [],
+    buildFrame?: (candidate: FailoverCandidate, newId: string) => ModelRequest,
   ): Promise<ProxyResponse & { body: string }> {
-    try {
-      const result = await new Promise<ModelResponse["payload"]>(
-        (resolve, reject) => {
-          const timer = setTimeout(() => {
-            this.pending.delete(requestId);
-            this.peerManager.router.markFailed(requestId);
-            reject(new Error("Model request timed out"));
-          }, MODEL_TIMEOUT);
-
-          this.pending.set(requestId, {
-            resolve: resolve as (v: unknown) => void,
-            reject, timer, stream: false, responseFormat,
-          });
-
-          const sent = this.peerManager.sendTo(targetNodeId, frame);
-          if (!sent) {
-            this.pending.delete(requestId);
-            clearTimeout(timer);
-            reject(new Error("Cannot reach model node"));
+    let currentId = requestId;
+    let currentTarget = targetNodeId;
+    let currentFrame = frame;
+    let remaining = failoverCandidates;
+    const maxAttempts = failoverCandidates.length + 1;
+
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      try {
+        const result = await this.sendNonStreamAndWait(currentId, currentTarget, currentFrame, responseFormat);
+
+        if (!result.success) {
+          // Upstream error — try failover if available
+          if (remaining.length > 0 && buildFrame) {
+            const next = remaining[0]!;
+            debug("proxy", `failover: remote error "${result.error}" → trying ${next.routeNodeId} (${remaining.length - 1} left)`);
+            currentId = crypto.randomUUID();
+            currentFrame = buildFrame(next, currentId);
+            currentTarget = next.routeNodeId;
+            remaining = remaining.slice(1);
+            continue;
           }
-        },
-      );
+          return {
+            status: 502,
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ error: { message: result.error } }),
+          };
+        }
 
-      if (!result.success) {
+        return this.formatNonStreamResult(result, currentId, currentFrame, responseFormat);
+      } catch (err) {
+        // Timeout or send failure — try failover
+        if (remaining.length > 0 && buildFrame) {
+          const next = remaining[0]!;
+          debug("proxy", `failover: ${err instanceof Error ? err.message : String(err)} → trying ${next.routeNodeId} (${remaining.length - 1} left)`);
+          currentId = crypto.randomUUID();
+          currentFrame = buildFrame(next, currentId);
+          currentTarget = next.routeNodeId;
+          remaining = remaining.slice(1);
+          continue;
+        }
         return {
           status: 502,
           headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({ error: { message: result.error } }),
+          body: JSON.stringify({ error: { message: err instanceof Error ? err.message : String(err) } }),
         };
       }
+    }
+    // Safety: should not reach here, but return error if loop exhausts
+    return {
+      status: 502,
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ error: { message: "All failover candidates exhausted" } }),
+    };
+  }
 
-      if (responseFormat === "responses") {
-        const msgId = `msg_${requestId}`;
-        const usageObj = result.usage
-          ? { input_tokens: result.usage.inputTokens, output_tokens: result.usage.outputTokens, total_tokens: result.usage.inputTokens + result.usage.outputTokens }
-          : { input_tokens: 0, output_tokens: 0, total_tokens: 0 };
-        // If upstream sent full output array (responses API), use it directly
-        const output = Array.isArray(result.message)
-          ? result.message
-          : [{
-              type: "message", id: msgId, role: "assistant",
-              content: [{ type: "output_text", text: result.content ?? "" }],
-              status: "completed",
-            }];
-        return {
-          status: 200,
-          headers: { "Content-Type": "application/json" },
-          body: JSON.stringify({
-            id: `resp_${requestId}`,
-            object: "response",
-            created_at: Math.floor(Date.now() / 1000),
-            status: "completed",
-            model: frame.payload.model,
-            output,
-            usage: usageObj,
-          }),
-        };
-      }
+  private sendNonStreamAndWait(
+    requestId: string,
+    targetNodeId: string,
+    frame: ModelRequest,
+    responseFormat: ResponseFormat,
+  ): Promise<ModelResponse["payload"]> {
+    return new Promise<ModelResponse["payload"]>((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(requestId);
+        this.peerManager.router.markFailed(requestId);
+        reject(new Error(`Model request to "${targetNodeId}" timed out`));
+      }, MODEL_TIMEOUT);
+
+      this.pending.set(requestId, {
+        resolve: resolve as (v: unknown) => void,
+        reject, timer, stream: false, responseFormat,
+      });
 
-      // Chat completions format — use full message object when available (has tool_calls etc.)
-      const msg = result.message as Record<string, unknown> | undefined;
-      const message = msg
-        ? { role: "assistant", ...msg }
-        : { role: "assistant", content: result.content };
-      const finishReason = msg?.tool_calls ? "tool_calls" : "stop";
+      const sent = this.peerManager.sendTo(targetNodeId, frame);
+      if (!sent) {
+        this.pending.delete(requestId);
+        clearTimeout(timer);
+        reject(new Error(`Cannot reach model node "${targetNodeId}"`));
+      }
+    });
+  }
 
+  private formatNonStreamResult(
+    result: ModelResponse["payload"],
+    requestId: string,
+    frame: ModelRequest,
+    responseFormat: ResponseFormat,
+  ): ProxyResponse & { body: string } {
+    if (responseFormat === "responses") {
+      const msgId = `msg_${requestId}`;
+      const usageObj = result.usage
+        ? { input_tokens: result.usage.inputTokens, output_tokens: result.usage.outputTokens, total_tokens: result.usage.inputTokens + result.usage.outputTokens }
+        : { input_tokens: 0, output_tokens: 0, total_tokens: 0 };
+      const output = Array.isArray(result.message)
+        ? result.message
+        : [{
+            type: "message", id: msgId, role: "assistant",
+            content: [{ type: "output_text", text: result.content ?? "" }],
+            status: "completed",
+          }];
       return {
         status: 200,
         headers: { "Content-Type": "application/json" },
         body: JSON.stringify({
-          id: `chatcmpl-${requestId}`,
-          object: "chat.completion",
-          created: Math.floor(Date.now() / 1000),
+          id: `resp_${requestId}`,
+          object: "response",
+          created_at: Math.floor(Date.now() / 1000),
+          status: "completed",
           model: frame.payload.model,
-          choices: [{
-            index: 0,
-            message,
-            finish_reason: finishReason,
-          }],
-          usage: result.usage
-            ? { prompt_tokens: result.usage.inputTokens, completion_tokens: result.usage.outputTokens, total_tokens: result.usage.inputTokens + result.usage.outputTokens }
-            : undefined,
+          output,
+          usage: usageObj,
         }),
       };
-    } catch (err) {
-      return {
-        status: 502,
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ error: { message: err instanceof Error ? err.message : String(err) } }),
-      };
     }
+
+    const msg = result.message as Record<string, unknown> | undefined;
+    const message = msg
+      ? { role: "assistant", ...msg }
+      : { role: "assistant", content: result.content };
+    const finishReason = msg?.tool_calls ? "tool_calls" : "stop";
+
+    return {
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        id: `chatcmpl-${requestId}`,
+        object: "chat.completion",
+        created: Math.floor(Date.now() / 1000),
+        model: frame.payload.model,
+        choices: [{
+          index: 0,
+          message,
+          finish_reason: finishReason,
+        }],
+        usage: result.usage
+          ? { prompt_tokens: result.usage.inputTokens, completion_tokens: result.usage.outputTokens, total_tokens: result.usage.inputTokens + result.usage.outputTokens }
+          : undefined,
+      }),
+    };
   }
 
   private handleListModels(): ProxyResponse & { body: string } {
@@ -622,17 +822,29 @@ export class ModelProxy {
       if (!frame.payload.success && pending.controller && pending.encoder) {
         clearTimeout(pending.timer);
         this.pending.delete(frame.id);
-        try {
-          const errMsg = `[ClawMatrix] Remote error: ${frame.payload.error}`;
-          if (pending.responseFormat === "responses") {
-            pending.controller.enqueue(pending.encoder.encode(`event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", delta: errMsg })}\n\n`));
-            this.enqueueResponsesStreamDone(pending.controller, pending.encoder, frame.id, pending.model ?? "", errMsg);
-          } else {
-            pending.controller.enqueue(pending.encoder.encode(`data: ${JSON.stringify({ id: `chatcmpl-${frame.id}`, object: "chat.completion.chunk", choices: [{ index: 0, delta: { content: errMsg }, finish_reason: "stop" }] })}\n\n`));
-            pending.controller.enqueue(pending.encoder.encode("data: [DONE]\n\n"));
-          }
-          pending.controller.close();
-        } catch { /* controller may already be closed */ }
+        this.streamText.delete(frame.id);
+        // Try failover if no content has been sent yet
+        if (!pending.hasContent && pending.failoverCandidates?.length && pending.buildFrame) {
+          this.tryStreamFailover(
+            pending.stableStreamId ?? frame.id, pending.responseFormat, pending.controller, pending.encoder,
+            pending.model ?? "", pending.failoverCandidates, pending.buildFrame,
+            `remote error: ${frame.payload.error}`,
+          );
+        } else {
+          const stableId = pending.stableStreamId ?? frame.id;
+          try {
+            const errMsg = `[ClawMatrix] Remote error: ${frame.payload.error}`;
+            if (pending.responseFormat === "responses") {
+              pending.controller.enqueue(pending.encoder.encode(`event: response.output_text.delta\ndata: ${JSON.stringify({ type: "response.output_text.delta", delta: errMsg })}\n\n`));
+              this.enqueueResponsesStreamDone(pending.controller, pending.encoder, stableId, pending.model ?? "", errMsg);
+            } else {
+              pending.controller.enqueue(pending.encoder.encode(`data: ${JSON.stringify({ id: `chatcmpl-${stableId}`, object: "chat.completion.chunk", choices: [{ index: 0, delta: { content: errMsg }, finish_reason: "stop" }] })}\n\n`));
+              pending.controller.enqueue(pending.encoder.encode("data: [DONE]\n\n"));
+            }
+            pending.controller.close();
+          } catch { /* controller may already be closed */ }
+          this.streamSetupSent.delete(stableId);
+        }
       }
       return;
     }
@@ -666,8 +878,9 @@ export class ModelProxy {
 
   private handleModelStreamChat(frame: ModelStreamChunk, pending: PendingModelReq) {
     if (frame.payload.done) {
+      const stableId = pending.stableStreamId ?? frame.id;
       const finalChunk: Record<string, unknown> = {
-        id: `chatcmpl-${frame.id}`, object: "chat.completion.chunk",
+        id: `chatcmpl-${stableId}`, object: "chat.completion.chunk",
         choices: [{ index: 0, delta: {}, finish_reason: "stop" }],
       };
       if (frame.payload.usage) {
@@ -678,27 +891,34 @@ export class ModelProxy {
       pending.controller!.close();
       clearTimeout(pending.timer);
       this.pending.delete(frame.id);
+      this.streamSetupSent.delete(stableId);
     } else {
       // Use full deltaObj when available (carries tool_calls etc.), otherwise simple text delta
       const delta = frame.payload.deltaObj ?? { content: frame.payload.delta };
-      const chunk = { id: `chatcmpl-${frame.id}`, object: "chat.completion.chunk", choices: [{ index: 0, delta, finish_reason: null }] };
+      const chunkStableId = pending.stableStreamId ?? frame.id;
+      const chunk = { id: `chatcmpl-${chunkStableId}`, object: "chat.completion.chunk", choices: [{ index: 0, delta, finish_reason: null }] };
       pending.controller!.enqueue(pending.encoder!.encode(`data: ${JSON.stringify(chunk)}\n\n`));
+      pending.hasContent = true;
     }
   }
 
   private handleModelStreamResponses(frame: ModelStreamChunk, pending: PendingModelReq) {
     if (frame.payload.done) {
+      const stableId = pending.stableStreamId ?? frame.id;
       const fullText = this.streamText.get(frame.id) ?? "";
       this.streamText.delete(frame.id);
-      this.enqueueResponsesStreamDone(pending.controller!, pending.encoder!, frame.id, pending.model ?? "", fullText, frame.payload.usage);
+      this.enqueueResponsesStreamDone(pending.controller!, pending.encoder!, stableId, pending.model ?? "", fullText, frame.payload.usage);
       pending.controller!.close();
       clearTimeout(pending.timer);
       this.pending.delete(frame.id);
+      this.streamSetupSent.delete(stableId);
     } else {
       // Accumulate text for done event
-      this.streamText.set(frame.id, (this.streamText.get(frame.id) ?? "") + frame.payload.delta);
-      const evt = { type: "response.output_text.delta", item_id: `msg_${frame.id}`, output_index: 0, content_index: 0, delta: frame.payload.delta };
+      this.streamText.set(frame.id, (this.streamText.get(frame.id) ?? "") + (frame.payload.delta ?? ""));
+      const respStableId = pending.stableStreamId ?? frame.id;
+      const evt = { type: "response.output_text.delta", item_id: `msg_${respStableId}`, output_index: 0, content_index: 0, delta: frame.payload.delta };
       pending.controller!.enqueue(pending.encoder!.encode(`event: response.output_text.delta\ndata: ${JSON.stringify(evt)}\n\n`));
+      pending.hasContent = true;
     }
   }
 
@@ -762,22 +982,34 @@ export class ModelProxy {
       }
 
       const modelField = endpoint.direct ? model.id : `${model.provider}/${model.id}`;
-      const requestBody = isResponsesApi
-        ? {
-            model: modelField,
-            input: payload.messages,
-            stream: payload.stream,
-            temperature: payload.temperature,
-            max_output_tokens: payload.maxTokens,
-          }
-        : {
-            model: modelField,
-            messages: payload.messages,
-            temperature: payload.temperature,
-            max_tokens: payload.maxTokens,
-            stream: payload.stream,
-            ...(payload.stream ? { stream_options: { include_usage: true } } : {}),
-          };
+      const srcFormat = payload.inputFormat ?? "chat";
+
+      // Convert messages between formats if source and target API differ
+      let requestBody: Record<string, unknown>;
+      if (isResponsesApi) {
+        const input = srcFormat === "responses"
+          ? payload.messages                                           // already Responses API format, pass through
+          : ModelProxy.chatToResponsesInput(payload.messages);         // chat → responses
+        requestBody = {
+          model: modelField,
+          input,
+          stream: payload.stream,
+          temperature: payload.temperature,
+          max_output_tokens: payload.maxTokens,
+        };
+      } else {
+        const messages = srcFormat === "chat"
+          ? payload.messages                                           // already chat format, pass through
+          : ModelProxy.normalizeResponsesInput(payload.messages);      // responses → chat
+        requestBody = {
+          model: modelField,
+          messages,
+          temperature: payload.temperature,
+          max_tokens: payload.maxTokens,
+          stream: payload.stream,
+          ...(payload.stream ? { stream_options: { include_usage: true } } : {}),
+        };
+      }
 
       const response = await fetch(url, {
         method: "POST",