@zhouzhengchang/token-party 0.0.13 → 0.0.14

package/dist/proxy/forwarder.js

@@ -1,14 +1,18 @@
 import { streamSSE } from "hono/streaming";
 import { getModelId, getModelPricing } from "../types/config.js";
-import { getConfig } from "../config.js";
 import { nanoid } from "nanoid";
 import { writeLog, headersToRecord } from "../store/log-writer.js";
 import { recordRequest } from "../metrics/collector.js";
 import { extractTags } from "../tags/registry.js";
 import { createGunzip, createInflate, createBrotliDecompress, createZstdDecompress } from "node:zlib";
 import { Readable, Transform } from "node:stream";
-import { request as httpsRequest } from "node:https";
-import { request as httpRequest } from "node:http";
+import { Agent as HttpsAgent, request as httpsRequest } from "node:https";
+import { Agent as HttpAgent, request as httpRequest } from "node:http";
+// Shared keepAlive agents for connection pooling.
+// Without these, every outgoing request opens a new TCP connection, causing
+// TIME_WAIT accumulation and ephemeral port exhaustion under sustained load.
+const httpAgent = new HttpAgent({ keepAlive: true, maxFreeSockets: 20, keepAliveMsecs: 30_000 });
+const httpsAgent = new HttpsAgent({ keepAlive: true, maxFreeSockets: 20, keepAliveMsecs: 30_000 });
 const roundRobinCounters = new Map();
 function selectApiKey(provider) {
     const keys = Array.isArray(provider.apiKey) ? provider.apiKey : [provider.apiKey];
@@ -24,37 +28,18 @@ function maskApiKey(key) {
         return "****";
     return key.slice(0, 4) + "****" + key.slice(-4);
 }
-export async function forwardRequest(c, provider, targetPath, transformedBody, entryProtocol, pricing, _routeTrace) {
+function isRetryableStatus(status) {
+    return status === 429 || status >= 500;
+}
+export async function forwardRequest(c, candidateProviders, targetPath, transformedBody, entryProtocol, _routeTrace) {
     const routeTrace = _routeTrace ?? [];
     const requestId = nanoid();
     const startTime = Date.now();
     const body = transformedBody ?? (await c.req.json());
     const isStreaming = body?.stream === true;
     const model = body?.model ?? "unknown";
-    const entry = entryProtocol ?? provider.type;
-    const needsStreamConversion = isStreaming && entry !== provider.type;
+    const entry = entryProtocol ?? candidateProviders[0].type;
     const isResponsesApi = !!body?.input && !body?.messages;
-    // Request usage in streaming for OpenAI chat completions (not responses API)
-    if (isStreaming && provider.type === "openai" && !isResponsesApi && !body.stream_options) {
-        body.stream_options = { include_usage: true };
-    }
-    const targetUrl = `${provider.baseUrl}${targetPath}`;
-    const { key: selectedKey, index: apiKeyIndex } = selectApiKey(provider);
-    const upstreamHeaders = {};
-    const skipHeaders = new Set(["host", "connection", "content-length"]);
-    c.req.raw.headers.forEach((value, key) => {
-        if (!skipHeaders.has(key.toLowerCase())) {
-            upstreamHeaders[key] = value;
-        }
-    });
-    if (provider.type === "openai") {
-        upstreamHeaders["authorization"] = `Bearer ${selectedKey}`;
-    }
-    else if (provider.type === "anthropic") {
-        delete upstreamHeaders["authorization"];
-        upstreamHeaders["x-api-key"] = selectedKey;
-        upstreamHeaders["anthropic-version"] ??= "2023-06-01";
-    }
     const reqHeaders = {};
     c.req.raw.headers.forEach((value, key) => {
         reqHeaders[key] = value;
@@ -62,148 +47,251 @@ export async function forwardRequest(c, provider, targetPath, transformedBody, e
     const extractedTags = extractTags({ headers: c.req.raw.headers, path: c.req.path, body, model });
     const agent = extractedTags.agent ?? "";
     const customTags = extractedTags.tags ?? "";
+    c.set("recorded", true);
+    const token = c.get("authToken");
     const logFile = writeLog(requestId, {
         type: "request",
         timestamp: startTime,
-        headers: { ...reqHeaders, "x-target-url": targetUrl, "x-entry-protocol": entry, "x-provider-type": provider.type, "x-api-key-index": String(apiKeyIndex), "x-api-key-used": maskApiKey(selectedKey) },
+        headers: reqHeaders,
         body,
     });
-    c.set("recorded", true);
-    const token = c.get("authToken");
+    // Loop through candidate providers (ordered by priority then price)
+    for (let i = 0; i < candidateProviders.length; i++) {
+        const provider = candidateProviders[i];
+        const providerPricing = getModelPricing(provider.models.find((m) => getModelId(m) === model));
+        const { key: selectedKey, index: apiKeyIndex } = selectApiKey(provider);
+        const targetUrl = `${provider.baseUrl}${targetPath}`;
+        const upstreamHeaders = {};
+        const skipHeaders = new Set(["host", "connection", "content-length"]);
+        c.req.raw.headers.forEach((value, key) => {
+            if (!skipHeaders.has(key.toLowerCase())) {
+                upstreamHeaders[key] = value;
+            }
+        });
+        if (provider.type === "openai") {
+            upstreamHeaders["authorization"] = `Bearer ${selectedKey}`;
+        }
+        else if (provider.type === "anthropic") {
+            delete upstreamHeaders["authorization"];
+            upstreamHeaders["x-api-key"] = selectedKey;
+            upstreamHeaders["anthropic-version"] ??= "2023-06-01";
+        }
+        // Request usage in streaming for OpenAI chat completions (not responses API)
+        const attemptBody = isStreaming && provider.type === "openai" && !isResponsesApi && !body.stream_options
+            ? { ...body, stream_options: { include_usage: true } }
+            : body;
+        const attemptResult = await attemptProvider({
+            provider,
+            targetUrl,
+            upstreamHeaders,
+            attemptBody,
+            isStreaming,
+            needsStreamConversion: isStreaming && entry !== provider.type,
+            entry,
+            c,
+            requestId,
+            startTime,
+            token,
+            logFile,
+            apiKeyIndex,
+            providerPricing,
+            agent,
+            customTags,
+            routeTrace,
+            model,
+        });
+        if (attemptResult.kind === "done") {
+            return attemptResult.response;
+        }
+        // Retryable error - log this attempt and try next candidate
+        const latencyMs = Date.now() - startTime;
+        const reason = attemptResult.status === 429 ? "rate_limited"
+            : attemptResult.error ? "network_error"
+                : `http_${attemptResult.status}`;
+        routeTrace.push({ provider: provider.id, status: attemptResult.status, latencyMs, reason });
+        recordRequest({
+            id: requestId,
+            tokenId: token.key,
+            providerId: provider.id,
+            model,
+            inputTokens: 0,
+            outputTokens: 0,
+            cacheReadTokens: 0,
+            cacheWriteTokens: 0,
+            latencyMs,
+            status: attemptResult.status,
+            logFile,
+            error: attemptResult.error,
+            apiKeyIndex,
+            pricing: providerPricing,
+            currency: provider.currency,
+            agent,
+            customTags,
+            routeTrace,
+        });
+        if (i < candidateProviders.length - 1) {
+            console.log(`[tokenparty] Falling back from ${provider.id} to ${candidateProviders[i + 1].id} for model ${model} (${reason})`);
+        }
+    }
+    // All candidates exhausted - return last retryable error as 502
+    return c.json({ error: "All provider candidates failed" }, 502);
+}
+async function attemptProvider(params) {
+    const { provider, targetUrl, upstreamHeaders, attemptBody, isStreaming, needsStreamConversion, entry, c, requestId, startTime, token, logFile, apiKeyIndex, providerPricing, agent, customTags, routeTrace, model, } = params;
     try {
-        // Same protocol streaming: use http.request for raw passthrough (no auto-decompression)
+        // Same protocol streaming: use http.request for raw passthrough
         if (isStreaming && !needsStreamConversion) {
+            const streamResult = await rawStreamPassthrough({
+                c, targetUrl, upstreamHeaders, body: attemptBody, requestId, provider,
+                model, token, startTime, logFile, apiKeyIndex, pricing: providerPricing,
+                agent, customTags, routeTrace,
+            });
+            if (streamResult.kind === "retryable") {
+                return streamResult;
+            }
             routeTrace.push({ provider: provider.id, status: 200, latencyMs: 0 });
-            return await rawStreamPassthrough(c, targetUrl, upstreamHeaders, body, requestId, provider, model, token, startTime, logFile, apiKeyIndex, pricing, agent, customTags, routeTrace);
+            return { kind: "done", response: streamResult.response };
         }
+        // Fetch path: non-streaming + cross-protocol streaming
         const response = await fetch(targetUrl, {
             method: "POST",
             headers: upstreamHeaders,
-            body: JSON.stringify(body),
+            body: JSON.stringify(attemptBody),
         });
+        // Check if retryable BEFORE reading/piping body
+        if (isRetryableStatus(response.status)) {
+            // Drain response to free connection
+            await response.body?.cancel();
+            return { kind: "retryable", status: response.status };
+        }
         const respHeaders = headersToRecord(response.headers);
         const latencyMs = Date.now() - startTime;
         if (isStreaming && response.ok) {
-            // Protocol conversion: decompress, parse, convert, re-emit
+            // Protocol conversion streaming
             c.header("Content-Type", "text/event-stream");
             c.header("Cache-Control", "no-cache");
             c.header("Connection", "keep-alive");
-            return streamSSE(c, async (s) => {
-                const reader = decompressResponse(response).getReader();
-                const decoder = new TextDecoder();
-                let buffer = "";
-                let fullContent = "";
-                let rawEvents = [];
-                let usage;
-                let chunkId = `chatcmpl-${requestId}`;
-                const o2aConverter = new OpenaiToAnthropicStreamConverter();
-                try {
-                    while (true) {
-                        const { done, value } = await reader.read();
-                        if (done)
-                            break;
-                        buffer += decoder.decode(value, { stream: true });
-                        const lines = buffer.split("\n");
-                        buffer = lines.pop() ?? "";
-                        for (const line of lines) {
-                            if (!line.startsWith("data: "))
-                                continue;
-                            const data = line.slice(6).trim();
-                            if (data === "[DONE]") {
-                                await s.writeSSE({ data: "[DONE]" });
-                                continue;
-                            }
-                            try {
-                                const parsed = JSON.parse(data);
-                                rawEvents.push(parsed);
-                                if (provider.type === "anthropic" && entry === "openai") {
-                                    const converted = convertAnthropicChunkToOpenai(parsed, model, chunkId);
-                                    if (converted) {
-                                        if (converted.content)
-                                            fullContent += converted.content;
-                                        await s.writeSSE({ data: JSON.stringify(converted.chunk) });
-                                    }
-                                    if (parsed.type === "message_start" && parsed.message?.usage) {
-                                        usage = { ...(usage ?? { input_tokens: 0, output_tokens: 0 }), input_tokens: parsed.message.usage.input_tokens ?? 0, cache_read_tokens: parsed.message.usage.cache_read_input_tokens ?? 0, cache_write_tokens: parsed.message.usage.cache_creation_input_tokens ?? 0 };
-                                    }
-                                    if (parsed.type === "message_delta" && parsed.usage) {
-                                        usage = { ...(usage ?? { input_tokens: 0, output_tokens: 0 }), output_tokens: parsed.usage.output_tokens ?? 0 };
-                                    }
+            return {
+                kind: "done",
+                response: streamSSE(c, async (s) => {
+                    const reader = decompressResponse(response).getReader();
+                    const decoder = new TextDecoder();
+                    let buffer = "";
+                    let fullContent = "";
+                    let rawEvents = [];
+                    let usage;
+                    let chunkId = `chatcmpl-${requestId}`;
+                    const o2aConverter = new OpenaiToAnthropicStreamConverter();
+                    try {
+                        while (true) {
+                            const { done, value } = await reader.read();
+                            if (done)
+                                break;
+                            buffer += decoder.decode(value, { stream: true });
+                            const lines = buffer.split("\n");
+                            buffer = lines.pop() ?? "";
+                            for (const line of lines) {
+                                if (!line.startsWith("data: "))
+                                    continue;
+                                const data = line.slice(6).trim();
+                                if (data === "[DONE]") {
+                                    await s.writeSSE({ data: "[DONE]" });
+                                    continue;
                                 }
-                                else if (provider.type === "openai" && entry === "anthropic") {
-                                    const converted = o2aConverter.convert(parsed, model);
-                                    if (converted) {
-                                        for (const event of converted.events) {
-                                            await s.writeSSE({ event: event.type, data: JSON.stringify(event.data) });
+                                try {
+                                    const parsed = JSON.parse(data);
+                                    rawEvents.push(parsed);
+                                    if (provider.type === "anthropic" && entry === "openai") {
+                                        const converted = convertAnthropicChunkToOpenai(parsed, model, chunkId);
+                                        if (converted) {
+                                            if (converted.content)
+                                                fullContent += converted.content;
+                                            await s.writeSSE({ data: JSON.stringify(converted.chunk) });
+                                        }
+                                        if (parsed.type === "message_start" && parsed.message?.usage) {
+                                            usage = { ...(usage ?? { input_tokens: 0, output_tokens: 0 }), input_tokens: parsed.message.usage.input_tokens ?? 0, cache_read_tokens: parsed.message.usage.cache_read_input_tokens ?? 0, cache_write_tokens: parsed.message.usage.cache_creation_input_tokens ?? 0 };
+                                        }
+                                        if (parsed.type === "message_delta" && parsed.usage) {
+                                            usage = { ...(usage ?? { input_tokens: 0, output_tokens: 0 }), output_tokens: parsed.usage.output_tokens ?? 0 };
                                         }
-                                        if (converted.content)
-                                            fullContent += converted.content;
                                     }
-                                    if (parsed.usage) {
-                                        usage = { input_tokens: parsed.usage.prompt_tokens ?? 0, output_tokens: parsed.usage.completion_tokens ?? 0, cache_read_tokens: parsed.usage.prompt_tokens_details?.cached_tokens ?? 0, cache_write_tokens: 0 };
+                                    else if (provider.type === "openai" && entry === "anthropic") {
+                                        const converted = o2aConverter.convert(parsed, model);
+                                        if (converted) {
+                                            for (const event of converted.events) {
+                                                await s.writeSSE({ event: event.type, data: JSON.stringify(event.data) });
+                                            }
+                                            if (converted.content)
+                                                fullContent += converted.content;
+                                        }
+                                        if (parsed.usage) {
+                                            usage = { input_tokens: parsed.usage.prompt_tokens ?? 0, output_tokens: parsed.usage.completion_tokens ?? 0, cache_read_tokens: parsed.usage.prompt_tokens_details?.cached_tokens ?? 0, cache_write_tokens: 0 };
+                                        }
                                     }
                                 }
+                                catch { }
                             }
-                            catch { }
                         }
                     }
-                }
-                finally {
-                    if (!usage) {
-                        for (let i = rawEvents.length - 1; i >= 0; i--) {
-                            const evt = rawEvents[i];
-                            if (evt.type === "response.completed" && evt.response?.usage) {
-                                usage = {
-                                    input_tokens: evt.response.usage.input_tokens ?? 0,
-                                    output_tokens: evt.response.usage.output_tokens ?? 0,
-                                    cache_read_tokens: evt.response.usage.cache_read_input_tokens ?? 0,
-                                    cache_write_tokens: evt.response.usage.cache_creation_input_tokens ?? 0,
-                                };
-                                break;
-                            }
-                            if (evt.usage && typeof evt.usage === "object" && (evt.usage.prompt_tokens || evt.usage.completion_tokens || evt.usage.input_tokens || evt.usage.output_tokens || evt.usage.total_tokens)) {
-                                usage = {
-                                    input_tokens: evt.usage.prompt_tokens ?? evt.usage.input_tokens ?? 0,
-                                    output_tokens: evt.usage.completion_tokens ?? evt.usage.output_tokens ?? 0,
-                                    cache_read_tokens: evt.usage.prompt_tokens_details?.cached_tokens ?? evt.usage.cache_read_input_tokens ?? 0,
-                                    cache_write_tokens: evt.usage.cache_creation_input_tokens ?? 0,
-                                };
-                                break;
+                    finally {
+                        if (!usage) {
+                            for (let i = rawEvents.length - 1; i >= 0; i--) {
+                                const evt = rawEvents[i];
+                                if (evt.type === "response.completed" && evt.response?.usage) {
+                                    usage = {
+                                        input_tokens: evt.response.usage.input_tokens ?? 0,
+                                        output_tokens: evt.response.usage.output_tokens ?? 0,
+                                        cache_read_tokens: evt.response.usage.cache_read_input_tokens ?? 0,
+                                        cache_write_tokens: evt.response.usage.cache_creation_input_tokens ?? 0,
+                                    };
+                                    break;
+                                }
+                                if (evt.usage && typeof evt.usage === "object" && (evt.usage.prompt_tokens || evt.usage.completion_tokens || evt.usage.input_tokens || evt.usage.output_tokens || evt.usage.total_tokens)) {
+                                    usage = {
+                                        input_tokens: evt.usage.prompt_tokens ?? evt.usage.input_tokens ?? 0,
+                                        output_tokens: evt.usage.completion_tokens ?? evt.usage.output_tokens ?? 0,
+                                        cache_read_tokens: evt.usage.prompt_tokens_details?.cached_tokens ?? evt.usage.cache_read_input_tokens ?? 0,
+                                        cache_write_tokens: evt.usage.cache_creation_input_tokens ?? 0,
+                                    };
+                                    break;
+                                }
                             }
                         }
+                        writeLog(requestId, {
+                            type: "response",
+                            timestamp: Date.now(),
+                            headers: respHeaders,
+                            streaming: true,
+                            streamContent: fullContent,
+                            body: rawEvents,
+                            usage,
+                            status: response.status,
+                        });
+                        routeTrace.push({ provider: provider.id, status: response.status, latencyMs: Date.now() - startTime });
+                        recordRequest({
+                            id: requestId,
+                            tokenId: token.key,
+                            providerId: provider.id,
+                            model,
+                            inputTokens: usage?.input_tokens ?? 0,
+                            outputTokens: usage?.output_tokens ?? 0,
+                            cacheReadTokens: usage?.cache_read_tokens ?? 0,
+                            cacheWriteTokens: usage?.cache_write_tokens ?? 0,
+                            latencyMs: Date.now() - startTime,
+                            status: response.status,
+                            logFile,
+                            apiKeyIndex,
+                            pricing: providerPricing,
+                            currency: provider.currency,
+                            agent,
+                            customTags,
+                            routeTrace,
+                        });
                     }
-                    writeLog(requestId, {
-                        type: "response",
-                        timestamp: Date.now(),
-                        headers: respHeaders,
-                        streaming: true,
-                        streamContent: fullContent,
-                        body: rawEvents,
-                        usage,
-                    });
-                    routeTrace.push({ provider: provider.id, status: response.status, latencyMs: Date.now() - startTime });
-                    recordRequest({
-                        id: requestId,
-                        tokenId: token.key,
-                        providerId: provider.id,
-                        model,
-                        inputTokens: usage?.input_tokens ?? 0,
-                        outputTokens: usage?.output_tokens ?? 0,
-                        cacheReadTokens: usage?.cache_read_tokens ?? 0,
-                        cacheWriteTokens: usage?.cache_write_tokens ?? 0,
-                        latencyMs: Date.now() - startTime,
-                        status: response.status,
-                        logFile,
-                        apiKeyIndex,
-                        pricing,
-                        currency: provider.currency,
-                        agent,
-                        customTags,
-                        routeTrace,
-                    });
-                }
-            });
+                }),
+            };
         }
+        // Non-streaming response
         const responseBody = await decompressJson(response);
         const usage = extractUsage(responseBody, provider.type);
         writeLog(requestId, {
@@ -212,33 +300,8 @@ export async function forwardRequest(c, provider, targetPath, transformedBody, e
             headers: respHeaders,
             body: responseBody,
             usage,
+            status: response.status,
         });
-        if ((response.status === 429 || response.status >= 500) && provider.fallback) {
-            const reason = response.status === 429 ? "rate_limited" : `http_${response.status}`;
-            routeTrace.push({ provider: provider.id, status: response.status, latencyMs, reason });
-            recordRequest({
-                id: requestId,
-                tokenId: token.key,
-                providerId: provider.id,
-                model,
-                inputTokens: usage?.input_tokens ?? 0,
-                outputTokens: usage?.output_tokens ?? 0,
-                cacheReadTokens: usage?.cache_read_tokens ?? 0,
-                cacheWriteTokens: usage?.cache_write_tokens ?? 0,
-                latencyMs,
-                status: response.status,
-                logFile,
-                apiKeyIndex,
-                pricing,
-                currency: provider.currency,
-                agent,
-                customTags,
-                routeTrace,
-            });
-            const fallbackResult = tryFallback(c, provider, model, targetPath, body, entryProtocol, routeTrace);
-            if (fallbackResult)
-                return fallbackResult;
-        }
         routeTrace.push({ provider: provider.id, status: response.status, latencyMs });
         recordRequest({
             id: requestId,
@@ -253,13 +316,13 @@ export async function forwardRequest(c, provider, targetPath, transformedBody, e
             status: response.status,
             logFile,
             apiKeyIndex,
-            pricing,
+            pricing: providerPricing,
             currency: provider.currency,
             agent,
             customTags,
             routeTrace,
         });
-        return c.json(responseBody, response.status);
+        return { kind: "done", response: c.json(responseBody, response.status) };
     }
     catch (error) {
         const latencyMs = Date.now() - startTime;
@@ -268,83 +331,32 @@ export async function forwardRequest(c, provider, targetPath, transformedBody, e
             timestamp: Date.now(),
             error: error.message,
         });
-        routeTrace.push({ provider: provider.id, status: null, latencyMs, reason: "network_error" });
-        if (provider.fallback) {
-            recordRequest({
-                id: requestId,
-                tokenId: token.key,
-                providerId: provider.id,
-                model,
-                inputTokens: 0,
-                outputTokens: 0,
-                latencyMs,
-                status: 502,
-                logFile,
-                error: error.message,
-                apiKeyIndex,
-                pricing,
-                currency: provider.currency,
-                agent,
-                customTags,
-                routeTrace,
-            });
-            const fallbackResult = tryFallback(c, provider, model, targetPath, body, entryProtocol, routeTrace);
-            if (fallbackResult)
-                return fallbackResult;
-        }
-        recordRequest({
-            id: requestId,
-            tokenId: token.key,
-            providerId: provider.id,
-            model,
-            inputTokens: 0,
-            outputTokens: 0,
-            latencyMs,
-            status: 502,
-            logFile,
-            error: error.message,
-            apiKeyIndex,
-            pricing,
-            currency: provider.currency,
-            agent,
-            customTags,
-            routeTrace,
-        });
-        return c.json({ error: "Upstream request failed", detail: error.message }, 502);
+        return { kind: "retryable", status: 502, error: error.message };
     }
 }
-function tryFallback(c, provider, model, targetPath, body, entryProtocol, routeTrace) {
-    if (!provider.fallback)
-        return null;
-    const config = getConfig();
-    const fallbackProvider = config.providers.find((p) => p.id === provider.fallback && p.enabled);
-    if (!fallbackProvider)
-        return null;
-    const modelConfig = fallbackProvider.models.find((m) => getModelId(m) === model);
-    if (!modelConfig)
-        return null;
-    const fallbackPricing = getModelPricing(modelConfig);
-    let fallbackPath = targetPath;
-    if (fallbackProvider.type !== provider.type) {
-        if (fallbackProvider.type === "anthropic")
-            fallbackPath = "/v1/messages";
-        else
-            fallbackPath = "/chat/completions";
-    }
-    console.log(`[tokenparty] Falling back from ${provider.id} to ${fallbackProvider.id} for model ${model}`);
-    return forwardRequest(c, fallbackProvider, fallbackPath, body, entryProtocol, fallbackPricing, routeTrace);
-}
-function rawStreamPassthrough(c, targetUrl, upstreamHeaders, body, requestId, provider, model, token, startTime, logFile, apiKeyIndex, pricing, agent, customTags, routeTrace) {
+function rawStreamPassthrough(params) {
+    const { targetUrl, upstreamHeaders, body, requestId, provider, model, token, startTime, logFile, apiKeyIndex, pricing, agent, customTags, routeTrace, } = params;
     const url = new URL(targetUrl);
     const reqFn = url.protocol === "https:" ? httpsRequest : httpRequest;
-    return new Promise((resolve, reject) => {
-        const req = reqFn(url, { method: "POST", headers: { ...upstreamHeaders, "content-type": "application/json" } }, (res) => {
+    return new Promise((resolve) => {
+        const keepAliveAgent = url.protocol === "https:" ? httpsAgent : httpAgent;
+        const req = reqFn(url, {
+            method: "POST",
+            headers: { ...upstreamHeaders, "content-type": "application/json" },
+            agent: keepAliveAgent,
+        }, (res) => {
             const respHeaders = {};
             for (const [key, val] of Object.entries(res.headers)) {
                 if (val)
                     respHeaders[key] = Array.isArray(val) ? val.join(", ") : val;
             }
             const status = res.statusCode ?? 502;
+            // Check if retryable BEFORE piping - destroy stream and return retryable
+            if (isRetryableStatus(status)) {
+                res.destroy();
+                resolve({ kind: "retryable", status });
+                return;
+            }
             // Passthrough all upstream headers, skip hop-by-hop
             const passthroughHeaders = new Headers();
             const hopByHop = new Set(["connection", "keep-alive", "transfer-encoding", "te", "trailer", "upgrade"]);
@@ -361,20 +373,21 @@ function rawStreamPassthrough(c, targetUrl, upstreamHeaders, body, requestId, pr
                     callback(null, chunk);
                 },
                 flush(callback) {
-                    // Async parse for logging after stream ends
-                    asyncParseBufferForLog(rawChunks, res.headers["content-encoding"], requestId, respHeaders, provider, model, token, startTime, logFile, apiKeyIndex, pricing, agent, customTags, routeTrace);
+                    asyncParseBufferForLog(rawChunks, res.headers["content-encoding"], requestId, respHeaders, provider, model, token, startTime, logFile, apiKeyIndex, pricing, agent, customTags, routeTrace, status);
                     callback();
                 },
             });
             const stream = Readable.toWeb(res.pipe(passthrough));
-            resolve(new Response(stream, { status, headers: passthroughHeaders }));
+            resolve({ kind: "done", response: new Response(stream, { status, headers: passthroughHeaders }) });
+        });
+        req.on("error", (error) => {
+            resolve({ kind: "retryable", status: 502, error: error.message });
         });
-        req.on("error", reject);
         req.write(JSON.stringify(body));
         req.end();
     });
 }
-function asyncParseBufferForLog(rawChunks, encoding, requestId, respHeaders, provider, model, token, startTime, logFile, apiKeyIndex, pricing, agent, customTags, routeTrace) {
+function asyncParseBufferForLog(rawChunks, encoding, requestId, respHeaders, provider, model, token, startTime, logFile, apiKeyIndex, pricing, agent, customTags, routeTrace, upstreamStatus) {
     (async () => {
         let text;
         const combined = Buffer.concat(rawChunks);
@@ -392,55 +405,74 @@ function asyncParseBufferForLog(rawChunks, encoding, requestId, respHeaders, pro
         else {
             text = combined.toString("utf-8");
         }
+        const contentType = respHeaders["content-type"] ?? "";
+        const isSse = contentType.includes("text/event-stream");
+        const recordedStatus = upstreamStatus ?? 200;
         let fullContent = "";
         let rawEvents = [];
         let usage;
-        for (const line of text.split("\n")) {
-            if (!line.startsWith("data: "))
-                continue;
-            const data = line.slice(6).trim();
-            if (data === "[DONE]")
-                continue;
-            try {
-                const parsed = JSON.parse(data);
-                rawEvents.push(parsed);
-                if (provider.type === "anthropic" && parsed.type === "content_block_delta") {
-                    if (parsed.delta?.text)
-                        fullContent += parsed.delta.text;
-                    if (parsed.delta?.thinking)
-                        fullContent += parsed.delta.thinking;
-                }
-                else if (provider.type === "openai" && parsed.choices?.[0]?.delta?.content) {
-                    fullContent += parsed.choices[0].delta.content;
+        let responseBody;
+        if (isSse) {
+            for (const line of text.split("\n")) {
+                if (!line.startsWith("data: "))
+                    continue;
+                const data = line.slice(6).trim();
+                if (data === "[DONE]")
+                    continue;
+                try {
+                    const parsed = JSON.parse(data);
+                    rawEvents.push(parsed);
+                    if (provider.type === "anthropic" && parsed.type === "content_block_delta") {
+                        if (parsed.delta?.text)
+                            fullContent += parsed.delta.text;
+                        if (parsed.delta?.thinking)
+                            fullContent += parsed.delta.thinking;
+                    }
+                    else if (provider.type === "openai" && parsed.choices?.[0]?.delta?.content) {
+                        fullContent += parsed.choices[0].delta.content;
+                    }
+                    else if (parsed.type === "response.output_text.delta" && parsed.delta) {
+                        fullContent += parsed.delta;
+                    }
+                    usage = extractUsageFromChunk(parsed, provider.type) ?? usage;
                 }
-                else if (parsed.type === "response.output_text.delta" && parsed.delta) {
-                    fullContent += parsed.delta;
+                catch { }
+            }
+            if (!usage) {
+                for (let i = rawEvents.length - 1; i >= 0; i--) {
+                    const evt = rawEvents[i];
+                    if (evt.type === "response.completed" && evt.response?.usage) {
+                        usage = { input_tokens: evt.response.usage.input_tokens ?? 0, output_tokens: evt.response.usage.output_tokens ?? 0, cache_read_tokens: evt.response.usage.cache_read_input_tokens ?? 0, cache_write_tokens: evt.response.usage.cache_creation_input_tokens ?? 0 };
+                        break;
+                    }
+                    if (evt.usage && typeof evt.usage === "object" && (evt.usage.prompt_tokens || evt.usage.completion_tokens || evt.usage.input_tokens || evt.usage.output_tokens || evt.usage.total_tokens)) {
+                        usage = { input_tokens: evt.usage.prompt_tokens ?? evt.usage.input_tokens ?? 0, output_tokens: evt.usage.completion_tokens ?? evt.usage.output_tokens ?? 0, cache_read_tokens: evt.usage.prompt_tokens_details?.cached_tokens ?? evt.usage.cache_read_input_tokens ?? 0, cache_write_tokens: evt.usage.cache_creation_input_tokens ?? 0 };
+                        break;
+                    }
                 }
-                usage = extractUsageFromChunk(parsed, provider.type) ?? usage;
             }
-            catch { }
+            responseBody = rawEvents;
         }
-        if (!usage) {
-            for (let i = rawEvents.length - 1; i >= 0; i--) {
-                const evt = rawEvents[i];
-                if (evt.type === "response.completed" && evt.response?.usage) {
-                    usage = { input_tokens: evt.response.usage.input_tokens ?? 0, output_tokens: evt.response.usage.output_tokens ?? 0, cache_read_tokens: evt.response.usage.cache_read_input_tokens ?? 0, cache_write_tokens: evt.response.usage.cache_creation_input_tokens ?? 0 };
-                    break;
-                }
-                if (evt.usage && typeof evt.usage === "object" && (evt.usage.prompt_tokens || evt.usage.completion_tokens || evt.usage.input_tokens || evt.usage.output_tokens || evt.usage.total_tokens)) {
-                    usage = { input_tokens: evt.usage.prompt_tokens ?? evt.usage.input_tokens ?? 0, output_tokens: evt.usage.completion_tokens ?? evt.usage.output_tokens ?? 0, cache_read_tokens: evt.usage.prompt_tokens_details?.cached_tokens ?? evt.usage.cache_read_input_tokens ?? 0, cache_write_tokens: evt.usage.cache_creation_input_tokens ?? 0 };
-                    break;
-                }
+        else {
+            // Upstream returned a non-SSE body (e.g. JSON error) despite stream:true request.
+            // Record the raw decoded text faithfully.
+            try {
+                responseBody = JSON.parse(text);
+                usage = extractUsage(responseBody, provider.type);
+            }
+            catch {
+                responseBody = text;
             }
         }
         writeLog(requestId, {
             type: "response",
             timestamp: Date.now(),
             headers: respHeaders,
-            streaming: true,
-            streamContent: fullContent,
-            body: rawEvents,
+            streaming: isSse,
+            streamContent: isSse ? fullContent : undefined,
+            body: responseBody,
             usage,
+            status: recordedStatus,
         });
         recordRequest({
             id: requestId,
@@ -452,7 +484,7 @@ function asyncParseBufferForLog(rawChunks, encoding, requestId, respHeaders, pro
             cacheReadTokens: usage?.cache_read_tokens ?? 0,
             cacheWriteTokens: usage?.cache_write_tokens ?? 0,
             latencyMs: Date.now() - startTime,
-            status: 200,
+            status: recordedStatus,
             logFile,
             apiKeyIndex,
             pricing,