llm-switcher 1.6.15 → 1.6.16

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog — LLM Switcher
 
+## Release 1.6.16 — 2026-10-13
+
+- **A silent upstream stream.** The proxy now ends an upstream stream that sends no byte for 5 minutes, with an error. Before, only the default limit of the Node HTTP client stopped it. Set `LLM_SWITCHER_UPSTREAM_IDLE_MS` to change the limit.
+- **A test for a stream that ends with no finish reason.** A converted stream that ends cleanly without a finish reason gives an error, not a finished answer. A test now holds this behavior.
+
 ## Release 1.6.15 — 2026-10-13
 
 - **A message inside a tool batch.** Codex can put a notice between two tool outputs of one parallel batch. The proxy now moves the notice after the batch. Before, the next real output was replaced with a placeholder.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llm-switcher",
-  "version": "1.6.15",
+  "version": "1.6.16",
   "description": "Zero-dependency multi-protocol edge gateway & provider switcher for Claude Code and Codex, with OpenAI-compatible, Anthropic and Vertex upstreams",
   "keywords": [
     "llm",

package/tests/agy.e2e.test.mjs

@@ -46,6 +46,12 @@ function startUpstream() {
       try { json = raw ? JSON.parse(raw) : null; } catch {}
       received.push({ method: req.method, url: req.url, headers: req.headers, raw, body: json });
 
+      if (req.url.startsWith('/eof/v1/chat/completions')) {
+        // A chunk of text, then a clean end with no finish_reason and no [DONE].
+        res.writeHead(200, { 'Content-Type': 'text/event-stream' });
+        res.write('data: {"id":"c","choices":[{"index":0,"delta":{"content":"half an ans"}}]}\n\n');
+        return res.end();
+      }
       if (req.url.startsWith('/fail/v1/')) {
         res.writeHead(500, { 'Content-Type': 'application/json' });
         return res.end(JSON.stringify({ error: { message: 'upstream exploded' } }));
@@ -188,6 +194,7 @@ before(async () => {
       agyb: { name: 'agy on intact', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/intact/v1`, apiKey: 'sk-intact', defaultModels: { main: 'antigravity/*' } },
       agyc: { name: 'agy converted', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-chat', defaultModels: { main: 'gem/flash' } },
       agye: { name: 'agy empty slot', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-chat', defaultModels: {} },
+      agyeof: { name: 'agy clean EOF', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/eof/v1`, apiKey: 'sk-eof', defaultModels: { main: 'gem/flash' } },
       agy500: { name: 'agy failing upstream', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/fail/v1`, apiKey: 'sk-fail', defaultModels: { main: 'gem/flash' } },
       agyf: { name: 'agy flaky lookup', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/flaky/v1`, apiKey: 'sk-flaky', defaultModels: { main: 'antigravity/*' } },
       agynf: { name: 'agy lookup 404', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/nf/v1`, apiKey: 'sk-nf', defaultModels: { main: 'antigravity/*' } },
@@ -456,6 +463,15 @@ test('an agy profile with no main slot sends the model that agy picked', async (
   assert.equal(lastTo('/chat/v1/chat/completions').body.model, 'gemini-3.8-flash-high');
 });
 
+test('a converted stream that ends cleanly without a finish reason is an error, not a finished answer', async () => {
+  // router-for-me/CLIProxyAPI#6533: a clean EOF must not become finishReason STOP.
+  const r = await fetch(url('/v1internal:streamGenerateContent?alt=sse'), { method: 'POST', headers: agyHeaders({ 'x-llm-profile': 'agyeof' }), body: JSON.stringify(envelope()) });
+  const text = await r.text();
+  assert.ok(text.includes('half an ans'), text);
+  assert.ok(!/"finishReason":"STOP"/.test(text), `a cut stream ended as STOP:\n${text}`);
+  assert.match(text, /error/i);
+});
+
 test('a converted agy turn whose upstream fails gets the Code Assist error shape', async () => {
   const r = await fetch(url('/v1internal:generateContent'), { method: 'POST', headers: agyHeaders({ 'x-llm-profile': 'agy500' }), body: JSON.stringify(envelope()) });
   assert.ok(r.status >= 500, `status ${r.status}`);

package/tests/upstream-stream.test.mjs

@@ -28,3 +28,22 @@ test('malformed JSON is surfaced and stopping consumption cancels the upstream',
   for await (const item of readUpstreamPayloads({ body })) { assert.equal(item.one, 1); break; }
   assert.equal(canceled, true);
 });
+
+test('an upstream that goes silent mid-stream ends with an error after the idle limit', async () => {
+  // A dead HTTP/2 session can hold a stream open with no bytes (router-for-me/CLIProxyAPI#6534).
+  let sent = false;
+  let cancelled = false;
+  const body = new ReadableStream({
+    pull(controller) {
+      if (!sent) { sent = true; controller.enqueue(Buffer.from('data: {"a":1}\n\n')); }
+      return new Promise(() => {});
+    },
+    cancel() { cancelled = true; }
+  });
+  const values = [];
+  await assert.rejects(async () => {
+    for await (const v of readUpstreamPayloads({ body }, { idleMs: 50 })) values.push(v);
+  }, /sent nothing for/);
+  assert.deepEqual(values, [{ a: 1 }]);
+  assert.ok(cancelled, 'the silent upstream body is cancelled');
+});

package/upstream-stream.mjs

@@ -1,8 +1,20 @@
 // SSE dispatches one event at a blank line; data fields are joined with newlines.
 // Vertex also sends bare JSON lines. Keep both framings without parsing data fields separately.
 // `state.done` turns true on an OpenAI [DONE] line, which ends a stream cleanly even without a finish reason.
+// A stream that sends no byte for `state.idleMs` ends with an error: a dead HTTP/2 session can hold it open
+// with no bytes, and the default limit of the HTTP client is not this module's to rely on.
+export const UPSTREAM_IDLE_MS = Number(process.env.LLM_SWITCHER_UPSTREAM_IDLE_MS) || 300000;
+
 export async function* readUpstreamPayloads(upstreamRes, state = {}) {
   const reader = upstreamRes.body.getReader();
+  const idleMs = state.idleMs || UPSTREAM_IDLE_MS;
+  const read = () => {
+    let timer;
+    const idle = new Promise((_, reject) => {
+      timer = setTimeout(() => reject(new Error(`upstream stream sent nothing for ${Math.round(idleMs / 1000)}s`)), idleMs);
+    });
+    return Promise.race([reader.read(), idle]).finally(() => clearTimeout(timer));
+  };
   const decoder = new TextDecoder();
   let buffer = '', data = [], finished = false;
   const parse = value => {
@@ -20,7 +32,7 @@ export async function* readUpstreamPayloads(upstreamRes, state = {}) {
   }
   try {
     while (true) {
-      const chunk = await reader.read();
+      const chunk = await read();
       if (chunk.done) { finished = true; buffer += decoder.decode(); break; }
       buffer += decoder.decode(chunk.value, { stream: true });
       let newline;