llm-switcher 1.6.14 → 1.6.16

package/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog — LLM Switcher
 
+## Release 1.6.16 — 2026-10-13
+
+- **A silent upstream stream.** The proxy now ends an upstream stream that sends no byte for 5 minutes, with an error. Before, only the default limit of the Node HTTP client stopped it. Set `LLM_SWITCHER_UPSTREAM_IDLE_MS` to change the limit.
+- **A test for a stream that ends with no finish reason.** A converted stream that ends cleanly without a finish reason gives an error, not a finished answer. A test now holds this behavior.
+
+## Release 1.6.15 — 2026-10-13
+
+- **A message inside a tool batch.** Codex can put a notice between two tool outputs of one parallel batch. The proxy now moves the notice after the batch. Before, the next real output was replaced with a placeholder.
+- **Files and documents in a tool output.** An `input_file` in a Responses tool output and a `document` in an Anthropic `tool_result` now go to the model as their name. Before, the proxy wrote their base64 data into the text, which can use millions of tokens. The `input_image` parts of a Responses tool output now reach the model as images.
+- **Two shims on `PATH`.** A shim now skips every LLM Switcher shim directory when it looks for the real binary. Before, two shims on `PATH` called each other with no end, and each call wrote a new CA bundle.
+
 ## Release 1.6.14 — 2026-10-12
 
 - **System turn with an odd content shape.** On the translated route, a `system` turn whose `content` is not text and not a list is dropped. Before, the request failed with a `TypeError`.

package/formats.mjs

@@ -509,7 +509,7 @@ function anthropicToIR(payload) {
                   return '';
                 }
                 if (c?.type === 'tool_reference' && typeof c.tool_name === 'string') return c;
-                return JSON.stringify(c);
+                return mediaPlaceholder(c) ?? JSON.stringify(c);
               }).filter(Boolean);
               resultText = resultParts.map(c => typeof c === 'string' ? c : JSON.stringify(c)).join('\n');
             } else if (part.content) resultText = JSON.stringify(part.content);
@@ -679,12 +679,30 @@ function responsesOutputToText(out) {
       if (typeof x === 'string') return x;
       if (typeof x?.text === 'string') return x.text;
       if (x?.type === 'input_image') return '[image]';
-      return JSON.stringify(x);
+      return mediaPlaceholder(x) ?? JSON.stringify(x);
     }).join('\n');
   }
   return out == null ? '' : JSON.stringify(out);
 }
 
+// The input_image parts of a Responses tool output, as chat image parts.
+function responsesOutputImages(out) {
+  if (!Array.isArray(out)) return [];
+  return out.filter(x => x?.type === 'input_image')
+    .map(x => (typeof x.image_url === 'string' ? x.image_url : x.image_url?.url))
+    .filter(Boolean)
+    .map(url => ({ type: 'image_url', image_url: { url } }));
+}
+
+// A file or document block carries its bytes as base64; stringified into text it costs millions of tokens
+// (The-NeXT-AI/ai-gateway#28), so it is named instead. Null for any other block.
+function mediaPlaceholder(b) {
+  if (b?.type === 'input_file') return `[file: ${b.filename || b.file_id || 'attachment'}]`;
+  if (b?.type === 'document') return `[document${b.title ? `: ${b.title}` : ''}]`;
+  if (b?.source?.type === 'base64' || typeof b?.file_data === 'string') return `[${b.type || 'attachment'}]`;
+  return null;
+}
+
 // OpenAI Responses API (Codex) -> IR.
 function responsesToIR(payload, { allowOpaqueCompaction = false } = {}) {
   const ir = baseIR();
@@ -772,7 +790,8 @@ function responsesToIR(payload, { allowOpaqueCompaction = false } = {}) {
         const a = item.action || {};
         pushCall({ id: item.call_id || item.id || null, name: 'local_shell', args: { command: a.command || [], workdir: a.working_directory ?? undefined, timeout_ms: a.timeout_ms ?? undefined } });
       } else if (item.type === 'function_call_output' || item.type === 'custom_tool_call_output' || item.type === 'local_shell_call_output') {
-        ir.messages.push({ role: 'tool', toolCallId: item.call_id || item.id, content: responsesOutputToText(item.output) });
+        const images = responsesOutputImages(item.output);
+        ir.messages.push({ role: 'tool', toolCallId: item.call_id || item.id, content: responsesOutputToText(item.output), ...(images.length ? { images } : {}) });
       } else if (item.type === 'additional_tools' && Array.isArray(item.tools)) {
         toolList.push(...item.tools);
       } else if (item.type === 'compaction') {
@@ -1032,13 +1051,15 @@ function hasNativeReasoning(model) {
 //    -> OpenAI: "assistant message with 'tool_calls' must be followed by tool messages"
 //
 // Rule: a tool result is only valid when placed right after the assistant turn that declared it. Orphaned results are
-// converted to user text (preserving context); tool calls missing a result get a placeholder result.
+// converted to user text (preserving context); tool calls missing a result get a placeholder result. A non-assistant
+// message inside an unanswered batch (Codex inserts notices there) moves after the batch, so later outputs still pair.
 const MISSING_TOOL_RESULT = '[Tool result unavailable: it was removed from the conversation history]';
 
 function healToolPairs(messages) {
   const out = [];
   let pending = null;   // Map<callId, name> of the most recent assistant turn
   let deferred = [];    // orphaned results seen while awaiting results -> flushed later to preserve adjacency
+  let held = [];        // non-tool messages seen while awaiting results -> emitted after the batch
   let seq = 0;
 
   const orphanText = (m) => ({
@@ -1055,6 +1076,8 @@ function healToolPairs(messages) {
     }
     for (const m of deferred) out.push(orphanText(m));
     deferred = [];
+    out.push(...held);
+    held = [];
   };
 
   let unnamed = [];     // ids given to calls that arrived with no id, in order
@@ -1068,6 +1091,7 @@ function healToolPairs(messages) {
         const name = pending.get(id);
         pending.delete(id);
         out.push({ ...m, toolCallId: id, name: m.name || name });
+        if (!pending.size) flush();
       } else if (pending) {
         deferred.push(m);
       } else {
@@ -1075,6 +1099,10 @@ function healToolPairs(messages) {
       }
       continue;
     }
+    if (pending?.size && m.role !== 'assistant') {
+      held.push(m);
+      continue;
+    }
     flush();
     if (m.role === 'assistant' && Array.isArray(m.toolCalls) && m.toolCalls.length) {
       unnamed = [];
@@ -1850,7 +1878,7 @@ function isBlankText(b) {
 function toolResultText(block) {
   const c = block.content;
   const text = typeof c === 'string' ? c
-    : Array.isArray(c) ? c.map(x => (x?.type === 'text' ? x.text : x?.type === 'image' ? '[image omitted]' : JSON.stringify(x))).join('\n')
+    : Array.isArray(c) ? c.map(x => (x?.type === 'text' ? x.text : x?.type === 'image' ? '[image omitted]' : mediaPlaceholder(x) ?? JSON.stringify(x))).join('\n')
     : c ? JSON.stringify(c) : '';
   return `[Tool Result${block.tool_use_id ? ` (${block.tool_use_id})` : ''}]: ${text}`;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llm-switcher",
-  "version": "1.6.14",
+  "version": "1.6.16",
   "description": "Zero-dependency multi-protocol edge gateway & provider switcher for Claude Code and Codex, with OpenAI-compatible, Anthropic and Vertex upstreams",
   "keywords": [
     "llm",

package/shim.mjs

@@ -201,7 +201,8 @@ for _v in $(compgen -v LLM_SWITCHER_CODEX_ 2>/dev/null); do
 done
 ${caBlock}
 
-# Find the real binary: drop the shim directory from PATH so it doesn't call itself.
+# Find the real binary: drop every switcher shim directory from PATH, this one and any other
+# (a second install, a test HOME), or two shims call each other forever.
 CLEAN_PATH=""
 IFS=':'
 for p in $PATH; do
@@ -209,6 +210,7 @@ for p in $PATH; do
     "$SHIM_DIR"|"$SHIM_DIR"/) continue ;;
   esac
   [ -z "$p" ] && continue
+  if [ -f "$p/${name}" ] && grep -q 'Auto-generated by LLM Switcher' "$p/${name}" 2>/dev/null; then continue; fi
   if [ -z "$CLEAN_PATH" ]; then CLEAN_PATH="$p"; else CLEAN_PATH="$CLEAN_PATH:$p"; fi
 done
 unset IFS

package/tests/agy.e2e.test.mjs

@@ -46,6 +46,12 @@ function startUpstream() {
       try { json = raw ? JSON.parse(raw) : null; } catch {}
       received.push({ method: req.method, url: req.url, headers: req.headers, raw, body: json });
 
+      if (req.url.startsWith('/eof/v1/chat/completions')) {
+        // A chunk of text, then a clean end with no finish_reason and no [DONE].
+        res.writeHead(200, { 'Content-Type': 'text/event-stream' });
+        res.write('data: {"id":"c","choices":[{"index":0,"delta":{"content":"half an ans"}}]}\n\n');
+        return res.end();
+      }
       if (req.url.startsWith('/fail/v1/')) {
         res.writeHead(500, { 'Content-Type': 'application/json' });
         return res.end(JSON.stringify({ error: { message: 'upstream exploded' } }));
@@ -188,6 +194,7 @@ before(async () => {
       agyb: { name: 'agy on intact', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/intact/v1`, apiKey: 'sk-intact', defaultModels: { main: 'antigravity/*' } },
       agyc: { name: 'agy converted', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-chat', defaultModels: { main: 'gem/flash' } },
       agye: { name: 'agy empty slot', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-chat', defaultModels: {} },
+      agyeof: { name: 'agy clean EOF', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/eof/v1`, apiKey: 'sk-eof', defaultModels: { main: 'gem/flash' } },
       agy500: { name: 'agy failing upstream', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/fail/v1`, apiKey: 'sk-fail', defaultModels: { main: 'gem/flash' } },
       agyf: { name: 'agy flaky lookup', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/flaky/v1`, apiKey: 'sk-flaky', defaultModels: { main: 'antigravity/*' } },
       agynf: { name: 'agy lookup 404', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/nf/v1`, apiKey: 'sk-nf', defaultModels: { main: 'antigravity/*' } },
@@ -456,6 +463,15 @@ test('an agy profile with no main slot sends the model that agy picked', async (
   assert.equal(lastTo('/chat/v1/chat/completions').body.model, 'gemini-3.8-flash-high');
 });
 
+test('a converted stream that ends cleanly without a finish reason is an error, not a finished answer', async () => {
+  // router-for-me/CLIProxyAPI#6533: a clean EOF must not become finishReason STOP.
+  const r = await fetch(url('/v1internal:streamGenerateContent?alt=sse'), { method: 'POST', headers: agyHeaders({ 'x-llm-profile': 'agyeof' }), body: JSON.stringify(envelope()) });
+  const text = await r.text();
+  assert.ok(text.includes('half an ans'), text);
+  assert.ok(!/"finishReason":"STOP"/.test(text), `a cut stream ended as STOP:\n${text}`);
+  assert.match(text, /error/i);
+});
+
 test('a converted agy turn whose upstream fails gets the Code Assist error shape', async () => {
   const r = await fetch(url('/v1internal:generateContent'), { method: 'POST', headers: agyHeaders({ 'x-llm-profile': 'agy500' }), body: JSON.stringify(envelope()) });
   assert.ok(r.status >= 500, `status ${r.status}`);

package/tests/formats.test.mjs

@@ -1699,3 +1699,58 @@ test('the placeholder strip refuses a client name that two padded tools share',
   assert.deepEqual(JSON.parse(vocab.args(alias[0], filled)), { env: {} });
   assert.equal(vocab.args('a_b', filled), filled, 'a_b declared its own _placeholder');
 });
+
+test('responsesToIR + irToChatBody: a notice inside a parallel batch moves after it and every real output is kept', () => {
+  // Codex inserts a notice between two outputs of one batch (openai/codex#52834).
+  const ir = responsesToIR({
+    model: 'x', input: [
+      { role: 'user', content: 'do two things' },
+      { type: 'function_call', call_id: 'c1', name: 'a', arguments: '{}' },
+      { type: 'function_call', call_id: 'c2', name: 'b', arguments: '{}' },
+      { type: 'function_call_output', call_id: 'c1', output: 'out1' },
+      { role: 'user', content: 'notice' },
+      { type: 'function_call_output', call_id: 'c2', output: 'out2' }
+    ]
+  });
+  for (const body of [irToChatBody(ir, 'gpt-4o')]) {
+    const shape = body.messages.map(m => m.role === 'tool' ? `tool:${m.content}` : `${m.role}`);
+    assert.deepEqual(shape, ['user', 'assistant', 'tool:out1', 'tool:out2', 'user']);
+  }
+  const anth = irToAnthropicBody(ir, 'claude-x');
+  const results = anth.messages[2].content.filter(b => b.type === 'tool_result').map(b => JSON.stringify(b.content));
+  assert.ok(results.every(r => !r.includes('unavailable')), `real outputs replaced: ${results}`);
+});
+
+test('a tool output that carries a file or an image never reaches the model as base64 text', () => {
+  // The-NeXT-AI/ai-gateway#28: a 5.9 MB image serialized as text became 4.1M tokens.
+  const blob = 'QUFB'.repeat(5000);
+  const fromResponses = responsesToIR({
+    model: 'x', input: [
+      { type: 'function_call', call_id: 'c1', name: 'shot', arguments: '{}' },
+      { type: 'function_call_output', call_id: 'c1', output: [
+        { type: 'input_image', image_url: `data:image/png;base64,${blob}` },
+        { type: 'input_file', filename: 'a.pdf', file_data: `data:application/pdf;base64,${blob}` }] }
+    ]
+  });
+  const fromAnthropic = anthropicToIR({
+    model: 'x', messages: [
+      { role: 'assistant', content: [{ type: 'tool_use', id: 't1', name: 'read', input: {} }] },
+      { role: 'user', content: [{ type: 'tool_result', tool_use_id: 't1', content: [
+        { type: 'document', source: { type: 'base64', media_type: 'application/pdf', data: blob } }] }] }
+    ]
+  });
+  for (const [name, ir] of [['responses', fromResponses], ['anthropic', fromAnthropic]]) {
+    const tool = ir.messages.find(m => m.role === 'tool');
+    assert.ok(!tool.content.includes(blob), `${name}: tool text carries the base64`);
+  }
+  const tool = fromResponses.messages.find(m => m.role === 'tool');
+  assert.match(tool.content, /a\.pdf/);
+  assert.equal(tool.images?.[0]?.image_url?.url, `data:image/png;base64,${blob}`);
+});
+
+test('irToAnthropicBody: a whitespace-only text block is blank too', () => {
+  // Anthropic: "text content blocks must contain non-whitespace text".
+  const ir = anthropicToIR({ model: 'x', max_tokens: 10, messages: [{ role: 'user', content: [{ type: 'text', text: ' \n' }, { type: 'text', text: 'hi' }] }] });
+  const body = irToAnthropicBody(ir, 'claude');
+  assert.ok(body.messages.every(m => m.content.every(b => b.type !== 'text' || b.text.trim())), JSON.stringify(body.messages));
+});

package/tests/shim.test.mjs

@@ -119,6 +119,18 @@ test('shim never recurses into itself', (t) => {
   assert.match(out, /ARGS=x/);
 });
 
+test('two switcher shims on PATH never call each other', (t) => {
+  // A test HOME's shim ahead of the account's own shim: each dropped only its own directory, so they
+  // found each other forever, and every hop built a new CA bundle (2026-10-13: 702 files in 2 minutes).
+  if (process.platform === 'win32') return t.skip('posix only');
+  const other = fs.mkdtempSync(path.join(SCRATCH, 'shimother-'));
+  fs.copyFileSync(path.join(RENDER_DIR, 'claude'), path.join(other, 'claude'));
+  fs.chmodSync(path.join(other, 'claude'), 0o755);
+  const fake = makeFakeBin('claude');
+  const out = runShim('claude', ['x'], { active: true, fakeDir: other, extraPath: `${fake}:/usr/bin:/bin` });
+  assert.match(out, /ARGS=x/);
+});
+
 test('shim fails loudly (127) when the real binary is missing', (t) => {
   if (process.platform === 'win32') return t.skip('posix only');
   const empty = fs.mkdtempSync(path.join(SCRATCH, 'shimempty-'));

package/tests/upstream-stream.test.mjs

@@ -28,3 +28,22 @@ test('malformed JSON is surfaced and stopping consumption cancels the upstream',
   for await (const item of readUpstreamPayloads({ body })) { assert.equal(item.one, 1); break; }
   assert.equal(canceled, true);
 });
+
+test('an upstream that goes silent mid-stream ends with an error after the idle limit', async () => {
+  // A dead HTTP/2 session can hold a stream open with no bytes (router-for-me/CLIProxyAPI#6534).
+  let sent = false;
+  let cancelled = false;
+  const body = new ReadableStream({
+    pull(controller) {
+      if (!sent) { sent = true; controller.enqueue(Buffer.from('data: {"a":1}\n\n')); }
+      return new Promise(() => {});
+    },
+    cancel() { cancelled = true; }
+  });
+  const values = [];
+  await assert.rejects(async () => {
+    for await (const v of readUpstreamPayloads({ body }, { idleMs: 50 })) values.push(v);
+  }, /sent nothing for/);
+  assert.deepEqual(values, [{ a: 1 }]);
+  assert.ok(cancelled, 'the silent upstream body is cancelled');
+});

package/upstream-stream.mjs

@@ -1,8 +1,20 @@
 // SSE dispatches one event at a blank line; data fields are joined with newlines.
 // Vertex also sends bare JSON lines. Keep both framings without parsing data fields separately.
 // `state.done` turns true on an OpenAI [DONE] line, which ends a stream cleanly even without a finish reason.
+// A stream that sends no byte for `state.idleMs` ends with an error: a dead HTTP/2 session can hold it open
+// with no bytes, and the default limit of the HTTP client is not this module's to rely on.
+export const UPSTREAM_IDLE_MS = Number(process.env.LLM_SWITCHER_UPSTREAM_IDLE_MS) || 300000;
+
 export async function* readUpstreamPayloads(upstreamRes, state = {}) {
   const reader = upstreamRes.body.getReader();
+  const idleMs = state.idleMs || UPSTREAM_IDLE_MS;
+  const read = () => {
+    let timer;
+    const idle = new Promise((_, reject) => {
+      timer = setTimeout(() => reject(new Error(`upstream stream sent nothing for ${Math.round(idleMs / 1000)}s`)), idleMs);
+    });
+    return Promise.race([reader.read(), idle]).finally(() => clearTimeout(timer));
+  };
   const decoder = new TextDecoder();
   let buffer = '', data = [], finished = false;
   const parse = value => {
@@ -20,7 +32,7 @@ export async function* readUpstreamPayloads(upstreamRes, state = {}) {
   }
   try {
     while (true) {
-      const chunk = await reader.read();
+      const chunk = await read();
       if (chunk.done) { finished = true; buffer += decoder.decode(); break; }
       buffer += decoder.decode(chunk.value, { stream: true });
       let newline;