llm-switcher 1.6.15 → 1.6.17

package/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog — LLM Switcher
 
+## Release 1.6.17 — 2026-10-13
+
+- **Documents in a tool result.** A document with a text source, a content source or a URL goes to the model as its text or its URL. Before 1.6.17, it went as a bare `[document]` label. Only base64 data becomes a label, for example `[file: a.pdf]`.
+- **The upstream idle limit.** `LLM_SWITCHER_UPSTREAM_IDLE_MS` must be a positive number. The proxy uses 300000 for any other value, and it writes a warning. A value above 2147483647 becomes 2147483647. Before 1.6.17, a negative, infinite or too large value ended every stream after 1 ms. The limit counts from the response headers. It applies to the body only, and an SSE comment line resets it.
+
+## Release 1.6.16 — 2026-10-13
+
+- **A silent upstream stream.** The proxy now ends an upstream stream that sends no byte for 5 minutes, with an error. Before, only the default limit of the Node HTTP client stopped it. Set `LLM_SWITCHER_UPSTREAM_IDLE_MS` to change the limit.
+- **A test for a stream that ends with no finish reason.** A converted stream that ends cleanly without a finish reason gives an error, not a finished answer. A test now holds this behavior.
+
 ## Release 1.6.15 — 2026-10-13
 
 - **A message inside a tool batch.** Codex can put a notice between two tool outputs of one parallel batch. The proxy now moves the notice after the batch. Before, the next real output was replaced with a placeholder.

package/formats.mjs

@@ -697,9 +697,22 @@ function responsesOutputImages(out) {
 // A file or document block carries its bytes as base64; stringified into text it costs millions of tokens
 // (The-NeXT-AI/ai-gateway#28), so it is named instead. Null for any other block.
 function mediaPlaceholder(b) {
-  if (b?.type === 'input_file') return `[file: ${b.filename || b.file_id || 'attachment'}]`;
-  if (b?.type === 'document') return `[document${b.title ? `: ${b.title}` : ''}]`;
-  if (b?.source?.type === 'base64' || typeof b?.file_data === 'string') return `[${b.type || 'attachment'}]`;
+  const src = b?.source;
+  if (src?.type === 'base64' || typeof b?.file_data === 'string') {
+    if (b.type === 'input_file') return `[file: ${b.filename || b.file_id || 'attachment'}]`;
+    if (b.type === 'document') return `[document${b.title ? `: ${b.title}` : ''}]`;
+    return `[${b.type || 'attachment'}]`;
+  }
+  // Not base64: the content itself goes on, as text the model can read.
+  if (b?.type === 'document' && src?.type === 'text' && typeof src.data === 'string') return src.data;
+  if (b?.type === 'document' && src?.type === 'content') {
+    if (typeof src.content === 'string') return src.content;
+    if (Array.isArray(src.content)) {
+      return src.content.map(c => (c?.type === 'text' ? c.text || '' : c?.type === 'image' ? '[image omitted]' : '')).filter(Boolean).join('\n');
+    }
+  }
+  if (b?.type === 'document' && src?.type === 'url' && src.url) return `[document: ${src.url}]`;
+  if (b?.type === 'input_file' && b.file_url) return `[file: ${b.file_url}]`;
   return null;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llm-switcher",
-  "version": "1.6.15",
+  "version": "1.6.17",
   "description": "Zero-dependency multi-protocol edge gateway & provider switcher for Claude Code and Codex, with OpenAI-compatible, Anthropic and Vertex upstreams",
   "keywords": [
     "llm",

package/tests/agy.e2e.test.mjs

@@ -46,6 +46,12 @@ function startUpstream() {
       try { json = raw ? JSON.parse(raw) : null; } catch {}
       received.push({ method: req.method, url: req.url, headers: req.headers, raw, body: json });
 
+      if (req.url.startsWith('/eof/v1/chat/completions')) {
+        // A chunk of text, then a clean end with no finish_reason and no [DONE].
+        res.writeHead(200, { 'Content-Type': 'text/event-stream' });
+        res.write('data: {"id":"c","choices":[{"index":0,"delta":{"content":"half an ans"}}]}\n\n');
+        return res.end();
+      }
       if (req.url.startsWith('/fail/v1/')) {
         res.writeHead(500, { 'Content-Type': 'application/json' });
         return res.end(JSON.stringify({ error: { message: 'upstream exploded' } }));
@@ -188,6 +194,7 @@ before(async () => {
       agyb: { name: 'agy on intact', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/intact/v1`, apiKey: 'sk-intact', defaultModels: { main: 'antigravity/*' } },
       agyc: { name: 'agy converted', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-chat', defaultModels: { main: 'gem/flash' } },
       agye: { name: 'agy empty slot', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-chat', defaultModels: {} },
+      agyeof: { name: 'agy clean EOF', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/eof/v1`, apiKey: 'sk-eof', defaultModels: { main: 'gem/flash' } },
       agy500: { name: 'agy failing upstream', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/fail/v1`, apiKey: 'sk-fail', defaultModels: { main: 'gem/flash' } },
       agyf: { name: 'agy flaky lookup', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/flaky/v1`, apiKey: 'sk-flaky', defaultModels: { main: 'antigravity/*' } },
       agynf: { name: 'agy lookup 404', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/nf/v1`, apiKey: 'sk-nf', defaultModels: { main: 'antigravity/*' } },
@@ -456,6 +463,15 @@ test('an agy profile with no main slot sends the model that agy picked', async (
   assert.equal(lastTo('/chat/v1/chat/completions').body.model, 'gemini-3.8-flash-high');
 });
 
+test('a converted stream that ends cleanly without a finish reason is an error, not a finished answer', async () => {
+  // router-for-me/CLIProxyAPI#6533: a clean EOF must not become finishReason STOP.
+  const r = await fetch(url('/v1internal:streamGenerateContent?alt=sse'), { method: 'POST', headers: agyHeaders({ 'x-llm-profile': 'agyeof' }), body: JSON.stringify(envelope()) });
+  const text = await r.text();
+  assert.ok(text.includes('half an ans'), text);
+  assert.ok(!/"finishReason":"STOP"/.test(text), `a cut stream ended as STOP:\n${text}`);
+  assert.match(text, /error/i);
+});
+
 test('a converted agy turn whose upstream fails gets the Code Assist error shape', async () => {
   const r = await fetch(url('/v1internal:generateContent'), { method: 'POST', headers: agyHeaders({ 'x-llm-profile': 'agy500' }), body: JSON.stringify(envelope()) });
   assert.ok(r.status >= 500, `status ${r.status}`);

package/tests/formats.test.mjs

@@ -1754,3 +1754,91 @@ test('irToAnthropicBody: a whitespace-only text block is blank too', () => {
   const body = irToAnthropicBody(ir, 'claude');
   assert.ok(body.messages.every(m => m.content.every(b => b.type !== 'text' || b.text.trim())), JSON.stringify(body.messages));
 });
+
+// ---- roundtable ruling 2026-10-13: media in tool results, held notices ----
+
+const BLOB = 'QUFB'.repeat(2000);
+const MISSING_TOOL_RESULT = '[Tool result unavailable: it was removed from the conversation history]';
+function anthropicToolText(content) {
+  const ir = anthropicToIR({ model: 'x', messages: [
+    { role: 'assistant', content: [{ type: 'tool_use', id: 't1', name: 'read', input: {} }] },
+    { role: 'user', content: [{ type: 'tool_result', tool_use_id: 't1', content }] }
+  ] });
+  return ir.messages.find(m => m.role === 'tool').content;
+}
+
+test('a tool-result document keeps its text, its URL and its label; only base64 becomes a placeholder', () => {
+  assert.match(anthropicToolText([{ type: 'document', source: { type: 'text', media_type: 'text/plain', data: 'hello doc' } }]), /hello doc/);
+  const content = anthropicToolText([{ type: 'document', source: { type: 'content', content: [
+    { type: 'text', text: 'part one' }, { type: 'image', source: { type: 'base64', media_type: 'image/png', data: BLOB } }] } }]);
+  assert.match(content, /part one/);
+  assert.match(content, /\[image omitted\]/);
+  assert.ok(!content.includes(BLOB));
+  assert.equal(anthropicToolText([{ type: 'document', source: { type: 'content', content: 'plain string' } }]), 'plain string');
+  assert.match(anthropicToolText([{ type: 'document', source: { type: 'url', url: 'https://example.com/a.pdf' } }]), /https:\/\/example\.com\/a\.pdf/);
+  assert.match(anthropicToolText([{ type: 'input_file', file_url: 'https://example.com/f.pdf' }]), /https:\/\/example\.com\/f\.pdf/);
+  assert.equal(anthropicToolText([{ type: 'input_file', filename: 'a.pdf', file_data: `data:application/pdf;base64,${BLOB}` }]), '[file: a.pdf]');
+  assert.equal(anthropicToolText([{ type: 'document', title: 'T', source: { type: 'base64', media_type: 'application/pdf', data: BLOB } }]), '[document: T]');
+  assert.ok(!anthropicToolText([{ type: 'document', source: { type: 'url' } }]).includes('[document: undefined]'));
+});
+
+test('an orphan tool result turned into text keeps a text document', () => {
+  const { payload } = healAnthropicPayload({ model: 'x', max_tokens: 10, messages: [
+    { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'orphan', content: [{ type: 'document', source: { type: 'text', data: 'hello doc' } }] }] }
+  ] });
+  assert.match(JSON.stringify(payload.messages), /hello doc/);
+});
+
+function chatShape(input) {
+  return irToChatBody(responsesToIR({ model: 'x', input }), 'gpt-4o').messages
+    .map(m => (m.role === 'tool' ? `tool:${m.content}` : m.role === 'user' ? `user:${typeof m.content === 'string' ? m.content : JSON.stringify(m.content)}` : m.role));
+}
+const callAB = [
+  { type: 'function_call', call_id: 'c1', name: 'a', arguments: '{}' },
+  { type: 'function_call', call_id: 'c2', name: 'b', arguments: '{}' }
+];
+
+test('healToolPairs: a held notice follows the placeholder of an output that never came', () => {
+  const missing = chatShape([...callAB, { type: 'function_call_output', call_id: 'c1', output: 'out1' }, { role: 'user', content: 'notice' }]);
+  assert.deepEqual(missing.slice(-4), ['assistant', 'tool:out1', `tool:${MISSING_TOOL_RESULT}`, 'user:notice']);
+  const later = chatShape([...callAB, { type: 'function_call_output', call_id: 'c1', output: 'out1' }, { role: 'user', content: 'notice' },
+    { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'next turn' }] }]);
+  assert.deepEqual(later.slice(-2), ['user:notice', 'assistant']);
+  const two = chatShape([...callAB, { role: 'user', content: 'notice1' }, { type: 'function_call_output', call_id: 'c1', output: 'out1' }, { role: 'user', content: 'notice2' }]);
+  assert.deepEqual(two.slice(-5), ['assistant', 'tool:out1', `tool:${MISSING_TOOL_RESULT}`, 'user:notice1', 'user:notice2']);
+});
+
+test('placeholders and URLs are exact on every route that turns a tool result into text', () => {
+  assert.equal(anthropicToolText([{ type: 'document', source: { type: 'base64', media_type: 'application/pdf', data: BLOB } }]), '[document]');
+  assert.equal(anthropicToolText([{ type: 'document', source: { type: 'url', url: 'https://example.com/a.pdf' } }]), '[document: https://example.com/a.pdf]');
+  assert.equal(anthropicToolText([{ type: 'input_file', file_url: 'https://example.com/f.pdf' }]), '[file: https://example.com/f.pdf]');
+  const orphanText = (content) => {
+    const { payload } = healAnthropicPayload({ model: 'x', max_tokens: 10, messages: [
+      { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'orphan', content }] }] });
+    return JSON.stringify(payload.messages);
+  };
+  const doc = orphanText([{ type: 'document', title: 'T', source: { type: 'base64', media_type: 'application/pdf', data: BLOB } }]);
+  assert.ok(!doc.includes(BLOB) && doc.includes('[document: T]'), doc.slice(0, 300));
+  const file = orphanText([{ type: 'input_file', filename: 'a.pdf', file_data: `data:application/pdf;base64,${BLOB}` }]);
+  assert.ok(!file.includes(BLOB) && file.includes('[file: a.pdf]'), file.slice(0, 300));
+});
+
+test('the image of a Responses tool output reaches the chat and Anthropic bodies', () => {
+  for (const image_url of ['data:image/png;base64,AAAA', { url: 'data:image/png;base64,AAAA' }]) {
+    const ir = responsesToIR({ model: 'x', input: [
+      { type: 'function_call', call_id: 'c1', name: 'shot', arguments: '{}' },
+      { type: 'function_call_output', call_id: 'c1', output: [{ type: 'input_image', image_url }] }] });
+    const chat = JSON.stringify(irToChatBody(ir, 'gpt-4o').messages);
+    assert.match(chat, /data:image\/png;base64,AAAA/);
+    const anth = JSON.stringify(irToAnthropicBody(ir, 'claude-x').messages);
+    assert.match(anth, /"type":"image"/);
+  }
+});
+
+test('healToolPairs: an orphan output and a held notice in one batch come out orphan first, then the notice', () => {
+  const shape = chatShape([...callAB,
+    { type: 'function_call_output', call_id: 'zz', output: 'stray' },
+    { role: 'user', content: 'notice' },
+    { type: 'function_call_output', call_id: 'c1', output: 'out1' }]);
+  assert.deepEqual(shape.slice(-5), ['assistant', 'tool:out1', `tool:${MISSING_TOOL_RESULT}`, 'user:[Tool Result (zz)]: stray', 'user:notice']);
+});

package/tests/upstream-stream.test.mjs

@@ -1,6 +1,6 @@
 import { test } from 'node:test';
 import assert from 'node:assert/strict';
-import { readUpstreamPayloads } from '../upstream-stream.mjs';
+import { readUpstreamPayloads, parseIdleMs } from '../upstream-stream.mjs';
 
 async function collect(text, widths = [1, 3, 7]) {
   const bytes = Buffer.from(text); let offset = 0, i = 0;
@@ -28,3 +28,44 @@ test('malformed JSON is surfaced and stopping consumption cancels the upstream',
   for await (const item of readUpstreamPayloads({ body })) { assert.equal(item.one, 1); break; }
   assert.equal(canceled, true);
 });
+
+test('an upstream that goes silent mid-stream ends with an error after the idle limit', async () => {
+  // A dead HTTP/2 session can hold a stream open with no bytes (router-for-me/CLIProxyAPI#6534).
+  let sent = false;
+  let cancelled = false;
+  const body = new ReadableStream({
+    pull(controller) {
+      if (!sent) { sent = true; controller.enqueue(Buffer.from('data: {"a":1}\n\n')); }
+      return new Promise(() => {});
+    },
+    cancel() { cancelled = true; }
+  });
+  const values = [];
+  await assert.rejects(async () => {
+    for await (const v of readUpstreamPayloads({ body }, { idleMs: 50 })) values.push(v);
+  }, /sent nothing for/);
+  assert.deepEqual(values, [{ a: 1 }]);
+  assert.ok(cancelled, 'the silent upstream body is cancelled');
+});
+
+test('parseIdleMs keeps the idle limit inside what setTimeout accepts', () => {
+  // Node fires a timer of a negative, infinite or > 2^31-1 delay after 1 ms.
+  assert.equal(parseIdleMs('3000000000'), 2147483647);
+  for (const bad of ['-5', 'Infinity', 'abc', '0', '0.5']) assert.equal(parseIdleMs(bad), 300000, bad);
+  assert.equal(parseIdleMs('120000'), 120000);
+  assert.equal(parseIdleMs(undefined), 300000);
+});
+
+test('the idle limit counts the gap between reads, not the whole stream', async () => {
+  let n = 0;
+  const body = new ReadableStream({
+    async pull(controller) {
+      if (n === 5) return controller.close();
+      await new Promise(r => setTimeout(r, 60));
+      controller.enqueue(Buffer.from(`data: {"n":${n++}}\n\n`));
+    }
+  });
+  const values = [];
+  for await (const v of readUpstreamPayloads({ body }, { idleMs: 200 })) values.push(v.n);
+  assert.deepEqual(values, [0, 1, 2, 3, 4]);
+});

package/upstream-stream.mjs

@@ -1,8 +1,34 @@
 // SSE dispatches one event at a blank line; data fields are joined with newlines.
 // Vertex also sends bare JSON lines. Keep both framings without parsing data fields separately.
 // `state.done` turns true on an OpenAI [DONE] line, which ends a stream cleanly even without a finish reason.
+// A stream that sends no byte for `state.idleMs` ends with an error: a dead HTTP/2 session can hold it open
+// with no bytes, and the default limit of the HTTP client is not this module's to rely on.
+const DEFAULT_IDLE_MS = 300000;
+const MAX_TIMER_MS = 2147483647;
+
+// The idle limit as setTimeout takes it: Node fires a negative, infinite or larger delay after 1 ms.
+export function parseIdleMs(value) {
+  if (value === undefined || value === null || value === '') return DEFAULT_IDLE_MS;
+  const n = Number(value);
+  if (!Number.isFinite(n) || n < 1) {
+    console.warn(`[llm-switcher] upstream idle limit ${JSON.stringify(value)} is not a positive number; using ${DEFAULT_IDLE_MS} ms`);
+    return DEFAULT_IDLE_MS;
+  }
+  return Math.min(n, MAX_TIMER_MS);
+}
+
+export const UPSTREAM_IDLE_MS = parseIdleMs(process.env.LLM_SWITCHER_UPSTREAM_IDLE_MS);
+
 export async function* readUpstreamPayloads(upstreamRes, state = {}) {
   const reader = upstreamRes.body.getReader();
+  const idleMs = state.idleMs === undefined ? UPSTREAM_IDLE_MS : parseIdleMs(state.idleMs);
+  const read = () => {
+    let timer;
+    const idle = new Promise((_, reject) => {
+      timer = setTimeout(() => reject(new Error(`upstream stream sent nothing for ${Math.round(idleMs / 1000)}s`)), idleMs);
+    });
+    return Promise.race([reader.read(), idle]).finally(() => clearTimeout(timer));
+  };
   const decoder = new TextDecoder();
   let buffer = '', data = [], finished = false;
   const parse = value => {
@@ -20,7 +46,7 @@ export async function* readUpstreamPayloads(upstreamRes, state = {}) {
   }
   try {
     while (true) {
-      const chunk = await reader.read();
+      const chunk = await read();
       if (chunk.done) { finished = true; buffer += decoder.decode(); break; }
       buffer += decoder.decode(chunk.value, { stream: true });
       let newline;