llm-switcher 1.6.16 → 1.6.17

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog — LLM Switcher
 
+## Release 1.6.17 — 2026-10-13
+
+- **Documents in a tool result.** A document with a text source, a content source or a URL goes to the model as its text or its URL. Before 1.6.17, it went as a bare `[document]` label. Only base64 data becomes a label, for example `[file: a.pdf]`.
+- **The upstream idle limit.** `LLM_SWITCHER_UPSTREAM_IDLE_MS` must be a positive number. The proxy uses 300000 for any other value, and it writes a warning. A value above 2147483647 becomes 2147483647. Before 1.6.17, a negative, infinite or too large value ended every stream after 1 ms. The limit counts from the response headers. It applies to the body only, and an SSE comment line resets it.
+
 ## Release 1.6.16 — 2026-10-13
 
 - **A silent upstream stream.** The proxy now ends an upstream stream that sends no byte for 5 minutes, with an error. Before, only the default limit of the Node HTTP client stopped it. Set `LLM_SWITCHER_UPSTREAM_IDLE_MS` to change the limit.

package/formats.mjs

@@ -697,9 +697,22 @@ function responsesOutputImages(out) {
 // A file or document block carries its bytes as base64; stringified into text it costs millions of tokens
 // (The-NeXT-AI/ai-gateway#28), so it is named instead. Null for any other block.
 function mediaPlaceholder(b) {
-  if (b?.type === 'input_file') return `[file: ${b.filename || b.file_id || 'attachment'}]`;
-  if (b?.type === 'document') return `[document${b.title ? `: ${b.title}` : ''}]`;
-  if (b?.source?.type === 'base64' || typeof b?.file_data === 'string') return `[${b.type || 'attachment'}]`;
+  const src = b?.source;
+  if (src?.type === 'base64' || typeof b?.file_data === 'string') {
+    if (b.type === 'input_file') return `[file: ${b.filename || b.file_id || 'attachment'}]`;
+    if (b.type === 'document') return `[document${b.title ? `: ${b.title}` : ''}]`;
+    return `[${b.type || 'attachment'}]`;
+  }
+  // Not base64: the content itself goes on, as text the model can read.
+  if (b?.type === 'document' && src?.type === 'text' && typeof src.data === 'string') return src.data;
+  if (b?.type === 'document' && src?.type === 'content') {
+    if (typeof src.content === 'string') return src.content;
+    if (Array.isArray(src.content)) {
+      return src.content.map(c => (c?.type === 'text' ? c.text || '' : c?.type === 'image' ? '[image omitted]' : '')).filter(Boolean).join('\n');
+    }
+  }
+  if (b?.type === 'document' && src?.type === 'url' && src.url) return `[document: ${src.url}]`;
+  if (b?.type === 'input_file' && b.file_url) return `[file: ${b.file_url}]`;
   return null;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llm-switcher",
-  "version": "1.6.16",
+  "version": "1.6.17",
   "description": "Zero-dependency multi-protocol edge gateway & provider switcher for Claude Code and Codex, with OpenAI-compatible, Anthropic and Vertex upstreams",
   "keywords": [
     "llm",

package/tests/formats.test.mjs

@@ -1754,3 +1754,91 @@ test('irToAnthropicBody: a whitespace-only text block is blank too', () => {
   const body = irToAnthropicBody(ir, 'claude');
   assert.ok(body.messages.every(m => m.content.every(b => b.type !== 'text' || b.text.trim())), JSON.stringify(body.messages));
 });
+
+// ---- roundtable ruling 2026-10-13: media in tool results, held notices ----
+
+const BLOB = 'QUFB'.repeat(2000);
+const MISSING_TOOL_RESULT = '[Tool result unavailable: it was removed from the conversation history]';
+function anthropicToolText(content) {
+  const ir = anthropicToIR({ model: 'x', messages: [
+    { role: 'assistant', content: [{ type: 'tool_use', id: 't1', name: 'read', input: {} }] },
+    { role: 'user', content: [{ type: 'tool_result', tool_use_id: 't1', content }] }
+  ] });
+  return ir.messages.find(m => m.role === 'tool').content;
+}
+
+test('a tool-result document keeps its text, its URL and its label; only base64 becomes a placeholder', () => {
+  assert.match(anthropicToolText([{ type: 'document', source: { type: 'text', media_type: 'text/plain', data: 'hello doc' } }]), /hello doc/);
+  const content = anthropicToolText([{ type: 'document', source: { type: 'content', content: [
+    { type: 'text', text: 'part one' }, { type: 'image', source: { type: 'base64', media_type: 'image/png', data: BLOB } }] } }]);
+  assert.match(content, /part one/);
+  assert.match(content, /\[image omitted\]/);
+  assert.ok(!content.includes(BLOB));
+  assert.equal(anthropicToolText([{ type: 'document', source: { type: 'content', content: 'plain string' } }]), 'plain string');
+  assert.match(anthropicToolText([{ type: 'document', source: { type: 'url', url: 'https://example.com/a.pdf' } }]), /https:\/\/example\.com\/a\.pdf/);
+  assert.match(anthropicToolText([{ type: 'input_file', file_url: 'https://example.com/f.pdf' }]), /https:\/\/example\.com\/f\.pdf/);
+  assert.equal(anthropicToolText([{ type: 'input_file', filename: 'a.pdf', file_data: `data:application/pdf;base64,${BLOB}` }]), '[file: a.pdf]');
+  assert.equal(anthropicToolText([{ type: 'document', title: 'T', source: { type: 'base64', media_type: 'application/pdf', data: BLOB } }]), '[document: T]');
+  assert.ok(!anthropicToolText([{ type: 'document', source: { type: 'url' } }]).includes('[document: undefined]'));
+});
+
+test('an orphan tool result turned into text keeps a text document', () => {
+  const { payload } = healAnthropicPayload({ model: 'x', max_tokens: 10, messages: [
+    { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'orphan', content: [{ type: 'document', source: { type: 'text', data: 'hello doc' } }] }] }
+  ] });
+  assert.match(JSON.stringify(payload.messages), /hello doc/);
+});
+
+function chatShape(input) {
+  return irToChatBody(responsesToIR({ model: 'x', input }), 'gpt-4o').messages
+    .map(m => (m.role === 'tool' ? `tool:${m.content}` : m.role === 'user' ? `user:${typeof m.content === 'string' ? m.content : JSON.stringify(m.content)}` : m.role));
+}
+const callAB = [
+  { type: 'function_call', call_id: 'c1', name: 'a', arguments: '{}' },
+  { type: 'function_call', call_id: 'c2', name: 'b', arguments: '{}' }
+];
+
+test('healToolPairs: a held notice follows the placeholder of an output that never came', () => {
+  const missing = chatShape([...callAB, { type: 'function_call_output', call_id: 'c1', output: 'out1' }, { role: 'user', content: 'notice' }]);
+  assert.deepEqual(missing.slice(-4), ['assistant', 'tool:out1', `tool:${MISSING_TOOL_RESULT}`, 'user:notice']);
+  const later = chatShape([...callAB, { type: 'function_call_output', call_id: 'c1', output: 'out1' }, { role: 'user', content: 'notice' },
+    { type: 'message', role: 'assistant', content: [{ type: 'output_text', text: 'next turn' }] }]);
+  assert.deepEqual(later.slice(-2), ['user:notice', 'assistant']);
+  const two = chatShape([...callAB, { role: 'user', content: 'notice1' }, { type: 'function_call_output', call_id: 'c1', output: 'out1' }, { role: 'user', content: 'notice2' }]);
+  assert.deepEqual(two.slice(-5), ['assistant', 'tool:out1', `tool:${MISSING_TOOL_RESULT}`, 'user:notice1', 'user:notice2']);
+});
+
+test('placeholders and URLs are exact on every route that turns a tool result into text', () => {
+  assert.equal(anthropicToolText([{ type: 'document', source: { type: 'base64', media_type: 'application/pdf', data: BLOB } }]), '[document]');
+  assert.equal(anthropicToolText([{ type: 'document', source: { type: 'url', url: 'https://example.com/a.pdf' } }]), '[document: https://example.com/a.pdf]');
+  assert.equal(anthropicToolText([{ type: 'input_file', file_url: 'https://example.com/f.pdf' }]), '[file: https://example.com/f.pdf]');
+  const orphanText = (content) => {
+    const { payload } = healAnthropicPayload({ model: 'x', max_tokens: 10, messages: [
+      { role: 'user', content: [{ type: 'tool_result', tool_use_id: 'orphan', content }] }] });
+    return JSON.stringify(payload.messages);
+  };
+  const doc = orphanText([{ type: 'document', title: 'T', source: { type: 'base64', media_type: 'application/pdf', data: BLOB } }]);
+  assert.ok(!doc.includes(BLOB) && doc.includes('[document: T]'), doc.slice(0, 300));
+  const file = orphanText([{ type: 'input_file', filename: 'a.pdf', file_data: `data:application/pdf;base64,${BLOB}` }]);
+  assert.ok(!file.includes(BLOB) && file.includes('[file: a.pdf]'), file.slice(0, 300));
+});
+
+test('the image of a Responses tool output reaches the chat and Anthropic bodies', () => {
+  for (const image_url of ['data:image/png;base64,AAAA', { url: 'data:image/png;base64,AAAA' }]) {
+    const ir = responsesToIR({ model: 'x', input: [
+      { type: 'function_call', call_id: 'c1', name: 'shot', arguments: '{}' },
+      { type: 'function_call_output', call_id: 'c1', output: [{ type: 'input_image', image_url }] }] });
+    const chat = JSON.stringify(irToChatBody(ir, 'gpt-4o').messages);
+    assert.match(chat, /data:image\/png;base64,AAAA/);
+    const anth = JSON.stringify(irToAnthropicBody(ir, 'claude-x').messages);
+    assert.match(anth, /"type":"image"/);
+  }
+});
+
+test('healToolPairs: an orphan output and a held notice in one batch come out orphan first, then the notice', () => {
+  const shape = chatShape([...callAB,
+    { type: 'function_call_output', call_id: 'zz', output: 'stray' },
+    { role: 'user', content: 'notice' },
+    { type: 'function_call_output', call_id: 'c1', output: 'out1' }]);
+  assert.deepEqual(shape.slice(-5), ['assistant', 'tool:out1', `tool:${MISSING_TOOL_RESULT}`, 'user:[Tool Result (zz)]: stray', 'user:notice']);
+});

package/tests/upstream-stream.test.mjs

@@ -1,6 +1,6 @@
 import { test } from 'node:test';
 import assert from 'node:assert/strict';
-import { readUpstreamPayloads } from '../upstream-stream.mjs';
+import { readUpstreamPayloads, parseIdleMs } from '../upstream-stream.mjs';
 
 async function collect(text, widths = [1, 3, 7]) {
   const bytes = Buffer.from(text); let offset = 0, i = 0;
@@ -47,3 +47,25 @@ test('an upstream that goes silent mid-stream ends with an error after the idle
   assert.deepEqual(values, [{ a: 1 }]);
   assert.ok(cancelled, 'the silent upstream body is cancelled');
 });
+
+test('parseIdleMs keeps the idle limit inside what setTimeout accepts', () => {
+  // Node fires a timer of a negative, infinite or > 2^31-1 delay after 1 ms.
+  assert.equal(parseIdleMs('3000000000'), 2147483647);
+  for (const bad of ['-5', 'Infinity', 'abc', '0', '0.5']) assert.equal(parseIdleMs(bad), 300000, bad);
+  assert.equal(parseIdleMs('120000'), 120000);
+  assert.equal(parseIdleMs(undefined), 300000);
+});
+
+test('the idle limit counts the gap between reads, not the whole stream', async () => {
+  let n = 0;
+  const body = new ReadableStream({
+    async pull(controller) {
+      if (n === 5) return controller.close();
+      await new Promise(r => setTimeout(r, 60));
+      controller.enqueue(Buffer.from(`data: {"n":${n++}}\n\n`));
+    }
+  });
+  const values = [];
+  for await (const v of readUpstreamPayloads({ body }, { idleMs: 200 })) values.push(v.n);
+  assert.deepEqual(values, [0, 1, 2, 3, 4]);
+});

package/upstream-stream.mjs

@@ -3,11 +3,25 @@
 // `state.done` turns true on an OpenAI [DONE] line, which ends a stream cleanly even without a finish reason.
 // A stream that sends no byte for `state.idleMs` ends with an error: a dead HTTP/2 session can hold it open
 // with no bytes, and the default limit of the HTTP client is not this module's to rely on.
-export const UPSTREAM_IDLE_MS = Number(process.env.LLM_SWITCHER_UPSTREAM_IDLE_MS) || 300000;
+const DEFAULT_IDLE_MS = 300000;
+const MAX_TIMER_MS = 2147483647;
+
+// The idle limit as setTimeout takes it: Node fires a negative, infinite or larger delay after 1 ms.
+export function parseIdleMs(value) {
+  if (value === undefined || value === null || value === '') return DEFAULT_IDLE_MS;
+  const n = Number(value);
+  if (!Number.isFinite(n) || n < 1) {
+    console.warn(`[llm-switcher] upstream idle limit ${JSON.stringify(value)} is not a positive number; using ${DEFAULT_IDLE_MS} ms`);
+    return DEFAULT_IDLE_MS;
+  }
+  return Math.min(n, MAX_TIMER_MS);
+}
+
+export const UPSTREAM_IDLE_MS = parseIdleMs(process.env.LLM_SWITCHER_UPSTREAM_IDLE_MS);
 
 export async function* readUpstreamPayloads(upstreamRes, state = {}) {
   const reader = upstreamRes.body.getReader();
-  const idleMs = state.idleMs || UPSTREAM_IDLE_MS;
+  const idleMs = state.idleMs === undefined ? UPSTREAM_IDLE_MS : parseIdleMs(state.idleMs);
   const read = () => {
     let timer;
     const idle = new Promise((_, reject) => {