llm-switcher 1.1.11 → 1.2.0

package/tests/blindfold.test.mjs

@@ -50,11 +50,16 @@ test('a real Codex API path is routed, and its query string survives', () => {
   assert.equal(toGatewayPath('/backend-api/codex/responses'), `${GATEWAY_PREFIX}/responses`);
 });
 
-// A CONNECT to any other host must be tunneled, not intercepted: the process then
-// only copies bytes and never holds that host's plaintext.
-test('only the target host is intercepted; every other public host is tunneled', () => {
-  assert.equal(isInterceptedHost('chatgpt.com'), true);
-  for (const other of ['api.openai.com', 'auth.openai.com', 'example.com', 'chatgpt.com.evil.test']) {
+// A CONNECT to any host outside the table must be tunneled, not intercepted: the process then
+// only copies bytes and never holds that host's plaintext. The table of R3 names exactly three
+// hosts, and it is an exact lookup — a name that merely ends in one of them is not in it.
+test('the host table names exactly three hosts; every other public host is tunneled', () => {
+  for (const host of ['api.anthropic.com', 'api.openai.com', 'chatgpt.com']) {
+    assert.equal(isInterceptedHost(host), true, `must intercept ${host}`);
+    assert.equal(isInterceptedHost(host.toUpperCase()), true, 'DNS case never matters');
+  }
+  for (const other of ['auth.openai.com', 'api.chatgpt.com', 'openai.com', 'example.com',
+    'chatgpt.com.evil.test', 'anthropic.com']) {
     assert.equal(isInterceptedHost(other), false, `must not intercept ${other}`);
   }
 });

package/tests/catalog.test.mjs

@@ -0,0 +1,147 @@
+// Unit tests for catalog.mjs (dynamic model discovery, caching and auto-role classification)
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import http from 'node:http';
+import {
+  classifyClaudeTier,
+  classifyCodexRole,
+  loadCatalogCache,
+  saveCatalogCache,
+  fetchToolModels,
+  refreshCatalog,
+  detectToolVersion,
+  checkVersionAndRefresh,
+  BASELINE_MODELS
+} from '../catalog.mjs';
+
+test('classifyClaudeTier categorizes all Claude models into correct tiers', () => {
+  assert.equal(classifyClaudeTier('claude-opus-5-5'), 'opus');
+  assert.equal(classifyClaudeTier('claude-opus-4-6'), 'opus');
+  assert.equal(classifyClaudeTier('claude-3-opus-20240229'), 'opus');
+
+  assert.equal(classifyClaudeTier('claude-sonnet-4'), 'sonnet');
+  assert.equal(classifyClaudeTier('claude-3-7-sonnet-20250219'), 'sonnet');
+  assert.equal(classifyClaudeTier('claude-3-5-sonnet-20241022'), 'sonnet');
+
+  assert.equal(classifyClaudeTier('claude-haiku-4'), 'haiku');
+  assert.equal(classifyClaudeTier('claude-3-5-haiku-20241022'), 'haiku');
+
+  assert.equal(classifyClaudeTier('claude-fable-4'), 'fable');
+  assert.equal(classifyClaudeTier('custom-model'), 'sonnet', 'defaults unknown Claude models to sonnet tier');
+});
+
+test('classifyCodexRole categorizes all Codex models into correct roles', () => {
+  assert.equal(classifyCodexRole('gpt-6-sol'), 'main');
+  assert.equal(classifyCodexRole('gpt-5.6-sol'), 'main');
+  assert.equal(classifyCodexRole('gpt-5.2'), 'main');
+  assert.equal(classifyCodexRole('o3'), 'main');
+
+  assert.equal(classifyCodexRole('gpt-6-terra'), 'review');
+  assert.equal(classifyCodexRole('gpt-5.6-terra'), 'review');
+  assert.equal(classifyCodexRole('codex-review-model'), 'review');
+
+  assert.equal(classifyCodexRole('gpt-6-luna'), 'subagent');
+  assert.equal(classifyCodexRole('gpt-5.6-luna'), 'subagent');
+  assert.equal(classifyCodexRole('codex-subagent-worker'), 'subagent');
+});
+
+test('loadCatalogCache returns baseline models when cache file does not exist', () => {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'llmsw-cat-test-'));
+  try {
+    const catalog = loadCatalogCache(tmp);
+    assert.ok(catalog.claude.models.length > 0);
+    assert.ok(catalog.codex.models.length > 0);
+    assert.ok(catalog.claude.models.some(m => m.id === 'claude-opus-5-5'));
+    assert.ok(catalog.codex.models.some(m => m.id === 'gpt-6-sol'));
+  } finally {
+    fs.rmSync(tmp, { recursive: true, force: true });
+  }
+});
+
+test('saveCatalogCache writes cache atomically and loadCatalogCache reads it back', () => {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'llmsw-cat-test-'));
+  try {
+    const sample = {
+      updatedAt: 123456789,
+      claude: { models: [{ id: 'claude-future-opus', tier: 'opus' }] },
+      codex: { models: [{ id: 'gpt-7-sol', role: 'main' }] }
+    };
+    const saved = saveCatalogCache(tmp, sample);
+    assert.equal(saved, true);
+    const loaded = loadCatalogCache(tmp);
+    assert.equal(loaded.updatedAt, 123456789);
+    assert.equal(loaded.claude.models[0].id, 'claude-future-opus');
+    assert.equal(loaded.codex.models[0].id, 'gpt-7-sol');
+  } finally {
+    fs.rmSync(tmp, { recursive: true, force: true });
+  }
+});
+
+test('fetchToolModels queries endpoint and parses model list', async () => {
+  const server = http.createServer((req, res) => {
+    if (req.url === '/v1/models') {
+      res.writeHead(200, { 'Content-Type': 'application/json' });
+      res.end(JSON.stringify({
+        data: [
+          { id: 'gpt-6-sol', object: 'model' },
+          { id: 'gpt-6-terra', object: 'model' },
+          { id: 'gpt-6-luna', object: 'model' }
+        ]
+      }));
+      return;
+    }
+    res.writeHead(404);
+    res.end('{}');
+  });
+
+  const port = await new Promise(r => server.listen(0, '127.0.0.1', () => r(server.address().port)));
+  try {
+    const res = await fetchToolModels('codex', { url: `http://127.0.0.1:${port}/v1/models` });
+    assert.equal(res.ok, true);
+    assert.equal(res.models.length, 3);
+    assert.equal(res.models.find(m => m.id === 'gpt-6-sol').role, 'main');
+    assert.equal(res.models.find(m => m.id === 'gpt-6-terra').role, 'review');
+    assert.equal(res.models.find(m => m.id === 'gpt-6-luna').role, 'subagent');
+  } finally {
+    server.close();
+  }
+});
+
+test('fetchToolModels handles network errors gracefully without throwing', async () => {
+  const res = await fetchToolModels('claude', { url: 'http://127.0.0.1:1/nonexistent', timeout: 50 });
+  assert.equal(res.ok, false);
+  assert.ok(res.error, 'reports error reason without crashing');
+});
+
+test('detectToolVersion extracts semantic version from various User-Agent strings', () => {
+  assert.equal(detectToolVersion({ 'user-agent': 'claude-cli/2.1.280 (external, cli)' }), '2.1.280');
+  assert.equal(detectToolVersion({ 'user-agent': 'codex-cli/0.157.1 (Windows NT 10.0; Win64; x64)' }), '0.157.1');
+  assert.equal(detectToolVersion({ 'user-agent': 'claude-code/2.2.0 darwin' }), '2.2.0');
+  assert.equal(detectToolVersion({ 'user-agent': 'curl/7.68.0' }), '');
+  assert.equal(detectToolVersion({}), '');
+});
+
+test('checkVersionAndRefresh records new version and triggers update only on version bumps', async () => {
+  const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'llmsw-bump-test-'));
+  try {
+    // Initial request from version 1.1.0
+    checkVersionAndRefresh('claude', { 'user-agent': 'claude-cli/1.1.0' }, tmp);
+    const cat1 = loadCatalogCache(tmp);
+    assert.equal(cat1.claude.lastSeenVersion, '1.1.0');
+
+    // Repeated request from the same version 1.1.0 -> no change, no duplicate trigger
+    checkVersionAndRefresh('claude', { 'user-agent': 'claude-cli/1.1.0' }, tmp);
+    const cat2 = loadCatalogCache(tmp);
+    assert.equal(cat2.claude.lastSeenVersion, '1.1.0');
+
+    // Tool updates to version 1.2.0 -> version bump detected and saved
+    checkVersionAndRefresh('claude', { 'user-agent': 'claude-cli/1.2.0' }, tmp);
+    const cat3 = loadCatalogCache(tmp);
+    assert.equal(cat3.claude.lastSeenVersion, '1.2.0');
+  } finally {
+    fs.rmSync(tmp, { recursive: true, force: true });
+  }
+});

package/tests/contract-lab.test.mjs

@@ -10,7 +10,7 @@ import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
 import { spawn, execFileSync } from 'node:child_process';
-import { fileURLToPath } from 'node:url';
+import { fileURLToPath, pathToFileURL } from 'node:url';
 import { createFrameReader } from '../blindfold/wsframe.mjs';
 import {
   createContractLab, createHalfTap, newTraceId, switcherVersion, toolVersionFromUA, capJson,
@@ -85,7 +85,7 @@ test('contractLab is absent by default, and a saved block survives a rewrite', (
   fs.writeFileSync(cfgPath, JSON.stringify(cfg));
   const out = JSON.parse(spawnNodeSync(`
     import fs from 'node:fs';
-    import { loadConfig, saveConfig, contractLabSettings, redactConfig } from '${ROOT}/state.mjs';
+    import { loadConfig, saveConfig, contractLabSettings, redactConfig } from '${pathToFileURL(path.join(ROOT, 'state.mjs')).href}';
     saveConfig({ ...loadConfig(), debug: true });
     const again = JSON.parse(fs.readFileSync(process.env.LLM_SWITCHER_CONFIG, 'utf8'));
     console.log(JSON.stringify({ settings: contractLabSettings(again), masked: redactConfig(again).contractLab }));
@@ -312,11 +312,13 @@ function startIntact() {
 
 async function startProxy(name, contractLab) {
   const port = await freePort();
+  const bfPort = await freePort();
   const dir = path.join(tmpDir, name);
   fs.mkdirSync(dir, { recursive: true });
   const models = { opus: 'up-opus', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' };
   const cfg = {
     port,
+    blindfold: { port: bfPort },
     activeProfiles: { anthropic: 'ant', responses: 'ant', 'openai-chat': 'ant', vertex: 'ant' },
     profiles: {
       ant: { name: 'Mock Anthropic', mode: 'direct', inFormat: 'auto', outFormat: 'anthropic', baseURL: `http://127.0.0.1:${upstreamPort}/ant`, apiKey: 'sk-secret-ant', defaultModels: models }
@@ -365,7 +367,16 @@ before(async () => {
 });
 
 after(() => {
-  for (const p of Object.values(proxies)) p.child?.kill();
+  for (const [name, p] of Object.entries(proxies)) {
+    try {
+      const bfState = JSON.parse(fs.readFileSync(path.join(tmpDir, name, 'blindfold.json'), 'utf8'));
+      if (bfState?.pid) {
+        if (process.platform === 'win32') execFileSync('taskkill', ['/F', '/PID', String(bfState.pid)], { stdio: 'ignore' });
+        else process.kill(bfState.pid, 'SIGTERM');
+      }
+    } catch {}
+    p.child?.kill();
+  }
   upstream?.close();
   intact?.close();
   if (tmpDir) fs.rmSync(tmpDir, { recursive: true, force: true });
@@ -459,10 +470,12 @@ test('failed exchanges upload no half: 400 upstream and mid-stream error are ign
 });
 
 test('a converted stream cut mid-way uploads no half, a complete one does', async () => {
-  const chat = (text) => fetch(`http://127.0.0.1:${proxies.on.port}/v1/chat/completions`, {
+  // R5: /v1/chat/completions no longer exists. A converted stream is now an Anthropic request
+  // whose upstream speaks the other protocol, which is the same conversion by another door.
+  const chat = (text) => fetch(`http://127.0.0.1:${proxies.on.port}/v1/messages`, {
     method: 'POST',
     headers: { 'Content-Type': 'application/json', 'user-agent': 'codex/1.0.0' },
-    body: JSON.stringify({ model: 'claude-opus-4-6', stream: true, messages: [{ role: 'user', content: text }] })
+    body: JSON.stringify({ model: 'claude-opus-4-6', max_tokens: 16, stream: true, messages: [{ role: 'user', content: text }] })
   });
   let traceOk;
   await waitFor(async () => {
@@ -636,10 +649,12 @@ test('Anthropic probe with thinking sets max_tokens greater than budget_tokens',
 
 test('the probe models are every mapped model of the active profiles, once each', () => {
   const cfg = {
-    activeProfiles: { anthropic: 'a', responses: 'a', 'openai-chat': 'b', vertex: null },
+    // R5: `openai-chat` and `vertex` are retired pointer keys. They name a real profile here on
+    // purpose - a retired key must not activate anything, so `never-probed` has to stay unprobed.
+    activeProfiles: { claude: 'a', codex: 'b', 'openai-chat': 'unused', vertex: null },
     profiles: {
       a: { inFormat: 'auto', defaultModels: { opus: 'up-opus', sonnet: 'up-sonnet', haiku: '', fable: 'up-opus' } },
-      b: { inFormat: 'openai-chat', defaultModels: { default: 'up-chat' } },
+      b: { inFormat: 'auto', defaultModels: { opus: 'up-chat' } },
       unused: { inFormat: 'auto', defaultModels: { opus: 'never-probed' } }
     }
   };

package/tests/formats.test.mjs

@@ -2,7 +2,7 @@
 import { test } from 'node:test';
 import assert from 'node:assert/strict';
 import {
-  anthropicToIR, chatToIR, responsesToIR, vertexToIR,
+  anthropicToIR, responsesToIR,
   healToolPairs, irToChatBody, irToAnthropicBody, irToVertexBody, toGeminiSchema,
   createUpstreamNormalizer, createCollector, createThinkTagSplitter,
   createAnthropicStream, createResponsesStream, createVertexStream,
@@ -62,11 +62,6 @@ test('healAnthropicPayload: native Anthropic passthrough keeps the billing heade
   assert.equal(payload.system[0].text, header);
 });
 
-test('chatToIR: reasoning_effort "none" disables thinking', () => {
-  const ir = chatToIR({ model: 'x', messages: [{ role: 'user', content: 'hi' }], reasoning_effort: 'none' });
-  assert.equal(ir.thinking.type, 'disabled');
-});
-
 test('healToolPairs: orphan result -> user text, missing result -> placeholder, adjacency kept', () => {
   const healed = healToolPairs([
     { role: 'user', content: 'start' },
@@ -118,21 +113,6 @@ test('responsesToIR: parallel function_call items merge into one assistant turn;
   assert.equal(messages[2].tool_calls.length, 2);
 });
 
-test('vertexToIR: functionCall/functionResponse are paired by generated ids', () => {
-  const ir = vertexToIR({
-    contents: [
-      { role: 'user', parts: [{ text: 'weather?' }] },
-      { role: 'model', parts: [{ functionCall: { name: 'w', args: { c: 'A' } } }, { functionCall: { name: 'w', args: { c: 'B' } } }] },
-      { role: 'function', parts: [{ functionResponse: { name: 'w', response: { t: 1 } } }, { functionResponse: { name: 'w', response: { t: 2 } } }] }
-    ]
-  });
-  const calls = ir.messages[1].toolCalls.map(t => t.id);
-  const results = ir.messages.filter(m => m.role === 'tool').map(m => m.toolCallId);
-  assert.deepEqual(results, calls);
-  const { messages } = irToChatBody(ir, 'gpt-4o');
-  assert.equal(messages.filter(m => m.role === 'tool').length, 2);
-});
-
 test('irToVertexBody: functionResponse uses the function name and parallel responses share one content', () => {
   const ir = anthropicToIR({
     model: 'x', messages: [
@@ -164,19 +144,19 @@ test('toGeminiSchema strips unsupported JSON Schema keywords', () => {
 });
 
 test('irToAnthropicBody: thinking constraints (budget < max_tokens, no top_k, temperature 1)', () => {
-  const ir = chatToIR({ model: 'x', messages: [{ role: 'user', content: 'hi' }], max_tokens: 3000, temperature: 0.2, reasoning_effort: 'high' });
+  const ir = anthropicToIR({ model: 'x', max_tokens: 3000, temperature: 0.2, messages: [{ role: 'user', content: 'hi' }], thinking: { type: 'enabled', budget_tokens: 2048 } });
   ir.params.topK = 5;
   const body = irToAnthropicBody(ir, 'claude-opus-4-6');
   assert.ok(body.thinking.budget_tokens < body.max_tokens);
   assert.equal(body.temperature, undefined);
   assert.equal(body.top_k, undefined);
 
-  const small = irToAnthropicBody(chatToIR({ model: 'x', messages: [{ role: 'user', content: 'hi' }], max_tokens: 100, reasoning_effort: 'high' }), 'claude');
+  const small = irToAnthropicBody(anthropicToIR({ model: 'x', max_tokens: 100, messages: [{ role: 'user', content: 'hi' }], thinking: { type: 'enabled', budget_tokens: 2048 } }), 'claude');
   assert.equal(small.thinking, undefined, 'thinking dropped when max_tokens <= 1024');
 });
 
 test('irToAnthropicBody: never emits empty text blocks', () => {
-  const ir = chatToIR({ model: 'x', messages: [{ role: 'user', content: [{ type: 'text', text: '' }, { type: 'text', text: 'hi' }] }] });
+  const ir = anthropicToIR({ model: 'x', max_tokens: 10, messages: [{ role: 'user', content: [{ type: 'text', text: '' }, { type: 'text', text: 'hi' }] }] });
   const body = irToAnthropicBody(ir, 'claude');
   assert.ok(body.messages.every(m => m.content.every(b => b.type !== 'text' || b.text)));
 });
@@ -699,26 +679,16 @@ test('Responses tool_choice: allowed_tools restricts the tools and keeps its mod
   assert.deepEqual(responsesToIR({ ...base, tool_choice: { type: 'function', name: 'b' } }).toolChoice, { name: 'b' });
 });
 
-test('Chat tool_choice: allowed_tools restricts the tools and keeps its mode', () => {
-  const tools = ['a', 'b'].map(name => ({ type: 'function', function: { name, parameters: { type: 'object', properties: {} } } }));
-  const ir = chatToIR({ model: 'm', messages: [{ role: 'user', content: 'x' }], tools,
-    tool_choice: { type: 'allowed_tools', allowed_tools: { mode: 'auto', tools: [{ type: 'function', function: { name: 'b' } }] } } });
-  assert.deepEqual(ir.tools.map(t => t.name), ['b']);
-  assert.equal(ir.toolChoice, 'auto');
-});
-
 test('smartText and smartReasoning return each string once', () => {
   assert.equal(smartText({ content: ['line 1', 'line 2'] }), 'line 1\nline 2');
   assert.equal(smartReasoning({ reasoning_content: 'step 1', parts: [{ text: 'step 1', thought: true }] }).text, 'step 1');
 });
 
-test('an empty stop string is dropped in every input format', () => {
+test('an empty stop string is dropped, and stop reaches the body only when it has values', () => {
   const msgs = [{ role: 'user', content: 'x' }];
-  assert.deepEqual(chatToIR({ model: 'm', messages: msgs, stop: '' }).params.stop, []);
-  assert.deepEqual(chatToIR({ model: 'm', messages: msgs, stop: ['', 'END'] }).params.stop, ['END']);
+  assert.deepEqual(anthropicToIR({ model: 'm', max_tokens: 10, messages: msgs, stop_sequences: [''] }).params.stop, []);
   assert.deepEqual(anthropicToIR({ model: 'm', max_tokens: 10, messages: msgs, stop_sequences: ['', 'END'] }).params.stop, ['END']);
-  assert.deepEqual(vertexToIR({ contents: [{ role: 'user', parts: [{ text: 'x' }] }], generationConfig: { stopSequences: [''] } }).params.stop, []);
-  const body = irToAnthropicBody(chatToIR({ model: 'm', messages: msgs, stop: '' }), 'claude-x');
+  const body = irToAnthropicBody(anthropicToIR({ model: 'm', max_tokens: 10, messages: msgs, stop_sequences: [''] }), 'claude-x');
   assert.equal(body.stop_sequences, undefined);
 });
 
@@ -736,7 +706,7 @@ test('Vertex usage: thoughts count as output, and the Vertex reply splits them b
 });
 
 test('reasoning model families get no <think> guide', () => {
-  const ir = chatToIR({ model: 'm', messages: [{ role: 'user', content: 'x' }], reasoning_effort: 'high' });
+  const ir = anthropicToIR({ model: 'm', max_tokens: 100, messages: [{ role: 'user', content: 'x' }], thinking: { type: 'enabled', budget_tokens: 2048 } });
   for (const model of ['o3-mini', 'openai/o1', 'o4-mini-high', 'deepseek-r1', 'deepseek-reasoner', 'qwq-32b']) {
     const sys = irToChatBody(ir, model).messages.find(m => m.role === 'system')?.content || '';
     assert.ok(!sys.includes('<think>'), `${model} must not get the guide`);
@@ -781,15 +751,10 @@ test('healAnthropicPayload keeps an unchanged user turn as the same object', ()
 });
 
 test('allowed_tools that matches no declared tool gives a request without tools, not a crash', () => {
-  const chat = chatToIR({ model: 'm', messages: [{ role: 'user', content: 'x' }],
-    tools: [{ type: 'function', function: { name: 'a', parameters: { type: 'object', properties: {} } } }],
-    tool_choice: { type: 'allowed_tools', allowed_tools: { mode: 'auto', tools: [{ type: 'function', function: { name: 'b' } }] } } });
   const hosted = responsesToIR({ model: 'm', input: 'x', tools: [fn('a')], tool_choice: { type: 'allowed_tools', mode: 'required', tools: [{ type: 'web_search' }] } });
-  for (const ir of [chat, hosted]) {
-    for (const body of [irToChatBody(ir, 'm'), irToAnthropicBody(ir, 'claude-x'), irToVertexBody(ir, 'gemini-x')]) {
-      assert.equal(body.tools, undefined);
-      assert.equal(body.tool_choice, undefined);
-    }
+  for (const body of [irToChatBody(hosted, 'm'), irToAnthropicBody(hosted, 'claude-x'), irToVertexBody(hosted, 'gemini-x')]) {
+    assert.equal(body.tools, undefined);
+    assert.equal(body.tool_choice, undefined);
   }
 });
 
@@ -815,3 +780,25 @@ test('tool schemas keep 0, false, "" and null values', () => {
   assert.equal(p.properties.tags.items.default, '');
   assert.equal(p.properties.note.default, null);
 });
+
+// Claude Code sends `advisor` with `input_schema: {}`. That survived as `parameters: {}`, so
+// Vertex failed the whole request with `tools.18.custom.input_schema.type: Field required` —
+// while the same call with a real tool list succeeded. An empty shape must be healed exactly
+// like a missing one, on the chat path a profile actually uses.
+test('an empty tool schema is healed, not forwarded as {}', () => {
+  assert.deepEqual(sanitizeJsonSchema({}), { type: 'object', properties: {} });
+  assert.deepEqual(
+    sanitizeJsonSchema({ type: 'object', properties: { nested: {} } }),
+    { type: 'object', properties: { nested: { type: 'object', properties: {} } } }
+  );
+  assert.deepEqual(toGeminiSchema({}), { type: 'object', properties: {} });
+
+  const body = irToChatBody(
+    anthropicToIR({
+      model: 'm', max_tokens: 10, messages: [{ role: 'user', content: 'x' }],
+      tools: [{ name: 'advisor', description: '', input_schema: {} }]
+    }),
+    'antigravity/claude-opus-4-6-thinking'
+  );
+  assert.deepEqual(body.tools[0].function.parameters, { type: 'object', properties: {} });
+});