llm-switcher 1.6.12 → 1.6.13

package/tests/formats.test.mjs

@@ -14,10 +14,11 @@ import {
   createCodeAssistStream,
   budgetToEffort, effortToBudget, normalizeEffort, capEffort, mapEquivalentEffort, EFFORT_LEVELS, parseArgs, buildAnthropicMessage,
   refusesImageInput, omitChatImages, IMAGE_OMITTED_TEXT,
-  stripGeminiPlaceholders, geminiPaddedTools, withGeminiPlaceholderStrip
+  stripGeminiPlaceholders, geminiPaddedTools, withGeminiPlaceholderStrip, toolVocabForIR, recordGeminiPadding
 } from '../formats.mjs';
 import { NULL_TOOL_VOCAB, createToolVocab } from '../toolvocab.mjs';
 import { assertValidAnthropicEvents } from './helpers.mjs';
+import { encodeCompaction } from '../compaction.mjs';
 
 test('anthropicToIR: stream defaults to false and thinking budget_tokens is preserved', () => {
   const ir = anthropicToIR({ model: 'claude-opus-4-6', max_tokens: 64000, messages: [{ role: 'user', content: 'hi' }], thinking: { type: 'enabled', budget_tokens: 31999 } });
@@ -111,6 +112,15 @@ test('healAnthropicPayload: native Anthropic passthrough keeps the billing heade
   assert.equal(payload.system[0].text, header);
 });
 
+test('anthropicToIR: a system turn folds into the IR system prompt', () => {
+  const ir = anthropicToIR({
+    model: 'claude-opus-4-6', max_tokens: 10,
+    messages: [{ role: 'system', content: 'You are Rem.' }, { role: 'user', content: 'hi' }]
+  });
+  assert.equal(ir.system, 'You are Rem.');
+  assert.deepEqual(ir.messages.map(m => m.role), ['user']);
+});
+
 test('healToolPairs: orphan result -> user text, missing result -> placeholder, adjacency kept', () => {
   const healed = healToolPairs([
     { role: 'user', content: 'start' },
@@ -1429,13 +1439,15 @@ test('healAnthropicPayload folds a mid-conversation system message into a user t
   const msgs = r.payload.messages;
   assert.ok(!msgs.some(m => m.role === 'system'), 'no system role remains');
   assert.ok(alternates(msgs), `roles alternate: ${roles(msgs)}`);
-  assert.deepEqual(roles(msgs), ['user', 'assistant', 'user', 'assistant']);
+  assert.deepEqual(roles(msgs), ['user', 'assistant', 'user', 'assistant', 'user']);
   assert.deepEqual(msgs[2].content, [
     { type: 'tool_result', tool_use_id: 'toolu_1', content: 'a.txt' },
-    { type: 'text', text: '[system] Use British spelling from now on.' },
-    { type: 'text', text: '[system] Be brief.' }
-  ], 'a tool_result stays first; with no user turn after it, the last text joins the previous user turn');
+    { type: 'text', text: '[system] Use British spelling from now on.' }
+  ], 'a tool_result stays first');
   assert.ok(!texts(msgs[3]).includes('[system]'));
+  assert.deepEqual(msgs[4], { role: 'user', content: [{ type: 'text', text: '[system] Be brief.' }] }, 'a system message after the last assistant turn stays after it');
+  assert.ok(r.notes.includes('folded system messages into user turns'));
+  assert.ok(!r.notes.includes('dropped system content with no text'));
 
   const only = healAnthropicPayload({ model: 'x', max_tokens: 10, messages: [
     { role: 'system', content: 'Answer in French.' },
@@ -1513,3 +1525,166 @@ test('the placeholder strip reaches a padded object inside an anyOf branch', ()
   assert.deepEqual(stripGeminiPlaceholders({ env: { A: '1', _placeholder: true } }, schema), { env: { A: '1' } });
   assert.deepEqual(stripGeminiPlaceholders({ env: 'plain' }, schema), { env: 'plain' });
 });
+
+// One anyOf branch padded, another declaring a real _placeholder: the key may be the client's own.
+test('the placeholder strip keeps _placeholder when another anyOf branch declares it', () => {
+  const ir = anthropicToIR({ model: 'x', max_tokens: 10, messages: [{ role: 'user', content: 'go' }], tools: [{ name: 'run', input_schema: {
+    type: 'object', properties: { env: { oneOf: [{ type: 'object' }, { type: 'object', properties: { _placeholder: { type: 'string' }, x: { type: 'string' } } }] } }
+  } }] });
+  const schema = geminiPaddedTools(JSON.parse(JSON.stringify(irToVertexBody(ir, 'gemini-3-pro')))).get('run');
+  assert.equal(schema?.properties?.env?.anyOf?.length, 2, 'fixture: a padded branch and a declaring branch');
+  assert.deepEqual(stripGeminiPlaceholders({ env: { _placeholder: 'keep', x: '1' } }, schema), { env: { _placeholder: 'keep', x: '1' } });
+  const lone = { anyOf: [schema.properties.env.anyOf[0], { type: 'STRING' }] };
+  assert.deepEqual(stripGeminiPlaceholders({ A: '1', _placeholder: true }, lone), { A: '1' }, 'with no declaring branch the padded branch still strips');
+});
+
+// The fold keeps each system text between the same turns it sat between, and says what it did.
+test('healAnthropicPayload keeps a folded system text in place and reports dropped system content', () => {
+  const heal = (messages) => healAnthropicPayload({ model: 'x', max_tokens: 10, messages });
+  const before = heal([
+    { role: 'user', content: 'first' },
+    { role: 'system', content: 'Be brief.' },
+    { role: 'assistant', content: 'ok' },
+    { role: 'user', content: 'second' }
+  ]).payload.messages;
+  assert.deepEqual(before, [
+    { role: 'user', content: [{ type: 'text', text: 'first' }, { type: 'text', text: '[system] Be brief.' }] },
+    { role: 'assistant', content: 'ok' },
+    { role: 'user', content: 'second' }
+  ], 'a system message before an assistant turn stays before it');
+
+  const next = heal([
+    { role: 'user', content: 'first' },
+    { role: 'assistant', content: 'ok' },
+    { role: 'system', content: 'Be brief.' },
+    { role: 'user', content: 'second' }
+  ]).payload.messages;
+  assert.deepEqual(next[2], { role: 'user', content: [{ type: 'text', text: '[system] Be brief.' }, { type: 'text', text: 'second' }] }, 'it precedes the user text that followed it');
+
+  const blank = heal([
+    { role: 'user', content: 'first' },
+    { role: 'system', content: '   ' },
+    { role: 'system', content: [{ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'AA==' } }] },
+    { role: 'assistant', content: 'ok' }
+  ]);
+  assert.equal(blank.changed, true);
+  assert.deepEqual(blank.payload.messages, [{ role: 'user', content: 'first' }, { role: 'assistant', content: 'ok' }]);
+  assert.deepEqual(blank.notes, ['dropped system content with no text'], 'nothing was folded, so the note does not say so');
+});
+
+// Only the blank or non-text blocks of a system message go; its text blocks stay, joined by a newline.
+test('healAnthropicPayload keeps the text of a partly blank system message and joins its blocks with a newline', () => {
+  const heal = (system) => healAnthropicPayload({ model: 'x', max_tokens: 10, messages: [
+    { role: 'user', content: 'first' },
+    { role: 'assistant', content: 'ok' },
+    { role: 'system', content: system },
+    { role: 'user', content: 'second' }
+  ] });
+  const joined = heal([{ type: 'text', text: 'Be brief.' }, { type: 'text', text: 'Use British spelling.' }]);
+  assert.deepEqual(joined.payload.messages[2], { role: 'user', content: [{ type: 'text', text: '[system] Be brief.\nUse British spelling.' }, { type: 'text', text: 'second' }] });
+  assert.deepEqual(joined.notes, ['folded system messages into user turns'], 'nothing was dropped');
+
+  const partial = heal([
+    { type: 'text', text: 'Be brief.' },
+    { type: 'text', text: '   ' },
+    { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'AA==' } },
+    { type: 'text', text: 'Use British spelling.' }
+  ]);
+  assert.equal(partial.changed, true);
+  assert.deepEqual(partial.payload.messages[2], { role: 'user', content: [{ type: 'text', text: '[system] Be brief.\nUse British spelling.' }, { type: 'text', text: 'second' }] }, 'the text is kept, the blank and image blocks are not');
+  assert.deepEqual(partial.notes, ['folded system messages into user turns', 'dropped system content with no text'], 'the note reports the dropped part');
+});
+
+// A compaction item replaces the turns before it, so reasoning still pending there belongs to none after it.
+test('responsesToIR drops pending reasoning at a compaction item', () => {
+  const ir = responsesToIR({ model: 'x', input: [
+    { type: 'message', role: 'user', content: 'go' },
+    { type: 'reasoning', summary: [{ type: 'summary_text', text: 'stale plan' }] },
+    { type: 'compaction', encrypted_content: encodeCompaction('earlier work') },
+    { type: 'message', role: 'assistant', content: 'done' }
+  ] });
+  const reply = ir.messages.find(m => m.role === 'assistant');
+  assert.equal(reply.content, 'done');
+  assert.equal(reply.reasoning, undefined);
+
+  const opaque = responsesToIR({ model: 'x', input: [
+    { type: 'message', role: 'user', content: 'go' },
+    { type: 'reasoning', summary: [{ type: 'summary_text', text: 'stale plan' }] },
+    { type: 'compaction', encrypted_content: 'opaque-blob' },
+    { type: 'message', role: 'assistant', content: 'done' }
+  ] }, { allowOpaqueCompaction: true });
+  assert.equal(opaque.opaqueCompaction, true, 'fixture: the opaque path');
+  assert.equal(opaque.messages.find(m => m.role === 'assistant').reasoning, undefined);
+
+  // compaction_trigger must be the last item, so nothing after it can take the reasoning.
+  const trigger = responsesToIR({ model: 'x', input: [
+    { type: 'message', role: 'user', content: 'go' },
+    { type: 'reasoning', summary: [{ type: 'summary_text', text: 'stale plan' }] },
+    { type: 'compaction_trigger' }
+  ] });
+  assert.equal(trigger.compaction, true);
+  assert.ok(!trigger.messages.some(m => m.reasoning), 'no turn carries the pending reasoning');
+});
+
+test('responsesToIR joins a second reasoning item onto an assistant turn that already has reasoning', () => {
+  const ir = responsesToIR({ model: 'x', input: [
+    { type: 'message', role: 'user', content: 'go' },
+    { type: 'reasoning', summary: [{ type: 'summary_text', text: 'first' }] },
+    { type: 'message', role: 'assistant', content: 'looking' },
+    { type: 'reasoning', summary: [{ type: 'summary_text', text: 'second' }] },
+    { type: 'function_call', call_id: 'c1', name: 'shell', arguments: '{}' }
+  ] });
+  const turns = ir.messages.filter(m => m.role === 'assistant');
+  assert.equal(turns.length, 1, 'fixture: the call joins the assistant turn');
+  assert.equal(turns[0].reasoning, 'first\nsecond');
+  assert.equal(turns[0].toolCalls.length, 1);
+});
+
+// A retry or fallback can emit the same IR again for another upstream; each emit gets its own strip.
+test('a rebuild for another upstream format does not reuse the tool vocabulary of the first', () => {
+  const schema = { type: 'object', properties: { env: { type: 'object' } } };
+  const ir = anthropicToIR({ model: 'x', max_tokens: 10, messages: [{ role: 'user', content: 'go' }], tools: [{ name: 'my-tool', input_schema: schema }, { name: 'run', input_schema: schema }] });
+  const filled = '{"env":{"A":"1","_placeholder":true}}';
+
+  recordGeminiPadding(ir, irToChatBody(ir, 'gpt-5'), false);
+  assert.equal(toolVocabForIR(ir, 'openai-chat').args('my-tool', filled), filled, 'a chat body padded nothing');
+  recordGeminiPadding(ir, JSON.parse(JSON.stringify(irToVertexBody(ir, 'gemini-3-pro'))), true);
+  const vertex = toolVocabForIR(ir, 'vertex');
+  assert.equal(vertex.name('my_tool'), 'my-tool', 'the Vertex name maps back');
+  assert.deepEqual(JSON.parse(vertex.args('my_tool', filled)), { env: { A: '1' } }, 'the Vertex body padded env');
+
+  assert.deepEqual(JSON.parse(vertex.args('run', filled)), { env: { A: '1' } });
+
+  recordGeminiPadding(ir, irToChatBody(ir, 'gpt-5'), false);
+  assert.equal(toolVocabForIR(ir, 'openai-chat').args('run', filled), filled, 'the padding of the Vertex emit is gone');
+});
+
+// Two padded upstream tools that one client name maps back to: neither schema may strip for that name.
+test('the placeholder strip refuses a client name that two padded tools share', () => {
+  const padded = geminiPaddedTools(JSON.parse(JSON.stringify(irToVertexBody(anthropicToIR({ model: 'x', max_tokens: 10, messages: [{ role: 'user', content: 'go' }], tools: [
+    { name: 'a_b', input_schema: { type: 'object', properties: { env: { type: 'object' }, opts: { type: 'string' } } } },
+    { name: 'a_c', input_schema: { type: 'object', properties: { opts: { type: 'object' }, env: { type: 'string' } } } }
+  ] }), 'gemini-3-pro'))));
+  assert.deepEqual([...padded.keys()], ['a_b', 'a_c'], 'fixture: both tools are padded, on different keys');
+  const strip = withGeminiPlaceholderStrip({ ...NULL_TOOL_VOCAB, name: n => (n === 'a_b' || n === 'a_c' ? 'a.b' : n) }, padded);
+  const args = '{"env":{"_placeholder":true},"opts":{"_placeholder":true}}';
+  assert.equal(strip.args('a.b', args), args, 'the client name is ambiguous: nothing is stripped');
+  assert.deepEqual(JSON.parse(strip.args('a_b', args)), { env: {}, opts: { _placeholder: true } }, 'an upstream name keeps its own schema');
+  assert.deepEqual(JSON.parse(strip.args('a_c', args)), { env: { _placeholder: true }, opts: {} });
+
+  // A real collision: `a.b` and `a_b` sanitize alike for Vertex, so `a.b` goes up under an alias with its own schema.
+  const ir = anthropicToIR({ model: 'x', max_tokens: 10, messages: [{ role: 'user', content: 'go' }], tools: [
+    { name: 'a.b', input_schema: { type: 'object', properties: { env: { type: 'object' } } } },
+    { name: 'a_b', input_schema: { type: 'object', properties: { env: { type: 'object', properties: { _placeholder: { type: 'boolean' } } } } } }
+  ] });
+  const body = JSON.parse(JSON.stringify(irToVertexBody(ir, 'gemini-3-pro')));
+  recordGeminiPadding(ir, body, true);
+  const alias = [...ir._geminiPadded.keys()];
+  assert.equal(alias.length, 1, 'fixture: only a.b is padded');
+  assert.notEqual(alias[0], 'a_b');
+  const vocab = toolVocabForIR(ir, 'vertex');
+  const filled = '{"env":{"_placeholder":true}}';
+  assert.equal(vocab.name(alias[0]), 'a.b');
+  assert.deepEqual(JSON.parse(vocab.args(alias[0], filled)), { env: {} });
+  assert.equal(vocab.args('a_b', filled), filled, 'a_b declared its own _placeholder');
+});

package/tests/gateway.e2e.test.mjs

@@ -20,6 +20,8 @@ const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 
 let upstream, upstreamPort, proxy, proxyPort, tmpDir, fakeInterceptor, blindfoldPort, certsDir, cancelExitReap;
 const received = []; // { url, headers, body }
+// A model list read, kept apart so a background read never shows up as the last request a test made.
+const modelReads = [];
 const hangState = { closed: false, slowAborted: false };
 const bigState = { finishedAt: 0 };
 
@@ -70,6 +72,23 @@ const MOCK_ERRORS = {
   underscore: [400, 'maximum context length is 131_072 tokens, you requested 140_000 tokens'],
   prefixed: [400, 'prompt is too long: 200000 tokens > 199999 maximum'],
   nonumber: [400, 'The input exceeds the context window of this model.'],
+  ksuffix: [400, 'maximum context length is 128k tokens, you requested 140k tokens'],
+  msuffix: [400, "This model's maximum context length is 1m tokens. However, your messages resulted in 1.2M tokens."],
+  narrownbsp: [400, 'maximum context length is 131\u202f072 tokens, you requested 140\u202f000 tokens'],
+  nbsp: [400, 'maximum context length is 131\u00a0072 tokens, you requested 140\u00a0000 tokens'],
+  apostrophe: [400, "maximum context length is 131'072 tokens, you requested 140'000 tokens"],
+  indian: [400, 'maximum context length is 1,31,072 tokens, you requested 1,40,000 tokens'],
+  tokenlimit: [400, 'context_length_exceeded: input is 140000 tokens, over the token limit of 131072'],
+  decimal: [400, 'context_length_exceeded: limit of 3.141 seconds per call; input is 140000 tokens'],
+  fraction: [400, 'context_length_exceeded: limit 0.125 of the window; input is 140000 tokens'],
+  limitof: [400, 'context_length_exceeded: limit of 131072 tokens, input is 140000 tokens'],
+  limitbare: [400, 'context_length_exceeded: limit 131072 tokens, input is 140000 tokens'],
+  limitnotokens: [400, 'context_length_exceeded: limit of 131072. input is 140000 tokens'],
+  hugek: [400, 'maximum context length is 100000000000000000000k tokens, you requested 200000000000000000000k tokens'],
+  hugedigits: [400, 'maximum context length is 123456789012345678901 tokens, you requested 223456789012345678901 tokens'],
+  zero: [400, 'maximum context length is 0 tokens, you requested 140000 tokens'],
+  zerok: [400, 'maximum context length is 0k tokens, you requested 140k tokens'],
+  zeroasked: [400, 'maximum context length is 131072 tokens, you requested 0 tokens'],
   rate: [429, 'Rate limit reached on context tokens per minute: maximum context length is 131,072'],
   server: [500, "Internal error: this model's maximum context length is 131,072 tokens"]
 };
@@ -80,7 +99,8 @@ function startUpstream() {
     req.on('data', c => { body += c; });
     req.on('end', () => {
       const json = body ? JSON.parse(body) : {};
-      received.push({ url: req.url, headers: req.headers, body: json, raw: body });
+      if (req.method === 'GET' && req.url.split('?')[0].endsWith('/models')) modelReads.push({ url: req.url, headers: req.headers });
+      else received.push({ url: req.url, headers: req.headers, body: json, raw: body });
       const lastUser = JSON.stringify(json.messages?.at(-1) ?? json.contents?.at(-1) ?? '');
 
       // A stand-in intact: claude/* is a Claude Code account, the rest is not.
@@ -91,6 +111,14 @@ function startUpstream() {
       }
       // A provider list with real windows: a 1M model that also names the point to compress at,
       // and a smaller one. A model the list says nothing of keeps the catalog template.
+      // Two accounts on one baseURL whose lists name different output limits, read a little slowly.
+      if (req.method === 'GET' && req.url.startsWith('/chat/v1/models') && /^Bearer sk-cap-[ab]$/.test(req.headers.authorization || '')) {
+        const output = req.headers.authorization === 'Bearer sk-cap-a' ? 8192 : 2048;
+        return setTimeout(() => {
+          res.writeHead(200, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ data: [{ id: 'up-capped', context_length: 200000, max_output_tokens: output }] }));
+        }, 100);
+      }
       if (req.method === 'GET' && req.url.startsWith('/chat/v1/models')) {
         res.writeHead(200, { 'Content-Type': 'application/json' });
         return res.end(JSON.stringify({ data: [
@@ -294,7 +322,11 @@ before(async () => {
       capped: { name: 'Mock Capped Output', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-secret-capped', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
       agyph: { name: 'Mock agy to AG via chat', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-secret-agy', defaultModels: { main: 'ag/mock-flash' } },
       cappedtwin: { name: 'Mock Capped Twin', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-secret-capped', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
+      capa: { name: 'Mock Capped A', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-cap-a', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
+      capb: { name: 'Mock Capped B', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-cap-b', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
       cappedold: { name: 'Mock Capped Old', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-secret-capped', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
+      ovr: { name: 'Mock Override', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/elsewhere/v1`, endpoints: { 'openai-chat': `${base}/chat/v1/chat/completions` }, apiKey: 'sk-secret-ovr', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
+      ovrplain: { name: 'Mock No Override', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-secret-ovrplain', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
       ant: { name: 'Mock Anthropic', mode: 'direct', inFormat: 'auto', outFormat: 'anthropic', baseURL: `${base}/ant`, apiKey: 'sk-secret-ant', defaultModels: models }
     }
   };
@@ -682,6 +714,57 @@ test('max_tokens is clamped only from a fresh cache entry of the same profile',
   assert.equal(await ask('capped'), 8192);
 });
 
+// No /v1/models call for these profiles: the first request reads the list in the background, the next is clamped.
+test('a profile with no fresh model list is clamped from the request after a background read', async () => {
+  const ask = async (profile, tag) => {
+    const res = await post('/v1/messages', { model: 'claude-opus-4-6', max_tokens: 64000, messages: [{ role: 'user', content: tag }] }, { 'x-llm-profile': profile });
+    assert.equal(res.status, 200, tag);
+    await res.text();
+    const sent = received.findLast(r => JSON.stringify(r.body.messages || '').includes(tag));
+    assert.equal(sent.body.model, 'up-capped', tag);
+    return sent.body.max_tokens;
+  };
+  const reads = (key) => modelReads.filter(r => r.headers.authorization === `Bearer ${key}`).length;
+  const cached = (profile) => {
+    try { return JSON.parse(fs.readFileSync(path.join(tmpDir, 'model-windows.json'), 'utf8'))[`${profile}@http://127.0.0.1:${upstreamPort}/chat/v1`]; } catch { return null; }
+  };
+
+  assert.deepEqual(await Promise.all([ask('capa', 'cap-a-1'), ask('capa', 'cap-a-2')]), [64000, 64000], 'the first requests do not wait for the list');
+  assert.ok(await until(() => cached('capa')), 'the list of capa was read in the background');
+  assert.equal(await ask('capa', 'cap-a-3'), 8192);
+  assert.equal(reads('sk-cap-a'), 1, 'one read for the profile, however many requests found no entry');
+
+  assert.equal(await ask('capb', 'cap-b-1'), 64000, 'capa\'s entry on the same baseURL is not capb\'s');
+  assert.ok(await until(() => cached('capb')), 'the list of capb was read in the background');
+  assert.equal(await ask('capb', 'cap-b-2'), 2048);
+  assert.equal(await ask('capa', 'cap-a-4'), 8192, 'each profile keeps its own limit');
+});
+
+// The requests of an override profile never reach its baseURL, so neither may its key.
+test('a profile that routes through an endpoint override reads no model list; a plain profile still does', async () => {
+  const ask = async (profile, tag) => {
+    const res = await post('/v1/messages', { model: 'claude-opus-4-6', max_tokens: 64000, messages: [{ role: 'user', content: tag }] }, { 'x-llm-profile': profile });
+    assert.equal(res.status, 200, tag);
+    await res.text();
+    return received.findLast(r => JSON.stringify(r.body.messages || '').includes(tag));
+  };
+  const reads = (key) => modelReads.filter(r => r.headers.authorization === `Bearer ${key}` || r.headers['x-api-key'] === key);
+
+  const sent = await ask('ovr', 'ovr-1');
+  assert.equal(sent.url, '/chat/v1/chat/completions', 'the request went to the override');
+  assert.equal(sent.body.max_tokens, 64000);
+  await ask('ovrplain', 'ovrplain-1');
+  assert.ok(await until(() => reads('sk-secret-ovrplain').length === 1), 'the plain profile reads its list in the background');
+  await ask('ovr', 'ovr-2');
+  await new Promise(r => setTimeout(r, 200));
+  assert.deepEqual(reads('sk-secret-ovr'), [], 'the override profile sent its key to no /models');
+  assert.equal((await ask('ovr', 'ovr-3')).body.max_tokens, 64000, 'nothing clamps an override profile');
+  const listed = await fetch(url('/v1/models'), { headers: { 'x-llm-profile': 'ovr' } });
+  assert.equal(listed.status, 200);
+  await listed.text();
+  assert.deepEqual(reads('sk-secret-ovr'), [], 'the /v1/models catalog sent the override key to no /models');
+});
+
 // Gemini gets `_placeholder` on an empty nested object; the client never declared it, so it must not come back.
 test('A _placeholder the gateway added for Gemini is stripped from tool-call arguments, stream and non-stream', async () => {
   const tools = [{ type: 'function', name: 'run', parameters: { type: 'object', properties: {
@@ -795,6 +878,19 @@ test('Context overflow numbers with thousands separators reach Claude Code whole
     assert.equal(error.type, 'invalid_request_error', key);
     assert.equal(error.message, `prompt is too long: 140000 tokens > 131072 maximum (${MOCK_ERRORS[key][1]})`, key);
   }
+  const whole = async (key, asked, max) => assert.equal((await ask(key)).error.message, `prompt is too long: ${asked} tokens > ${max} maximum (${MOCK_ERRORS[key][1]})`, key);
+  await whole('ksuffix', 140000, 128000);
+  await whole('msuffix', 1200000, 1000000);
+  for (const key of ['narrownbsp', 'nbsp', 'apostrophe', 'indian', 'tokenlimit', 'limitof', 'limitbare']) await whole(key, 140000, 131072);
+  for (const key of ['decimal', 'fraction', 'limitnotokens']) {
+    assert.equal((await ask(key)).error.message, `prompt is too long: ${MOCK_ERRORS[key][1]}`, `${key}: a number after 'limit' that is no token count is ignored`);
+  }
+  for (const key of ['hugek', 'hugedigits']) {
+    assert.equal((await ask(key)).error.message, `prompt is too long: ${MOCK_ERRORS[key][1]}`, `${key}: a count past the safe integer range is no token count`);
+  }
+  for (const key of ['zero', 'zerok', 'zeroasked']) {
+    assert.equal((await ask(key)).error.message, `prompt is too long: ${MOCK_ERRORS[key][1]}`, `${key}: a count of zero is no token count`);
+  }
   assert.deepEqual(await ask('prefixed'), { status: 400, error: { type: 'invalid_request_error', message: MOCK_ERRORS.prefixed[1] } });
   assert.deepEqual(await ask('nonumber'), { status: 400, error: { type: 'invalid_request_error', message: `prompt is too long: ${MOCK_ERRORS.nonumber[1]}` } });
   assert.deepEqual(await ask('rate'), { status: 429, error: { type: 'rate_limit_error', message: MOCK_ERRORS.rate[1] } });
@@ -811,6 +907,11 @@ test('Codex WS: an upstream context overflow fails the turn with context_length_
   ws.socket.write(clientFrame(1, JSON.stringify({ type: 'response.create', model: 'main', input: 'MOCK_ERR:rate over ws' })));
   assert.ok(await until(() => ws.messages.filter(m => m.type === 'response.failed').length === 2), 'no second response.failed');
   assert.equal(ws.messages.filter(m => m.type === 'response.failed')[1].response.error.code, 'rate_limit_exceeded');
+  ws.socket.write(clientFrame(1, JSON.stringify({ type: 'response.create', model: 'main', input: 'MOCK_ERR:server over ws' })));
+  assert.ok(await until(() => ws.messages.filter(m => m.type === 'response.failed').length === 3), 'no third response.failed');
+  const server = ws.messages.filter(m => m.type === 'response.failed')[2].response.error;
+  assert.equal(server.code, 'server_error', 'a 500 that names the context length is a server error, not an overflow');
+  assert.equal(server.message, MOCK_ERRORS.server[1]);
   ws.socket.destroy();
 });
 

package/tests/goalcheck.e2e.test.mjs

@@ -1,4 +1,4 @@
-import { test } from 'node:test';
+import { test, after } from 'node:test';
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import os from 'node:os';
@@ -6,10 +6,15 @@ import path from 'node:path';
 import http from 'node:http';
 import { spawn } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
+import { pidsNaming, reapDir } from './helpers.mjs';
 
 const root = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
 const listen = server => new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
 
+// Every temp directory goes once more after the file, so a setup that fails early leaves none behind.
+const made = [];
+after(() => { for (const d of made) fs.rmSync(d, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
+
 async function until(check, timeoutMs = 6000) {
   const deadline = Date.now() + timeoutMs;
   while (Date.now() < deadline) {
@@ -21,12 +26,14 @@ async function until(check, timeoutMs = 6000) {
 
 async function fixture(t) {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'llmsw-goalcheck-'));
+  made.push(dir);
   const adminToken = 'admin-secret-token-123';
   fs.writeFileSync(path.join(dir, 'admin.token'), adminToken, { mode: 0o600 });
 
   let nativeVerdict = { ok: true, reason: 'looks complete' };
   let jevScores = { complete: 0.14, evidence: 0.84, unfinished: 0.63 };
   let jevStatus = 200;
+  let nativeDrops = false;
   const chatRequests = [];
 
   // Mock native upstream (Anthropic format)
@@ -38,6 +45,7 @@ async function fixture(t) {
         res.setHeader('content-type', 'application/json');
         return res.end('{"data":[]}');
       }
+      if (nativeDrops) return req.socket.destroy();
       const body = JSON.parse(raw);
       const isStream = Boolean(body.stream);
       const respText = JSON.stringify(nativeVerdict);
@@ -102,14 +110,21 @@ async function fixture(t) {
   });
   await listen(jevUp);
 
-  const reserve = http.createServer();
-  await listen(reserve);
-  const port = reserve.address().port;
-  await new Promise(r => reserve.close(r));
+  const freePort = async () => {
+    const reserve = http.createServer();
+    await listen(reserve);
+    const p = reserve.address().port;
+    await new Promise(r => reserve.close(r));
+    return p;
+  };
+  const port = await freePort();
+  // Without its own port the interceptor the proxy starts takes the default 3457.
+  const blindfoldPort = await freePort();
 
   const configPath = path.join(dir, 'config.json');
   fs.writeFileSync(configPath, JSON.stringify({
     port,
+    blindfold: { port: blindfoldPort },
     activeProfile: 'p1',
     activeProfiles: { claude: 'p1' },
     profiles: {
@@ -171,7 +186,7 @@ async function fixture(t) {
       new Promise(r => nativeUp.close(r)),
       new Promise(r => jevUp.close(r))
     ]);
-    fs.rmSync(dir, { recursive: true, force: true });
+    await reapDir(dir);
   });
 
   await until(async () => (await fetch(`http://127.0.0.1:${port}/health`).catch(() => null))?.ok);
@@ -204,6 +219,7 @@ async function fixture(t) {
     setNativeVerdict: v => { nativeVerdict = v; },
     setJevScores: s => { jevScores = s; },
     setJevStatus: st => { jevStatus = st; },
+    setNativeDrops: v => { nativeDrops = v; },
     jevRequests,
     chatRequests,
     logs: () => logs
@@ -358,3 +374,25 @@ test('goalcheck e2e: an openai-chat evaluator call that a text-only model refuse
   assert.equal(f.chatRequests.length, 2, 'one refusal, one retry');
   assert.ok(JSON.stringify(f.chatRequests[1]).includes('[image omitted'), 'the retry carries text in place of the image');
 });
+
+// The gate fails closed: an evaluator call that throws never lets the goal through, whatever Jev says.
+test('goalcheck e2e: a native evaluator call that throws fails closed on both upstream routes', async t => {
+  const f = await fixture(t);
+  await f.sendAdmin('/api/goal-check', 'POST', { enabled: true });
+  f.setJevScores({ complete: 0.95, evidence: 0.90, unfinished: 0.05 });
+  f.setNativeVerdict({ ok: true, reason: 'claimed done' });
+  f.setNativeDrops(true);
+  for (const profile of ['p1', 'chat']) {
+    const res = await f.sendClaude(evaluatorRequest, false, { 'x-llm-profile': profile });
+    assert.equal(res.status, 200, profile);
+    const verdict = JSON.parse((await res.json()).content[0].text);
+    assert.equal(verdict.ok, false, profile);
+    assert.match(verdict.reason, /Native evaluator unavailable/, profile);
+  }
+  const logs = (await (await f.sendAdmin('/api/goal-check/logs')).json()).logs;
+  assert.deepEqual(logs.slice(0, 2).map(l => l.verdictType), ['Unavailable', 'Unavailable']);
+});
+
+test('goalcheck e2e: the fixtures leave no process behind', () => {
+  assert.deepEqual(made.flatMap(pidsNaming), []);
+});

package/tests/idlecompact.e2e.test.mjs

@@ -1,5 +1,5 @@
 // Actual proxy -> HTTP mock upstream; no external provider or official account is used.
-import { test } from 'node:test';
+import { test, after } from 'node:test';
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import os from 'node:os';
@@ -28,8 +28,13 @@ function history() {
   return messages;
 }
 
+// Every temp directory goes once more after the file, so a setup that fails early leaves none behind.
+const made = [];
+after(() => { for (const d of made) fs.rmSync(d, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
+
 async function fixture(t, { model = '', codex = false } = {}) {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'llmsw-idle-e2e-'));
+  made.push(dir);
   const received = []; const pendingSummaries = [];
   const upstream = http.createServer((req, res) => {
     let raw = '';

package/tests/lifecycle.test.mjs

@@ -4,7 +4,7 @@
 // gateway port, `switch on` saved config.json before it refused, and only the CLI ever started the
 // interceptor, so the dashboard, `switch port` and a service start left HTTPS_PROXY pointing at nothing.
 
-import test from 'node:test';
+import test, { after } from 'node:test';
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import os from 'node:os';
@@ -42,8 +42,12 @@ const freePort = () => new Promise(r => {
   const s = net.createServer().listen(0, '127.0.0.1', () => { const { port } = s.address(); s.close(() => r(port)); });
 });
 
+// Every temp directory goes once more after the file, so a setup that fails early leaves none behind.
+const made = [];
+after(() => { for (const d of made) fs.rmSync(d, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 function makeWorkspace({ certs = false } = {}) {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'llmsw-life-'));
+  made.push(dir);
   const certDir = path.join(dir, 'certs');
   if (certs) execFileSync('bash', [path.join(ROOT, 'blindfold', 'make-certs.sh'), 'chatgpt.com', certDir], { stdio: 'ignore' });
   return { dir, certDir, cfgPath: path.join(dir, 'config.json'), claudeDir: path.join(dir, 'claude') };

package/tests/run-tests.test.mjs

@@ -0,0 +1,50 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { compileCacheIsNew, removeQuietly, isMainModule } from '../scripts/run-tests.mjs';
+
+const tmpDirs = [];
+test.after(() => { for (const d of tmpDirs) fs.rmSync(d, { recursive: true, force: true }); });
+const tmpDir = () => {
+  const d = fs.mkdtempSync(path.join(os.tmpdir(), 'lsw-runtests-'));
+  tmpDirs.push(d);
+  return d;
+};
+const npm = { npm_lifecycle_event: 'test' };
+
+// npm writes its compile cache only when it exits, so the cache it made for this run has no file yet.
+test('npm test removes only a compile cache that npm made for this run', () => {
+  const made = path.join(tmpDir(), 'node-compile-cache');
+  fs.mkdirSync(path.join(made, 'v26-x64-1000'), { recursive: true });
+  assert.equal(compileCacheIsNew(made, npm), true, 'empty: made at the start of this run');
+
+  const kept = path.join(tmpDir(), 'node-compile-cache');
+  fs.mkdirSync(path.join(kept, 'v26-x64-1000'), { recursive: true });
+  fs.writeFileSync(path.join(kept, 'v26-x64-1000', '0c92995d'), 'x');
+  assert.equal(compileCacheIsNew(kept, npm), false, 'a file in it: it was there before this run, however new');
+
+  assert.equal(compileCacheIsNew(made, {}), false, 'run without npm: the cache is not ours');
+  assert.equal(compileCacheIsNew(path.join(tmpDir(), 'none'), npm), false, 'no cache, nothing to remove');
+});
+
+test('a failed cleanup never throws, so the exit status of the tests stays', () => {
+  const dir = tmpDir();
+  assert.doesNotThrow(() => removeQuietly(dir, () => { throw Object.assign(new Error('busy'), { code: 'EBUSY' }); }));
+  removeQuietly(dir);
+  assert.equal(fs.existsSync(dir), false);
+});
+
+// node resolves a symlinked script to its real path, so the guard must compare real paths or the tests never run.
+test('the main-module guard sees the script through a symlink, and a false guard only returns false', () => {
+  const script = fileURLToPath(new URL('../scripts/run-tests.mjs', import.meta.url));
+  const link = path.join(tmpDir(), 'run-tests-link.mjs');
+  fs.symlinkSync(script, link);
+  assert.equal(isMainModule(script), true);
+  assert.equal(isMainModule(link), true, 'an absolute symlinked path is the script');
+  assert.equal(isMainModule(fileURLToPath(import.meta.url)), false, 'another file is not');
+  assert.equal(isMainModule(path.join(tmpDir(), 'missing.mjs')), false, 'a path that does not exist is not');
+  assert.equal(isMainModule(undefined), false);
+});

package/tests/shim.test.mjs

@@ -541,19 +541,22 @@ async function agyListener(kind) {
 async function runAgyWindowsShim(port) {
   const shimDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agyshim-'));
   const fakeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'agyfake-'));
-  fs.writeFileSync(path.join(fakeDir, 'agy.cmd'), `@echo off\r\necho CCU=[%CLOUD_CODE_URL%]\r\nnode "${AGY_REQUEST}"\r\n`);
-  for (const f of fs.readdirSync(STATE)) if (fs.statSync(path.join(STATE, f)).isFile()) fs.rmSync(path.join(STATE, f), { force: true });
-  fs.writeFileSync(path.join(STATE, 'active.flag'), 'active');
-  fs.writeFileSync(path.join(STATE, 'env-agy.cmd'), `SET "CLOUD_CODE_URL=http://127.0.0.1:${port}"\r\n`);
-  const shim = path.join(shimDir, 'agy.cmd');
-  fs.writeFileSync(shim, renderShim('agy', 'win32'), 'utf8');
-  const started = Date.now();
-  const child = await runAsync('cmd.exe', ['/d', '/c', shim], {
-    env: { ...process.env, LLM_SWITCHER_CONFIG: AGY_CONFIG, PATH: [fakeDir, shimDir, path.join(process.env.SystemRoot, 'System32'), path.dirname(process.execPath)].join(path.delimiter) }
-  });
-  fs.rmSync(shimDir, { recursive: true, force: true });
-  fs.rmSync(fakeDir, { recursive: true, force: true });
-  return { out: child.stdout || '', err: child.stderr || '', ms: Date.now() - started };
+  try {
+    fs.writeFileSync(path.join(fakeDir, 'agy.cmd'), `@echo off\r\necho CCU=[%CLOUD_CODE_URL%]\r\nnode "${AGY_REQUEST}"\r\n`);
+    for (const f of fs.readdirSync(STATE)) if (fs.statSync(path.join(STATE, f)).isFile()) fs.rmSync(path.join(STATE, f), { force: true });
+    fs.writeFileSync(path.join(STATE, 'active.flag'), 'active');
+    fs.writeFileSync(path.join(STATE, 'env-agy.cmd'), `SET "CLOUD_CODE_URL=http://127.0.0.1:${port}"\r\n`);
+    const shim = path.join(shimDir, 'agy.cmd');
+    fs.writeFileSync(shim, renderShim('agy', 'win32'), 'utf8');
+    const started = Date.now();
+    const child = await runAsync('cmd.exe', ['/d', '/c', shim], {
+      env: { ...process.env, LLM_SWITCHER_CONFIG: AGY_CONFIG, PATH: [fakeDir, shimDir, path.join(process.env.SystemRoot, 'System32'), path.dirname(process.execPath)].join(path.delimiter) }
+    });
+    return { out: child.stdout || '', err: child.stderr || '', ms: Date.now() - started };
+  } finally {
+    fs.rmSync(shimDir, { recursive: true, force: true });
+    fs.rmSync(fakeDir, { recursive: true, force: true });
+  }
 }
 
 for (const [kind, routed] of [['forged', false], ['silent', false], ['real', true]]) {