llm-switcher 1.6.11 → 1.6.13

package/tests/gateway.e2e.test.mjs

@@ -20,6 +20,8 @@ const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 
 let upstream, upstreamPort, proxy, proxyPort, tmpDir, fakeInterceptor, blindfoldPort, certsDir, cancelExitReap;
 const received = []; // { url, headers, body }
+// A model list read, kept apart so a background read never shows up as the last request a test made.
+const modelReads = [];
 const hangState = { closed: false, slowAborted: false };
 const bigState = { finishedAt: 0 };
 
@@ -62,13 +64,43 @@ const sse = (res, objs, { raw = [] } = {}) => {
   res.end();
 };
 
+// Upstream errors keyed by a MOCK_ERR:<key> marker in the last user message: [status, message].
+const MOCK_ERRORS = {
+  comma: [400, "This model's maximum context length is 131,072 tokens. However, your messages resulted in 140,000 tokens."],
+  dot: [400, 'Input exceeds the context window of 131.072 tokens: requested 140.000 tokens'],
+  space: [400, 'maximum context length is 131 072 tokens, you requested 140 000 tokens'],
+  underscore: [400, 'maximum context length is 131_072 tokens, you requested 140_000 tokens'],
+  prefixed: [400, 'prompt is too long: 200000 tokens > 199999 maximum'],
+  nonumber: [400, 'The input exceeds the context window of this model.'],
+  ksuffix: [400, 'maximum context length is 128k tokens, you requested 140k tokens'],
+  msuffix: [400, "This model's maximum context length is 1m tokens. However, your messages resulted in 1.2M tokens."],
+  narrownbsp: [400, 'maximum context length is 131\u202f072 tokens, you requested 140\u202f000 tokens'],
+  nbsp: [400, 'maximum context length is 131\u00a0072 tokens, you requested 140\u00a0000 tokens'],
+  apostrophe: [400, "maximum context length is 131'072 tokens, you requested 140'000 tokens"],
+  indian: [400, 'maximum context length is 1,31,072 tokens, you requested 1,40,000 tokens'],
+  tokenlimit: [400, 'context_length_exceeded: input is 140000 tokens, over the token limit of 131072'],
+  decimal: [400, 'context_length_exceeded: limit of 3.141 seconds per call; input is 140000 tokens'],
+  fraction: [400, 'context_length_exceeded: limit 0.125 of the window; input is 140000 tokens'],
+  limitof: [400, 'context_length_exceeded: limit of 131072 tokens, input is 140000 tokens'],
+  limitbare: [400, 'context_length_exceeded: limit 131072 tokens, input is 140000 tokens'],
+  limitnotokens: [400, 'context_length_exceeded: limit of 131072. input is 140000 tokens'],
+  hugek: [400, 'maximum context length is 100000000000000000000k tokens, you requested 200000000000000000000k tokens'],
+  hugedigits: [400, 'maximum context length is 123456789012345678901 tokens, you requested 223456789012345678901 tokens'],
+  zero: [400, 'maximum context length is 0 tokens, you requested 140000 tokens'],
+  zerok: [400, 'maximum context length is 0k tokens, you requested 140k tokens'],
+  zeroasked: [400, 'maximum context length is 131072 tokens, you requested 0 tokens'],
+  rate: [429, 'Rate limit reached on context tokens per minute: maximum context length is 131,072'],
+  server: [500, "Internal error: this model's maximum context length is 131,072 tokens"]
+};
+
 function startUpstream() {
   upstream = http.createServer((req, res) => {
     let body = '';
     req.on('data', c => { body += c; });
     req.on('end', () => {
       const json = body ? JSON.parse(body) : {};
-      received.push({ url: req.url, headers: req.headers, body: json, raw: body });
+      if (req.method === 'GET' && req.url.split('?')[0].endsWith('/models')) modelReads.push({ url: req.url, headers: req.headers });
+      else received.push({ url: req.url, headers: req.headers, body: json, raw: body });
       const lastUser = JSON.stringify(json.messages?.at(-1) ?? json.contents?.at(-1) ?? '');
 
       // A stand-in intact: claude/* is a Claude Code account, the rest is not.
@@ -79,6 +111,14 @@ function startUpstream() {
       }
       // A provider list with real windows: a 1M model that also names the point to compress at,
       // and a smaller one. A model the list says nothing of keeps the catalog template.
+      // Two accounts on one baseURL whose lists name different output limits, read a little slowly.
+      if (req.method === 'GET' && req.url.startsWith('/chat/v1/models') && /^Bearer sk-cap-[ab]$/.test(req.headers.authorization || '')) {
+        const output = req.headers.authorization === 'Bearer sk-cap-a' ? 8192 : 2048;
+        return setTimeout(() => {
+          res.writeHead(200, { 'Content-Type': 'application/json' });
+          res.end(JSON.stringify({ data: [{ id: 'up-capped', context_length: 200000, max_output_tokens: output }] }));
+        }, 100);
+      }
       if (req.method === 'GET' && req.url.startsWith('/chat/v1/models')) {
         res.writeHead(200, { 'Content-Type': 'application/json' });
         return res.end(JSON.stringify({ data: [
@@ -123,10 +163,28 @@ function startUpstream() {
             { choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }], usage: { prompt_tokens: 10, completion_tokens: 15 } }
           ], { raw: ['data: [DONE]\n\n'] });
         }
+        if (lastUser.includes('PLACEHOLDER_ECHO')) {
+          const args = '{"cmd":"ls","env":{"A":"1","_placeholder":true},"opts":{"_placeholder":"keep"},"_placeholder":true}';
+          if (!json.stream) {
+            res.writeHead(200, { 'Content-Type': 'application/json' });
+            return res.end(JSON.stringify({ choices: [{ index: 0, message: { role: 'assistant', content: null, tool_calls: [{ id: 'call_ph', type: 'function', function: { name: 'run', arguments: args } }] }, finish_reason: 'tool_calls' }], usage: { prompt_tokens: 5, completion_tokens: 5 } }));
+          }
+          return sse(res, [
+            { choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: 'call_ph', type: 'function', function: { name: 'run', arguments: args.slice(0, 30) } }] } }] },
+            { choices: [{ index: 0, delta: { tool_calls: [{ index: 0, function: { arguments: args.slice(30) } }] } }] },
+            { choices: [{ index: 0, delta: {}, finish_reason: 'tool_calls' }], usage: { prompt_tokens: 5, completion_tokens: 5 } }
+          ], { raw: ['data: [DONE]\n\n'] });
+        }
         if (lastUser.includes('RATE_LIMIT')) {
           res.writeHead(429, { 'Content-Type': 'application/json', 'retry-after': '7' });
           return res.end(JSON.stringify({ error: { message: 'slow down' } }));
         }
+        const mockErr = lastUser.match(/MOCK_ERR:(\w+)/)?.[1];
+        if (mockErr && MOCK_ERRORS[mockErr]) {
+          const [status, message] = MOCK_ERRORS[mockErr];
+          res.writeHead(status, { 'Content-Type': 'application/json' });
+          return res.end(JSON.stringify({ error: { message, type: 'invalid_request_error' } }));
+        }
         if (lastUser.includes('CONTEXT_OVERFLOW')) {
           res.writeHead(400, { 'Content-Type': 'application/json' });
           return res.end(JSON.stringify({ error: { message: "This model's maximum context length is 131072 tokens. However, you requested 140000 tokens (132000 in the messages, 8000 in the completion). Please reduce the length of the messages or completion.", type: 'invalid_request_error', code: 'invalid_request_error' } }));
@@ -262,6 +320,13 @@ before(async () => {
       intactcc: { name: 'Mock intact', mode: 'convert', tool: 'claude', outFormat: 'openai-chat', thinkingMode: 'off', baseURL: `${base}/intact/v1`, apiKey: 'sk-intact-user', defaultModels: { opus: 'claude/claude-opus-5', sonnet: 'gem/flash', haiku: 'claude/claude-haiku-4-5', fable: 'gem/flash' } },
       textonly: { name: 'Mock Text Only', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-secret-textonly', defaultModels: { opus: 'up-textonly', sonnet: 'up-badreq', haiku: 'up-haiku', fable: 'up-fable' } },
       capped: { name: 'Mock Capped Output', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-secret-capped', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
+      agyph: { name: 'Mock agy to AG via chat', mode: 'convert', tool: 'agy', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-secret-agy', defaultModels: { main: 'ag/mock-flash' } },
+      cappedtwin: { name: 'Mock Capped Twin', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-secret-capped', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
+      capa: { name: 'Mock Capped A', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-cap-a', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
+      capb: { name: 'Mock Capped B', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-cap-b', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
+      cappedold: { name: 'Mock Capped Old', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-secret-capped', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
+      ovr: { name: 'Mock Override', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/elsewhere/v1`, endpoints: { 'openai-chat': `${base}/chat/v1/chat/completions` }, apiKey: 'sk-secret-ovr', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
+      ovrplain: { name: 'Mock No Override', mode: 'convert', inFormat: 'auto', outFormat: 'openai-chat', baseURL: `${base}/chat/v1`, apiKey: 'sk-secret-ovrplain', defaultModels: { opus: 'up-capped', sonnet: 'up-sonnet', haiku: 'up-haiku', fable: 'up-fable' } },
       ant: { name: 'Mock Anthropic', mode: 'direct', inFormat: 'auto', outFormat: 'anthropic', baseURL: `${base}/ant`, apiKey: 'sk-secret-ant', defaultModels: models }
     }
   };
@@ -628,6 +693,123 @@ test('max_tokens above the listed output limit is clamped; a model with no liste
   assert.equal(received.at(-1).body.max_tokens, 64000);
 });
 
+// The clamp trusts only the list this profile read within the hour, whatever other profile shares its baseURL.
+test('max_tokens is clamped only from a fresh cache entry of the same profile', async () => {
+  const listed = await fetch(url('/v1/models'), { headers: { 'x-llm-profile': 'capped' } });
+  await listed.text();
+  const ask = async (profile) => {
+    const res = await post('/v1/messages', { model: 'claude-opus-4-6', max_tokens: 64000, messages: [{ role: 'user', content: 'hi' }] }, { 'x-llm-profile': profile });
+    assert.equal(res.status, 200, profile);
+    await res.text();
+    assert.equal(received.at(-1).body.model, 'up-capped', profile);
+    return received.at(-1).body.max_tokens;
+  };
+  assert.equal(await ask('cappedtwin'), 64000, 'another profile on the same baseURL has not read its list');
+  const cacheFile = path.join(tmpDir, 'model-windows.json');
+  const cache = JSON.parse(fs.readFileSync(cacheFile, 'utf8'));
+  const base = `http://127.0.0.1:${upstreamPort}/chat/v1`;
+  cache[`cappedold@${base}`] = { at: Date.now() - 2 * 3_600_000, baseURL: base, models: { 'up-capped': { context: 200000, compact: 0, output: 4096 } } };
+  fs.writeFileSync(cacheFile, JSON.stringify(cache));
+  assert.equal(await ask('cappedold'), 64000, 'an entry two hours old is not trusted');
+  assert.equal(await ask('capped'), 8192);
+});
+
+// No /v1/models call for these profiles: the first request reads the list in the background, the next is clamped.
+test('a profile with no fresh model list is clamped from the request after a background read', async () => {
+  const ask = async (profile, tag) => {
+    const res = await post('/v1/messages', { model: 'claude-opus-4-6', max_tokens: 64000, messages: [{ role: 'user', content: tag }] }, { 'x-llm-profile': profile });
+    assert.equal(res.status, 200, tag);
+    await res.text();
+    const sent = received.findLast(r => JSON.stringify(r.body.messages || '').includes(tag));
+    assert.equal(sent.body.model, 'up-capped', tag);
+    return sent.body.max_tokens;
+  };
+  const reads = (key) => modelReads.filter(r => r.headers.authorization === `Bearer ${key}`).length;
+  const cached = (profile) => {
+    try { return JSON.parse(fs.readFileSync(path.join(tmpDir, 'model-windows.json'), 'utf8'))[`${profile}@http://127.0.0.1:${upstreamPort}/chat/v1`]; } catch { return null; }
+  };
+
+  assert.deepEqual(await Promise.all([ask('capa', 'cap-a-1'), ask('capa', 'cap-a-2')]), [64000, 64000], 'the first requests do not wait for the list');
+  assert.ok(await until(() => cached('capa')), 'the list of capa was read in the background');
+  assert.equal(await ask('capa', 'cap-a-3'), 8192);
+  assert.equal(reads('sk-cap-a'), 1, 'one read for the profile, however many requests found no entry');
+
+  assert.equal(await ask('capb', 'cap-b-1'), 64000, 'capa\'s entry on the same baseURL is not capb\'s');
+  assert.ok(await until(() => cached('capb')), 'the list of capb was read in the background');
+  assert.equal(await ask('capb', 'cap-b-2'), 2048);
+  assert.equal(await ask('capa', 'cap-a-4'), 8192, 'each profile keeps its own limit');
+});
+
+// The requests of an override profile never reach its baseURL, so neither may its key.
+test('a profile that routes through an endpoint override reads no model list; a plain profile still does', async () => {
+  const ask = async (profile, tag) => {
+    const res = await post('/v1/messages', { model: 'claude-opus-4-6', max_tokens: 64000, messages: [{ role: 'user', content: tag }] }, { 'x-llm-profile': profile });
+    assert.equal(res.status, 200, tag);
+    await res.text();
+    return received.findLast(r => JSON.stringify(r.body.messages || '').includes(tag));
+  };
+  const reads = (key) => modelReads.filter(r => r.headers.authorization === `Bearer ${key}` || r.headers['x-api-key'] === key);
+
+  const sent = await ask('ovr', 'ovr-1');
+  assert.equal(sent.url, '/chat/v1/chat/completions', 'the request went to the override');
+  assert.equal(sent.body.max_tokens, 64000);
+  await ask('ovrplain', 'ovrplain-1');
+  assert.ok(await until(() => reads('sk-secret-ovrplain').length === 1), 'the plain profile reads its list in the background');
+  await ask('ovr', 'ovr-2');
+  await new Promise(r => setTimeout(r, 200));
+  assert.deepEqual(reads('sk-secret-ovr'), [], 'the override profile sent its key to no /models');
+  assert.equal((await ask('ovr', 'ovr-3')).body.max_tokens, 64000, 'nothing clamps an override profile');
+  const listed = await fetch(url('/v1/models'), { headers: { 'x-llm-profile': 'ovr' } });
+  assert.equal(listed.status, 200);
+  await listed.text();
+  assert.deepEqual(reads('sk-secret-ovr'), [], 'the /v1/models catalog sent the override key to no /models');
+});
+
+// Gemini gets `_placeholder` on an empty nested object; the client never declared it, so it must not come back.
+test('A _placeholder the gateway added for Gemini is stripped from tool-call arguments, stream and non-stream', async () => {
+  const tools = [{ type: 'function', name: 'run', parameters: { type: 'object', properties: {
+    cmd: { type: 'string' },
+    env: { type: 'object', additionalProperties: { type: 'string' } },
+    opts: { type: 'object', properties: { _placeholder: { type: 'string' } } }
+  } } }];
+  const want = { cmd: 'ls', env: { A: '1' }, opts: { _placeholder: 'keep' }, _placeholder: true };
+
+  const plain = await post('/v1/responses', { model: 'main', stream: false, tools, input: 'PLACEHOLDER_ECHO' }, { 'x-llm-profile': 'agmock' });
+  assert.equal(plain.status, 200);
+  const sent = received.at(-1).body.tools.find(t => t.function.name === 'run').function.parameters;
+  assert.deepEqual(Object.keys(sent.properties.env.properties), ['_placeholder'], 'the schema sent upstream was padded');
+  const call = (await plain.json()).output.find(o => o.type === 'function_call');
+  assert.deepEqual(JSON.parse(call.arguments), want);
+
+  const streamed = await post('/v1/responses', { model: 'main', stream: true, tools, input: 'PLACEHOLDER_ECHO' }, { 'x-llm-profile': 'agmock' });
+  const done = parseSSE(await streamed.text()).find(e => e.data?.type === 'response.completed');
+  assert.ok(done, 'the stream completes');
+  assert.deepEqual(JSON.parse(done.data.response.output.find(o => o.type === 'function_call').arguments), want);
+});
+
+test('A Code Assist (agy) client gets tool-call args without the padded _placeholder, stream and non-stream', async () => {
+  const envelope = { project: 'p', requestId: 'agent/s/1/t/1', model: 'gemini-3.8-flash-high', userAgent: 'antigravity', requestType: 'agent', request: {
+    contents: [{ role: 'user', parts: [{ text: 'PLACEHOLDER_ECHO' }] }],
+    tools: [{ functionDeclarations: [{ name: 'run', description: 'Run.', parameters: { type: 'OBJECT', properties: {
+      cmd: { type: 'STRING' }, env: { type: 'OBJECT' }, opts: { type: 'OBJECT', properties: { _placeholder: { type: 'STRING' } } }
+    } } }] }]
+  } };
+  const want = { cmd: 'ls', env: { A: '1' }, opts: { _placeholder: 'keep' }, _placeholder: true };
+  const headers = { 'content-type': 'application/json', 'x-llm-profile': 'agyph' };
+  const callOf = (chunks) => chunks.flatMap(c => c.response?.candidates?.[0]?.content?.parts || []).find(p => p.functionCall)?.functionCall;
+
+  const whole = await fetch(url('/v1internal:generateContent'), { method: 'POST', headers, body: JSON.stringify(envelope) });
+  assert.equal(whole.status, 200);
+  const sent = received.at(-1).body.tools.find(t => t.function.name === 'run').function.parameters;
+  assert.deepEqual(Object.keys(sent.properties.env.properties), ['_placeholder'], 'the schema sent upstream was padded');
+  assert.deepEqual(callOf([await whole.json()]).args, want);
+
+  const streamed = await fetch(url('/v1internal:streamGenerateContent?alt=sse'), { method: 'POST', headers, body: JSON.stringify(envelope) });
+  assert.equal(streamed.status, 200);
+  const chunks = parseSSE(await streamed.text()).map(e => e.data).filter(Boolean);
+  assert.deepEqual(callOf(chunks).args, want);
+});
+
 // A text-only model refuses an image that stays in the history, so every later turn would fail too.
 test('An image refused by a text-only chat model is retried once as text in each request', async () => {
   const image = { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0KGgo=' } };
@@ -685,6 +867,54 @@ test('sendClientError maps context overflow per client', async () => {
   assert.match(codexJson.error.message, /maximum context length is 131072/);
 });
 
+test('Context overflow numbers with thousands separators reach Claude Code whole', async () => {
+  const ask = async (key) => {
+    const res = await post('/v1/messages', { model: 'claude-opus-4-6', max_tokens: 10, messages: [{ role: 'user', content: `MOCK_ERR:${key}` }] });
+    return { status: res.status, error: (await res.json()).error };
+  };
+  for (const key of ['comma', 'dot', 'space', 'underscore']) {
+    const { status, error } = await ask(key);
+    assert.equal(status, 400, key);
+    assert.equal(error.type, 'invalid_request_error', key);
+    assert.equal(error.message, `prompt is too long: 140000 tokens > 131072 maximum (${MOCK_ERRORS[key][1]})`, key);
+  }
+  const whole = async (key, asked, max) => assert.equal((await ask(key)).error.message, `prompt is too long: ${asked} tokens > ${max} maximum (${MOCK_ERRORS[key][1]})`, key);
+  await whole('ksuffix', 140000, 128000);
+  await whole('msuffix', 1200000, 1000000);
+  for (const key of ['narrownbsp', 'nbsp', 'apostrophe', 'indian', 'tokenlimit', 'limitof', 'limitbare']) await whole(key, 140000, 131072);
+  for (const key of ['decimal', 'fraction', 'limitnotokens']) {
+    assert.equal((await ask(key)).error.message, `prompt is too long: ${MOCK_ERRORS[key][1]}`, `${key}: a number after 'limit' that is no token count is ignored`);
+  }
+  for (const key of ['hugek', 'hugedigits']) {
+    assert.equal((await ask(key)).error.message, `prompt is too long: ${MOCK_ERRORS[key][1]}`, `${key}: a count past the safe integer range is no token count`);
+  }
+  for (const key of ['zero', 'zerok', 'zeroasked']) {
+    assert.equal((await ask(key)).error.message, `prompt is too long: ${MOCK_ERRORS[key][1]}`, `${key}: a count of zero is no token count`);
+  }
+  assert.deepEqual(await ask('prefixed'), { status: 400, error: { type: 'invalid_request_error', message: MOCK_ERRORS.prefixed[1] } });
+  assert.deepEqual(await ask('nonumber'), { status: 400, error: { type: 'invalid_request_error', message: `prompt is too long: ${MOCK_ERRORS.nonumber[1]}` } });
+  assert.deepEqual(await ask('rate'), { status: 429, error: { type: 'rate_limit_error', message: MOCK_ERRORS.rate[1] } });
+  assert.deepEqual(await ask('server'), { status: 500, error: { type: 'api_error', message: MOCK_ERRORS.server[1] } });
+});
+
+test('Codex WS: an upstream context overflow fails the turn with context_length_exceeded', async () => {
+  const ws = await rawWs();
+  ws.socket.write(clientFrame(1, JSON.stringify({ type: 'response.create', model: 'main', input: 'MOCK_ERR:comma over ws' })));
+  assert.ok(await until(() => ws.messages.some(m => m.type === 'response.failed')), 'no response.failed');
+  const failed = ws.messages.find(m => m.type === 'response.failed').response.error;
+  assert.equal(failed.code, 'context_length_exceeded');
+  assert.equal(failed.message, MOCK_ERRORS.comma[1]);
+  ws.socket.write(clientFrame(1, JSON.stringify({ type: 'response.create', model: 'main', input: 'MOCK_ERR:rate over ws' })));
+  assert.ok(await until(() => ws.messages.filter(m => m.type === 'response.failed').length === 2), 'no second response.failed');
+  assert.equal(ws.messages.filter(m => m.type === 'response.failed')[1].response.error.code, 'rate_limit_exceeded');
+  ws.socket.write(clientFrame(1, JSON.stringify({ type: 'response.create', model: 'main', input: 'MOCK_ERR:server over ws' })));
+  assert.ok(await until(() => ws.messages.filter(m => m.type === 'response.failed').length === 3), 'no third response.failed');
+  const server = ws.messages.filter(m => m.type === 'response.failed')[2].response.error;
+  assert.equal(server.code, 'server_error', 'a 500 that names the context length is a server error, not an overflow');
+  assert.equal(server.message, MOCK_ERRORS.server[1]);
+  ws.socket.destroy();
+});
+
 test('Mid-stream upstream error surfaces as an error event instead of a fake end_turn', async () => {
   const res = await post('/v1/messages', { model: 'claude-opus-4-6', max_tokens: 10, stream: true, messages: [{ role: 'user', content: 'MID_STREAM_ERROR' }] });
   const events = parseSSE(await res.text());

package/tests/goalcheck.e2e.test.mjs

@@ -1,4 +1,4 @@
-import { test } from 'node:test';
+import { test, after } from 'node:test';
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import os from 'node:os';
@@ -6,10 +6,15 @@ import path from 'node:path';
 import http from 'node:http';
 import { spawn } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
+import { pidsNaming, reapDir } from './helpers.mjs';
 
 const root = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
 const listen = server => new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
 
+// Every temp directory goes once more after the file, so a setup that fails early leaves none behind.
+const made = [];
+after(() => { for (const d of made) fs.rmSync(d, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
+
 async function until(check, timeoutMs = 6000) {
   const deadline = Date.now() + timeoutMs;
   while (Date.now() < deadline) {
@@ -21,12 +26,15 @@ async function until(check, timeoutMs = 6000) {
 
 async function fixture(t) {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'llmsw-goalcheck-'));
+  made.push(dir);
   const adminToken = 'admin-secret-token-123';
   fs.writeFileSync(path.join(dir, 'admin.token'), adminToken, { mode: 0o600 });
 
   let nativeVerdict = { ok: true, reason: 'looks complete' };
   let jevScores = { complete: 0.14, evidence: 0.84, unfinished: 0.63 };
   let jevStatus = 200;
+  let nativeDrops = false;
+  const chatRequests = [];
 
   // Mock native upstream (Anthropic format)
   const nativeUp = http.createServer((req, res) => {
@@ -37,9 +45,20 @@ async function fixture(t) {
         res.setHeader('content-type', 'application/json');
         return res.end('{"data":[]}');
       }
+      if (nativeDrops) return req.socket.destroy();
       const body = JSON.parse(raw);
       const isStream = Boolean(body.stream);
       const respText = JSON.stringify(nativeVerdict);
+      // A text-only chat model: it refuses any image part.
+      if (req.url.endsWith('/chat/completions')) {
+        chatRequests.push(body);
+        res.setHeader('content-type', 'application/json');
+        if (body.messages.some(m => Array.isArray(m.content) && m.content.some(c => c?.type === 'image_url'))) {
+          res.statusCode = 400;
+          return res.end(JSON.stringify({ error: { message: 'Failed to deserialize the JSON body: messages[1]: unknown variant `image_url`, expected `text`' } }));
+        }
+        return res.end(JSON.stringify({ id: 'c1', object: 'chat.completion', model: body.model, choices: [{ index: 0, message: { role: 'assistant', content: respText }, finish_reason: 'stop' }] }));
+      }
 
       if (isStream) {
         res.setHeader('content-type', 'text/event-stream');
@@ -91,14 +110,21 @@ async function fixture(t) {
   });
   await listen(jevUp);
 
-  const reserve = http.createServer();
-  await listen(reserve);
-  const port = reserve.address().port;
-  await new Promise(r => reserve.close(r));
+  const freePort = async () => {
+    const reserve = http.createServer();
+    await listen(reserve);
+    const p = reserve.address().port;
+    await new Promise(r => reserve.close(r));
+    return p;
+  };
+  const port = await freePort();
+  // Without its own port the interceptor the proxy starts takes the default 3457.
+  const blindfoldPort = await freePort();
 
   const configPath = path.join(dir, 'config.json');
   fs.writeFileSync(configPath, JSON.stringify({
     port,
+    blindfold: { port: blindfoldPort },
     activeProfile: 'p1',
     activeProfiles: { claude: 'p1' },
     profiles: {
@@ -110,6 +136,16 @@ async function fixture(t) {
         outFormat: 'anthropic',
         baseURL: `http://127.0.0.1:${nativeUp.address().port}`,
         apiKey: 'sk-native-key'
+      },
+      chat: {
+        name: 'ChatMock',
+        mode: 'convert',
+        tool: 'claude',
+        inFormat: 'auto',
+        outFormat: 'openai-chat',
+        baseURL: `http://127.0.0.1:${nativeUp.address().port}/v1`,
+        apiKey: 'sk-chat-key',
+        defaultModels: { opus: 'up-textonly', sonnet: 'up-textonly', haiku: 'up-textonly', fable: 'up-textonly' }
       }
     },
     goalCheck: {
@@ -150,7 +186,7 @@ async function fixture(t) {
       new Promise(r => nativeUp.close(r)),
       new Promise(r => jevUp.close(r))
     ]);
-    fs.rmSync(dir, { recursive: true, force: true });
+    await reapDir(dir);
   });
 
   await until(async () => (await fetch(`http://127.0.0.1:${port}/health`).catch(() => null))?.ok);
@@ -166,11 +202,12 @@ async function fixture(t) {
     body: body ? JSON.stringify(body) : undefined
   });
 
-  const sendClaude = (body, stream = false) => fetch(`http://127.0.0.1:${port}/v1/messages`, {
+  const sendClaude = (body, stream = false, headers = {}) => fetch(`http://127.0.0.1:${port}/v1/messages`, {
     method: 'POST',
     headers: {
       'content-type': 'application/json',
-      'user-agent': 'claude-cli/2.1.295'
+      'user-agent': 'claude-cli/2.1.295',
+      ...headers
     },
     body: JSON.stringify({ ...body, stream })
   });
@@ -182,7 +219,9 @@ async function fixture(t) {
     setNativeVerdict: v => { nativeVerdict = v; },
     setJevScores: s => { jevScores = s; },
     setJevStatus: st => { jevStatus = st; },
+    setNativeDrops: v => { nativeDrops = v; },
     jevRequests,
+    chatRequests,
     logs: () => logs
   };
 }
@@ -314,3 +353,46 @@ test('goalcheck e2e: live interception, veto, and allow behaviors', async t => {
   assert.ok(logsData.logs.length >= 4);
   assert.equal(logsData.logs[0].verdictType, 'Unavailable');
 });
+
+test('goalcheck e2e: an openai-chat evaluator call that a text-only model refuses for an image is retried as text', async t => {
+  const f = await fixture(t);
+  await f.sendAdmin('/api/goal-check', 'POST', { enabled: true });
+  f.setJevScores({ complete: 0.95, evidence: 0.90, unfinished: 0.05 });
+  f.setNativeVerdict({ ok: true, reason: 'screenshot shows green tests' });
+  const image = { type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'iVBORw0KGgo=' } };
+  const request = {
+    ...evaluatorRequest,
+    messages: [
+      { role: 'user', content: [image, { type: 'text', text: 'Please fix the failing tests in repo.' }] },
+      ...evaluatorRequest.messages.slice(1)
+    ]
+  };
+  const res = await f.sendClaude(request, false, { 'x-llm-profile': 'chat' });
+  assert.equal(res.status, 200);
+  const verdict = JSON.parse((await res.json()).content[0].text);
+  assert.deepEqual(verdict, { ok: true, reason: 'screenshot shows green tests' });
+  assert.equal(f.chatRequests.length, 2, 'one refusal, one retry');
+  assert.ok(JSON.stringify(f.chatRequests[1]).includes('[image omitted'), 'the retry carries text in place of the image');
+});
+
+// The gate fails closed: an evaluator call that throws never lets the goal through, whatever Jev says.
+test('goalcheck e2e: a native evaluator call that throws fails closed on both upstream routes', async t => {
+  const f = await fixture(t);
+  await f.sendAdmin('/api/goal-check', 'POST', { enabled: true });
+  f.setJevScores({ complete: 0.95, evidence: 0.90, unfinished: 0.05 });
+  f.setNativeVerdict({ ok: true, reason: 'claimed done' });
+  f.setNativeDrops(true);
+  for (const profile of ['p1', 'chat']) {
+    const res = await f.sendClaude(evaluatorRequest, false, { 'x-llm-profile': profile });
+    assert.equal(res.status, 200, profile);
+    const verdict = JSON.parse((await res.json()).content[0].text);
+    assert.equal(verdict.ok, false, profile);
+    assert.match(verdict.reason, /Native evaluator unavailable/, profile);
+  }
+  const logs = (await (await f.sendAdmin('/api/goal-check/logs')).json()).logs;
+  assert.deepEqual(logs.slice(0, 2).map(l => l.verdictType), ['Unavailable', 'Unavailable']);
+});
+
+test('goalcheck e2e: the fixtures leave no process behind', () => {
+  assert.deepEqual(made.flatMap(pidsNaming), []);
+});

package/tests/idlecompact.e2e.test.mjs

@@ -1,5 +1,5 @@
 // Actual proxy -> HTTP mock upstream; no external provider or official account is used.
-import { test } from 'node:test';
+import { test, after } from 'node:test';
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import os from 'node:os';
@@ -28,8 +28,13 @@ function history() {
   return messages;
 }
 
+// Every temp directory goes once more after the file, so a setup that fails early leaves none behind.
+const made = [];
+after(() => { for (const d of made) fs.rmSync(d, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
+
 async function fixture(t, { model = '', codex = false } = {}) {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'llmsw-idle-e2e-'));
+  made.push(dir);
   const received = []; const pendingSummaries = [];
   const upstream = http.createServer((req, res) => {
     let raw = '';

package/tests/lifecycle.test.mjs

@@ -4,7 +4,7 @@
 // gateway port, `switch on` saved config.json before it refused, and only the CLI ever started the
 // interceptor, so the dashboard, `switch port` and a service start left HTTPS_PROXY pointing at nothing.
 
-import test from 'node:test';
+import test, { after } from 'node:test';
 import assert from 'node:assert/strict';
 import fs from 'node:fs';
 import os from 'node:os';
@@ -42,8 +42,12 @@ const freePort = () => new Promise(r => {
   const s = net.createServer().listen(0, '127.0.0.1', () => { const { port } = s.address(); s.close(() => r(port)); });
 });
 
+// Every temp directory goes once more after the file, so a setup that fails early leaves none behind.
+const made = [];
+after(() => { for (const d of made) fs.rmSync(d, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); });
 function makeWorkspace({ certs = false } = {}) {
   const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'llmsw-life-'));
+  made.push(dir);
   const certDir = path.join(dir, 'certs');
   if (certs) execFileSync('bash', [path.join(ROOT, 'blindfold', 'make-certs.sh'), 'chatgpt.com', certDir], { stdio: 'ignore' });
   return { dir, certDir, cfgPath: path.join(dir, 'config.json'), claudeDir: path.join(dir, 'claude') };

package/tests/run-tests.test.mjs

@@ -0,0 +1,50 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { compileCacheIsNew, removeQuietly, isMainModule } from '../scripts/run-tests.mjs';
+
+const tmpDirs = [];
+test.after(() => { for (const d of tmpDirs) fs.rmSync(d, { recursive: true, force: true }); });
+const tmpDir = () => {
+  const d = fs.mkdtempSync(path.join(os.tmpdir(), 'lsw-runtests-'));
+  tmpDirs.push(d);
+  return d;
+};
+const npm = { npm_lifecycle_event: 'test' };
+
+// npm writes its compile cache only when it exits, so the cache it made for this run has no file yet.
+test('npm test removes only a compile cache that npm made for this run', () => {
+  const made = path.join(tmpDir(), 'node-compile-cache');
+  fs.mkdirSync(path.join(made, 'v26-x64-1000'), { recursive: true });
+  assert.equal(compileCacheIsNew(made, npm), true, 'empty: made at the start of this run');
+
+  const kept = path.join(tmpDir(), 'node-compile-cache');
+  fs.mkdirSync(path.join(kept, 'v26-x64-1000'), { recursive: true });
+  fs.writeFileSync(path.join(kept, 'v26-x64-1000', '0c92995d'), 'x');
+  assert.equal(compileCacheIsNew(kept, npm), false, 'a file in it: it was there before this run, however new');
+
+  assert.equal(compileCacheIsNew(made, {}), false, 'run without npm: the cache is not ours');
+  assert.equal(compileCacheIsNew(path.join(tmpDir(), 'none'), npm), false, 'no cache, nothing to remove');
+});
+
+test('a failed cleanup never throws, so the exit status of the tests stays', () => {
+  const dir = tmpDir();
+  assert.doesNotThrow(() => removeQuietly(dir, () => { throw Object.assign(new Error('busy'), { code: 'EBUSY' }); }));
+  removeQuietly(dir);
+  assert.equal(fs.existsSync(dir), false);
+});
+
+// node resolves a symlinked script to its real path, so the guard must compare real paths or the tests never run.
+test('the main-module guard sees the script through a symlink, and a false guard only returns false', () => {
+  const script = fileURLToPath(new URL('../scripts/run-tests.mjs', import.meta.url));
+  const link = path.join(tmpDir(), 'run-tests-link.mjs');
+  fs.symlinkSync(script, link);
+  assert.equal(isMainModule(script), true);
+  assert.equal(isMainModule(link), true, 'an absolute symlinked path is the script');
+  assert.equal(isMainModule(fileURLToPath(import.meta.url)), false, 'another file is not');
+  assert.equal(isMainModule(path.join(tmpDir(), 'missing.mjs')), false, 'a path that does not exist is not');
+  assert.equal(isMainModule(undefined), false);
+});