llm-switcher 1.3.8 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +25 -17
- package/README.vi.md +24 -13
- package/config.example.json +0 -11
- package/docs/TOKEN-OPTIMIZER-INTEROP.md +110 -110
- package/docs/response-matrix.json +1130 -1130
- package/docs/tool-vocabulary.md +236 -0
- package/formats.mjs +277 -55
- package/package.json +1 -1
- package/proxy.mjs +55 -19
- package/skills/llm-switcher/SKILL.md +93 -93
- package/state.mjs +23 -91
- package/switch +0 -0
- package/switch.cmd +2 -2
- package/switch.mjs +2 -2
- package/tests/dashboard.e2e.test.mjs +10 -13
- package/tests/formats.test.mjs +98 -1
- package/tests/gateway.e2e.test.mjs +23 -9
- package/tests/helpers.mjs +24 -24
- package/tests/live-optimizer-interop.mjs +205 -205
- package/tests/state.test.mjs +6 -7
- package/tests/tool-repair.test.mjs +280 -0
- package/tests/ui.test.mjs +22 -15
- package/tests/windows.test.mjs +124 -0
- package/toolvocab.mjs +272 -0
- package/ui.html +32 -76
- package/windows.mjs +141 -0
package/proxy.mjs
CHANGED
|
@@ -23,9 +23,11 @@ import {
|
|
|
23
23
|
isProfileActive, profileAcceptsTarget, applyLaunchState, readLaunchFlags, redactConfig, MASKED_KEY,
|
|
24
24
|
modelForSlot, primaryModel, codexPublicModel, isSafeModelName, parsePort, CODEX_MODEL_SLOTS,
|
|
25
25
|
ensureAdminToken, ensureGatewaySecret, relayProof, identityProof, reconcileBlindfold, checkBlindfoldTarget,
|
|
26
|
-
codexModelEntry,
|
|
26
|
+
codexModelEntry,
|
|
27
27
|
contractLabSettings, STATE_DIR, stopRecordedBlindfold
|
|
28
28
|
} from './state.mjs';
|
|
29
|
+
import { parseModelWindows, resolveProfileWindows, readLocalCodexWindows } from './windows.mjs';
|
|
30
|
+
import { createToolVocab } from './toolvocab.mjs';
|
|
29
31
|
import { classifyCodexRole, classifyClaudeTier, syncLocalCatalog, refreshCatalog, checkVersionAndRefresh } from './catalog.mjs';
|
|
30
32
|
import { checkForUpdate } from './version.mjs';
|
|
31
33
|
import { applyUpdate } from './update.mjs';
|
|
@@ -487,12 +489,26 @@ function parsePayloadLine(line) {
|
|
|
487
489
|
try { return JSON.parse(s); } catch { return null; }
|
|
488
490
|
}
|
|
489
491
|
|
|
492
|
+
// The tool vocabulary the caller declared, so a model that answers with a name from its own
|
|
493
|
+
// training is written back in the caller's words instead of ending the turn with "No such tool
|
|
494
|
+
// available". Built once per request from the tools the caller actually sent, so a request that
|
|
495
|
+
// declared none pays nothing and a turn where the model agreed with the list changes nothing.
|
|
496
|
+
let toolVocabCache = null;
|
|
497
|
+
let toolVocabForIr = null;
|
|
498
|
+
function toolVocabFor(ir) {
|
|
499
|
+
if (toolVocabForIr !== ir) {
|
|
500
|
+
toolVocabForIr = ir;
|
|
501
|
+
toolVocabCache = createToolVocab(ir?.tools);
|
|
502
|
+
}
|
|
503
|
+
return toolVocabCache;
|
|
504
|
+
}
|
|
505
|
+
|
|
490
506
|
function clientRenderer(clientFormat, res, model, opts = {}) {
|
|
491
|
-
if (clientFormat === 'anthropic') return createAnthropicStream((e, d) => sendSSE(res, e, d), model);
|
|
507
|
+
if (clientFormat === 'anthropic') return createAnthropicStream((e, d) => sendSSE(res, e, d), model, opts);
|
|
492
508
|
if (clientFormat === 'responses') return createResponsesStream((e, d) => sendSSE(res, e, d), model, opts);
|
|
493
509
|
if (clientFormat === 'vertex') return createVertexStream((e, d) => sendSSE(res, null, d), model);
|
|
494
510
|
if (clientFormat === 'codeassist') return createCodeAssistStream((e, d) => sendSSE(res, null, d), model);
|
|
495
|
-
return createChatStream((e, d) => sendSSE(res, null, d), model);
|
|
511
|
+
return createChatStream((e, d) => sendSSE(res, null, d), model, opts);
|
|
496
512
|
}
|
|
497
513
|
|
|
498
514
|
function clientMessage(clientFormat, args) {
|
|
@@ -942,7 +958,7 @@ async function handleConvert(clientFormat, req, res, bodyBuffer, opts = {}) {
|
|
|
942
958
|
const out = clientMessage(clientFormat, {
|
|
943
959
|
model: requestedModel || mappedModel, think, text: [split.text], tools, toolMeta: ir.toolMeta,
|
|
944
960
|
finish: col.finish, prompt: col.prompt, completion, cached: col.cached,
|
|
945
|
-
reasoning: col.reasoning, sig: col.sig
|
|
961
|
+
reasoning: col.reasoning, sig: col.sig, vocab: toolVocabFor(ir)
|
|
946
962
|
});
|
|
947
963
|
sendJson(res, 200, out);
|
|
948
964
|
answered = true;
|
|
@@ -959,7 +975,7 @@ async function handleConvert(clientFormat, req, res, bodyBuffer, opts = {}) {
|
|
|
959
975
|
'Connection': 'keep-alive',
|
|
960
976
|
'X-Accel-Buffering': 'no'
|
|
961
977
|
});
|
|
962
|
-
const renderer = clientRenderer(clientFormat, res, requestedModel || mappedModel, { toolMeta: ir.toolMeta });
|
|
978
|
+
const renderer = clientRenderer(clientFormat, res, requestedModel || mappedModel, { toolMeta: ir.toolMeta, vocab: toolVocabFor(ir) });
|
|
963
979
|
renderer.start();
|
|
964
980
|
const splitter = createThinkTagSplitter(t => renderer.think(t), t => renderer.text(t));
|
|
965
981
|
const streamError = await pumpStream(upstreamRes, { normalize, col, renderer, splitter, signal: ac.signal, sink: res, tag: profileKey });
|
|
@@ -1227,15 +1243,18 @@ async function fetchModels(body, cfg) {
|
|
|
1227
1243
|
}
|
|
1228
1244
|
|
|
1229
1245
|
// A listed model's token limits, as intact and OpenRouter write them. Null when the list gives none.
|
|
1246
|
+
// The two windows come from windows.mjs, so /api/fetch-models and /v1/models cannot read a model
|
|
1247
|
+
// differently.
|
|
1230
1248
|
function modelLimits(m) {
|
|
1231
1249
|
if (!m || typeof m !== 'object') return null;
|
|
1250
|
+
const w = parseModelWindows(m);
|
|
1232
1251
|
const n = v => (Number.isFinite(v) && v > 0 ? v : 0);
|
|
1233
1252
|
const top = m.top_provider && typeof m.top_provider === 'object' ? m.top_provider : {};
|
|
1234
1253
|
const out = {};
|
|
1235
|
-
const context = n(m.context_length) || n(top.context_length) || n(m.context_window);
|
|
1236
1254
|
const input = n(m.max_input_tokens);
|
|
1237
1255
|
const output = n(m.max_output_tokens) || n(top.max_completion_tokens);
|
|
1238
|
-
if (context) out.context = context;
|
|
1256
|
+
if (w?.context) out.context = w.context;
|
|
1257
|
+
if (w?.compact) out.compact = w.compact;
|
|
1239
1258
|
if (input) out.input = input;
|
|
1240
1259
|
if (output) out.output = output;
|
|
1241
1260
|
return Object.keys(out).length ? out : null;
|
|
@@ -1339,12 +1358,12 @@ async function route(req, res) {
|
|
|
1339
1358
|
// Without publicModels, fall back to the deduplicated mapped upstream IDs.
|
|
1340
1359
|
if (method === 'GET' && (pathname === '/v1/models' || pathname === '/models')) {
|
|
1341
1360
|
checkVersionAndRefresh(req.headers['anthropic-version'] ? 'claude' : 'codex', req.headers, STATE_DIR);
|
|
1342
|
-
const { ids, windows } = servedModels(req);
|
|
1361
|
+
const { ids, windows } = await servedModels(req);
|
|
1343
1362
|
const created = Math.floor(Date.now() / 1000);
|
|
1344
1363
|
return sendJson(res, 200, {
|
|
1345
1364
|
object: 'list',
|
|
1346
1365
|
data: ids.map(id => ({ id, object: 'model', created, owned_by: 'system' })),
|
|
1347
|
-
models: ids.map(id => ({ ...codexModelEntry(id, windows
|
|
1366
|
+
models: ids.map(id => ({ ...codexModelEntry(id, windows[id]), id, object: 'model', created, owned_by: 'system' }))
|
|
1348
1367
|
});
|
|
1349
1368
|
}
|
|
1350
1369
|
|
|
@@ -1352,8 +1371,8 @@ async function route(req, res) {
|
|
|
1352
1371
|
if (method === 'GET' && (pathname.startsWith('/v1/models/') || pathname.startsWith('/models/'))) {
|
|
1353
1372
|
const modelId = decodeURIComponent(pathname.replace(/^\/(v1\/)?models\//, ''));
|
|
1354
1373
|
if (modelId) {
|
|
1355
|
-
const { windows } = servedModels(req);
|
|
1356
|
-
return sendJson(res, 200, { ...codexModelEntry(modelId, windows
|
|
1374
|
+
const { windows } = await servedModels(req);
|
|
1375
|
+
return sendJson(res, 200, { ...codexModelEntry(modelId, windows[modelId]), id: modelId, object: 'model', created: Math.floor(Date.now() / 1000), owned_by: 'system' });
|
|
1357
1376
|
}
|
|
1358
1377
|
}
|
|
1359
1378
|
|
|
@@ -1501,15 +1520,28 @@ function passthroughCodeAssist(req, res, body) {
|
|
|
1501
1520
|
else req.pipe(up);
|
|
1502
1521
|
}
|
|
1503
1522
|
|
|
1504
|
-
// The names /v1/models serves, and the window of each. Official names when the profile
|
|
1505
|
-
// them, otherwise the mapped upstream IDs
|
|
1506
|
-
|
|
1507
|
-
|
|
1523
|
+
// The names /v1/models serves, and the real window of each. Official names when the profile
|
|
1524
|
+
// publishes them, otherwise the mapped upstream IDs.
|
|
1525
|
+
//
|
|
1526
|
+
// The window is what the model really has, so the tool sizes its own session: the profile's list
|
|
1527
|
+
// first (intact serves context_length and compact_window), then the list the Codex CLI cached for
|
|
1528
|
+
// its account, and the template only when neither names a window.
|
|
1529
|
+
async function servedModels(req) {
|
|
1530
|
+
const { profileKey, profile } = getFirstActiveProfile(['responses', 'anthropic'], req);
|
|
1531
|
+
const windows = await modelWindowsFor(profileKey, profile);
|
|
1508
1532
|
if (Array.isArray(profile?.publicModels) && profile.publicModels.length) {
|
|
1509
|
-
return { ids: [...new Set(profile.publicModels.filter(Boolean))], windows
|
|
1533
|
+
return { ids: [...new Set(profile.publicModels.filter(Boolean))], windows };
|
|
1510
1534
|
}
|
|
1511
|
-
|
|
1512
|
-
|
|
1535
|
+
return { ids: [...new Set(Object.values(profile?.defaultModels || {}).filter(Boolean))], windows };
|
|
1536
|
+
}
|
|
1537
|
+
|
|
1538
|
+
// The windows of every model the profile serves. A profile that names no model in its own list
|
|
1539
|
+
// still gets the cached numbers, because an entry with the template window is better than none.
|
|
1540
|
+
async function modelWindowsFor(profileKey, profile) {
|
|
1541
|
+
if (!profile?.baseURL) return {};
|
|
1542
|
+
const listed = await resolveProfileWindows(STATE_DIR, `${profileKey || 'profile'}@${profile.baseURL}`, profile.baseURL, profile.apiKey);
|
|
1543
|
+
if (Object.keys(listed).length) return listed;
|
|
1544
|
+
return readLocalCodexWindows();
|
|
1513
1545
|
}
|
|
1514
1546
|
|
|
1515
1547
|
async function routeApi(req, res, method, pathname) {
|
|
@@ -1687,6 +1719,10 @@ async function routeConfigApi(res, method, pathname, body) {
|
|
|
1687
1719
|
for (const k of ['outFormat', 'optimizerURL', 'thinkingMode']) {
|
|
1688
1720
|
if (Object.hasOwn(profile, k) && !profile[k]) delete merged[k];
|
|
1689
1721
|
}
|
|
1722
|
+
// The merge above keeps a key the payload does not mention, so a retired one has to be dropped
|
|
1723
|
+
// here: the window comes from the model's own list now, and a flag left in config.json would
|
|
1724
|
+
// be read by nothing while still telling the next reader that the window is a choice.
|
|
1725
|
+
delete merged.model1M;
|
|
1690
1726
|
cfg.profiles[key] = merged;
|
|
1691
1727
|
|
|
1692
1728
|
// A target assigned to this profile whose new inFormat no longer supports it -> unassign that target.
|
|
@@ -2031,7 +2067,7 @@ async function handleWsResponseCreate(socket, payload, req, ac, history = null)
|
|
|
2031
2067
|
if (socket.writable) {
|
|
2032
2068
|
socket.write(encodeWsFrame(JSON.stringify(d)));
|
|
2033
2069
|
}
|
|
2034
|
-
}, requestedModel || mappedModel, { toolMeta: ir.toolMeta });
|
|
2070
|
+
}, requestedModel || mappedModel, { toolMeta: ir.toolMeta, vocab: toolVocabFor(ir) });
|
|
2035
2071
|
|
|
2036
2072
|
renderer.start();
|
|
2037
2073
|
const splitter = createThinkTagSplitter(t => renderer.think(t), t => renderer.text(t));
|
|
@@ -1,93 +1,93 @@
|
|
|
1
|
-
---
|
|
2
|
-
name: llm-switcher
|
|
3
|
-
description: Mandatory guidance for AI coding agents and token compression tools (Headroom, RTK, Ponytail, Repomix). Enforces routing all LLM traffic through the local LLM Switcher edge gateway (http://127.0.0.1:3456) instead of directly calling provider endpoints.
|
|
4
|
-
---
|
|
5
|
-
|
|
6
|
-
# LLM Switcher Edge Gateway Skill
|
|
7
|
-
|
|
8
|
-
This skill governs how AI coding agents (Claude Code, Codex, the Antigravity CLI `agy`, Cursor, Windsurf, Opencode) and prompt optimization/compression tools (Headroom, RTK, Ponytail, Repomix) must interact with LLM providers on this workstation.
|
|
9
|
-
|
|
10
|
-
## 1. The Core Directive
|
|
11
|
-
|
|
12
|
-
> **ALL OUTBOUND LLM REQUESTS MUST ROUTE THROUGH LLM SWITCHER (`http://127.0.0.1:3456`).**
|
|
13
|
-
>
|
|
14
|
-
> Never configure any tool, script, sub-agent, or token compression proxy to call external provider endpoints directly (such as `https://api.anthropic.com`, `https://api.openai.com`, or `https://generativelanguage.googleapis.com`).
|
|
15
|
-
>
|
|
16
|
-
> If a token compression tool (Headroom, RTK, Ponytail) is used, it MUST be positioned as an **inner processing layer**, and its **upstream destination MUST be configured to point to LLM Switcher** at `http://127.0.0.1:3456`.
|
|
17
|
-
|
|
18
|
-
```
|
|
19
|
-
[Client CLI / Agent]
|
|
20
|
-
│
|
|
21
|
-
▼
|
|
22
|
-
[Token Compressor: RTK / Headroom / Ponytail] <-- Inner Layer
|
|
23
|
-
│
|
|
24
|
-
▼ (MANDATORY: Upstream MUST point to http://127.0.0.1:3456)
|
|
25
|
-
[LLM Switcher Edge Gateway (:3456)] <-- Outermost Gatekeeper
|
|
26
|
-
│
|
|
27
|
-
▼ (Heals schemas, maps 1M context, extracts thinking, converts protocol)
|
|
28
|
-
[Internet / intact (Recommended) / 9Router / Upstream LLM]
|
|
29
|
-
```
|
|
30
|
-
|
|
31
|
-
## 2. Why This is Mandatory
|
|
32
|
-
|
|
33
|
-
Direct outbound calls from token compressors will break the developer environment for several reasons:
|
|
34
|
-
|
|
35
|
-
1. **Schema Breakage & The Healer Engine:**
|
|
36
|
-
Token optimizers frequently prune conversation history to save tokens. In doing so, they often delete an `assistant` turn containing a `tool_use`, while leaving the subsequent `user` turn containing a `tool_result`.
|
|
37
|
-
- Anthropic and OpenAI APIs strictly reject this with `HTTP 400 (orphaned tool_use_id)`.
|
|
38
|
-
- **LLM Switcher** intercepts these malformed requests and runs its **Healer Engine**, automatically converting orphaned results into safe context text blocks and merging consecutive same-role turns so upstream providers accept the payload cleanly.
|
|
39
|
-
2. **Thinking / Reasoning Protection:**
|
|
40
|
-
Many token compressors naively strip `thinking` parameters or reasoning budgets to reduce token count.
|
|
41
|
-
- When routed through LLM Switcher, the gateway detects reasoning models (such as `ag/claude-opus-4-6-thinking` or `ag/gemini-3.8-flash`) and **automatically restores thinking parameters**, ensuring the model retains its full reasoning capabilities.
|
|
42
|
-
3. **Official Context Windows & Dynamic Model Discovery:**
|
|
43
|
-
Context windows follow the model you select without mutating permanent settings. The gateway auto-discovers official models from Anthropic and OpenAI, refreshing mappings on tool version upgrades (`switch models`).
|
|
44
|
-
4. **Server Gateway Synergy (intact / 9Router):**
|
|
45
|
-
LLM Switcher pairs directly with server-side gateways like **intact** (strongly recommended for credential proxying, account pooling, and contract lab self-healing) or 9Router, handling retries and quotas far better than any local tool.
|
|
46
|
-
|
|
47
|
-
## 3. Configuring Token Compression Tools
|
|
48
|
-
|
|
49
|
-
When setting up or assisting the user with token compression tools, apply these exact configurations:
|
|
50
|
-
|
|
51
|
-
### A. Headroom (Prompt Compression Proxy)
|
|
52
|
-
When configuring Headroom's upstream endpoint:
|
|
53
|
-
- In Headroom config (`headroom.yaml` or environment variables):
|
|
54
|
-
```bash
|
|
55
|
-
ANTHROPIC_BASE_URL="http://127.0.0.1:3456"
|
|
56
|
-
OPENAI_BASE_URL="http://127.0.0.1:3456/v1"
|
|
57
|
-
```
|
|
58
|
-
- Point your CLI to Headroom's listening port (e.g. `http://127.0.0.1:8787`). Headroom compresses the prompt and forwards it to LLM Switcher on port `3456`.
|
|
59
|
-
|
|
60
|
-
### B. RTK (Rust Token Killer / Repomix)
|
|
61
|
-
- In RTK configuration or execution flags:
|
|
62
|
-
```bash
|
|
63
|
-
# Ensure RTK's target upstream proxy is LLM Switcher
|
|
64
|
-
export UPSTREAM_URL="http://127.0.0.1:3456"
|
|
65
|
-
```
|
|
66
|
-
|
|
67
|
-
### C. Ponytail / Custom Proxies
|
|
68
|
-
- Always set the proxy's upstream host to:
|
|
69
|
-
`http://127.0.0.1:3456` (or `http://127.0.0.1:3456/v1` for OpenAI-compatible tools).
|
|
70
|
-
|
|
71
|
-
### D. Sub-Agents & Spawned Terminal Commands
|
|
72
|
-
When an agent spawns sub-processes that call LLM CLIs:
|
|
73
|
-
- **Never advise sourcing `env.sh` or `env.cmd`.** Those files carry no proxy variables any
|
|
74
|
-
more, so sourcing one changes nothing — and a stale variable an older version once wrote
|
|
75
|
-
would still point the tool at a port where nothing listens.
|
|
76
|
-
- Run `switch claude <profile>`, `switch codex <profile>` or `switch agy <profile>` instead. The
|
|
77
|
-
shims in `~/.llm-switcher/bin` inject the gateway variables into the tool process itself and
|
|
78
|
-
clear anything stale first. For `agy` the variable is `CLOUD_CODE_URL`, not a proxy.
|
|
79
|
-
- `claude`, `codex` and `agy` on `PATH` already are those shims: invoke them normally, and never
|
|
80
|
-
wrap them in another script.
|
|
81
|
-
|
|
82
|
-
## 4. Operational Rules for AI Agents
|
|
83
|
-
|
|
84
|
-
1. **NEVER edit `~/.claude/settings.json` directly.**
|
|
85
|
-
LLM Switcher uses zero-mutation launcher flags (`active.flag`). Writing custom base URLs or auth tokens into `settings.json` triggers warning banners in Claude Code and risks breaking the user's official subscription. Use `switch` CLI commands instead.
|
|
86
|
-
2. **Check Gateway Health Before Complex Operations:**
|
|
87
|
-
Run `switch status` or call the `switcher_audit` MCP tool to confirm:
|
|
88
|
-
- LLM Switcher is active on port `3456`.
|
|
89
|
-
- The active profile matches the intended CLI target (Claude Code, Codex, or the Antigravity CLI).
|
|
90
|
-
3. **Verify Routing When Errors Occur:**
|
|
91
|
-
If a tool fails with `HTTP 400`, `HTTP 502`, or connection errors:
|
|
92
|
-
- Run `switch doctor` to audit port collisions and environment variables.
|
|
93
|
-
- Inspect recent request logs via `http://127.0.0.1:3456/ui` (Tab 4: Live Inspector) to see if an intermediary tool mangled the payload.
|
|
1
|
+
---
|
|
2
|
+
name: llm-switcher
|
|
3
|
+
description: Mandatory guidance for AI coding agents and token compression tools (Headroom, RTK, Ponytail, Repomix). Enforces routing all LLM traffic through the local LLM Switcher edge gateway (http://127.0.0.1:3456) instead of directly calling provider endpoints.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# LLM Switcher Edge Gateway Skill
|
|
7
|
+
|
|
8
|
+
This skill governs how AI coding agents (Claude Code, Codex, the Antigravity CLI `agy`, Cursor, Windsurf, Opencode) and prompt optimization/compression tools (Headroom, RTK, Ponytail, Repomix) must interact with LLM providers on this workstation.
|
|
9
|
+
|
|
10
|
+
## 1. The Core Directive
|
|
11
|
+
|
|
12
|
+
> **ALL OUTBOUND LLM REQUESTS MUST ROUTE THROUGH LLM SWITCHER (`http://127.0.0.1:3456`).**
|
|
13
|
+
>
|
|
14
|
+
> Never configure any tool, script, sub-agent, or token compression proxy to call external provider endpoints directly (such as `https://api.anthropic.com`, `https://api.openai.com`, or `https://generativelanguage.googleapis.com`).
|
|
15
|
+
>
|
|
16
|
+
> If a token compression tool (Headroom, RTK, Ponytail) is used, it MUST be positioned as an **inner processing layer**, and its **upstream destination MUST be configured to point to LLM Switcher** at `http://127.0.0.1:3456`.
|
|
17
|
+
|
|
18
|
+
```
|
|
19
|
+
[Client CLI / Agent]
|
|
20
|
+
│
|
|
21
|
+
▼
|
|
22
|
+
[Token Compressor: RTK / Headroom / Ponytail] <-- Inner Layer
|
|
23
|
+
│
|
|
24
|
+
▼ (MANDATORY: Upstream MUST point to http://127.0.0.1:3456)
|
|
25
|
+
[LLM Switcher Edge Gateway (:3456)] <-- Outermost Gatekeeper
|
|
26
|
+
│
|
|
27
|
+
▼ (Heals schemas, maps 1M context, extracts thinking, converts protocol)
|
|
28
|
+
[Internet / intact (Recommended) / 9Router / Upstream LLM]
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
## 2. Why This is Mandatory
|
|
32
|
+
|
|
33
|
+
Direct outbound calls from token compressors will break the developer environment for several reasons:
|
|
34
|
+
|
|
35
|
+
1. **Schema Breakage & The Healer Engine:**
|
|
36
|
+
Token optimizers frequently prune conversation history to save tokens. In doing so, they often delete an `assistant` turn containing a `tool_use`, while leaving the subsequent `user` turn containing a `tool_result`.
|
|
37
|
+
- Anthropic and OpenAI APIs strictly reject this with `HTTP 400 (orphaned tool_use_id)`.
|
|
38
|
+
- **LLM Switcher** intercepts these malformed requests and runs its **Healer Engine**, automatically converting orphaned results into safe context text blocks and merging consecutive same-role turns so upstream providers accept the payload cleanly.
|
|
39
|
+
2. **Thinking / Reasoning Protection:**
|
|
40
|
+
Many token compressors naively strip `thinking` parameters or reasoning budgets to reduce token count.
|
|
41
|
+
- When routed through LLM Switcher, the gateway detects reasoning models (such as `ag/claude-opus-4-6-thinking` or `ag/gemini-3.8-flash`) and **automatically restores thinking parameters**, ensuring the model retains its full reasoning capabilities.
|
|
42
|
+
3. **Official Context Windows & Dynamic Model Discovery:**
|
|
43
|
+
Context windows follow the model you select without mutating permanent settings. The gateway auto-discovers official models from Anthropic and OpenAI, refreshing mappings on tool version upgrades (`switch models`).
|
|
44
|
+
4. **Server Gateway Synergy (intact / 9Router):**
|
|
45
|
+
LLM Switcher pairs directly with server-side gateways like **intact** (strongly recommended for credential proxying, account pooling, and contract lab self-healing) or 9Router, handling retries and quotas far better than any local tool.
|
|
46
|
+
|
|
47
|
+
## 3. Configuring Token Compression Tools
|
|
48
|
+
|
|
49
|
+
When setting up or assisting the user with token compression tools, apply these exact configurations:
|
|
50
|
+
|
|
51
|
+
### A. Headroom (Prompt Compression Proxy)
|
|
52
|
+
When configuring Headroom's upstream endpoint:
|
|
53
|
+
- In Headroom config (`headroom.yaml` or environment variables):
|
|
54
|
+
```bash
|
|
55
|
+
ANTHROPIC_BASE_URL="http://127.0.0.1:3456"
|
|
56
|
+
OPENAI_BASE_URL="http://127.0.0.1:3456/v1"
|
|
57
|
+
```
|
|
58
|
+
- Point your CLI to Headroom's listening port (e.g. `http://127.0.0.1:8787`). Headroom compresses the prompt and forwards it to LLM Switcher on port `3456`.
|
|
59
|
+
|
|
60
|
+
### B. RTK (Rust Token Killer / Repomix)
|
|
61
|
+
- In RTK configuration or execution flags:
|
|
62
|
+
```bash
|
|
63
|
+
# Ensure RTK's target upstream proxy is LLM Switcher
|
|
64
|
+
export UPSTREAM_URL="http://127.0.0.1:3456"
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
### C. Ponytail / Custom Proxies
|
|
68
|
+
- Always set the proxy's upstream host to:
|
|
69
|
+
`http://127.0.0.1:3456` (or `http://127.0.0.1:3456/v1` for OpenAI-compatible tools).
|
|
70
|
+
|
|
71
|
+
### D. Sub-Agents & Spawned Terminal Commands
|
|
72
|
+
When an agent spawns sub-processes that call LLM CLIs:
|
|
73
|
+
- **Never advise sourcing `env.sh` or `env.cmd`.** Those files carry no proxy variables any
|
|
74
|
+
more, so sourcing one changes nothing — and a stale variable an older version once wrote
|
|
75
|
+
would still point the tool at a port where nothing listens.
|
|
76
|
+
- Run `switch claude <profile>`, `switch codex <profile>` or `switch agy <profile>` instead. The
|
|
77
|
+
shims in `~/.llm-switcher/bin` inject the gateway variables into the tool process itself and
|
|
78
|
+
clear anything stale first. For `agy` the variable is `CLOUD_CODE_URL`, not a proxy.
|
|
79
|
+
- `claude`, `codex` and `agy` on `PATH` already are those shims: invoke them normally, and never
|
|
80
|
+
wrap them in another script.
|
|
81
|
+
|
|
82
|
+
## 4. Operational Rules for AI Agents
|
|
83
|
+
|
|
84
|
+
1. **NEVER edit `~/.claude/settings.json` directly.**
|
|
85
|
+
LLM Switcher uses zero-mutation launcher flags (`active.flag`). Writing custom base URLs or auth tokens into `settings.json` triggers warning banners in Claude Code and risks breaking the user's official subscription. Use `switch` CLI commands instead.
|
|
86
|
+
2. **Check Gateway Health Before Complex Operations:**
|
|
87
|
+
Run `switch status` or call the `switcher_audit` MCP tool to confirm:
|
|
88
|
+
- LLM Switcher is active on port `3456`.
|
|
89
|
+
- The active profile matches the intended CLI target (Claude Code, Codex, or the Antigravity CLI).
|
|
90
|
+
3. **Verify Routing When Errors Occur:**
|
|
91
|
+
If a tool fails with `HTTP 400`, `HTTP 502`, or connection errors:
|
|
92
|
+
- Run `switch doctor` to audit port collisions and environment variables.
|
|
93
|
+
- Inspect recent request logs via `http://127.0.0.1:3456/ui` (Tab 4: Live Inspector) to see if an intermediary tool mangled the payload.
|
package/state.mjs
CHANGED
|
@@ -371,14 +371,6 @@ function getClaudeSlots(p) {
|
|
|
371
371
|
return slots;
|
|
372
372
|
}
|
|
373
373
|
|
|
374
|
-
function getClaude1M(p) {
|
|
375
|
-
const flags = {};
|
|
376
|
-
for (const s of CLAUDE_MODEL_SLOTS) {
|
|
377
|
-
if (p.model1M && Object.hasOwn(p.model1M, s)) flags[s] = p.model1M[s];
|
|
378
|
-
}
|
|
379
|
-
return flags;
|
|
380
|
-
}
|
|
381
|
-
|
|
382
374
|
function getCodexSlots(p) {
|
|
383
375
|
const slots = {};
|
|
384
376
|
for (const s of CODEX_MODEL_SLOTS) {
|
|
@@ -387,17 +379,11 @@ function getCodexSlots(p) {
|
|
|
387
379
|
return slots;
|
|
388
380
|
}
|
|
389
381
|
|
|
390
|
-
function getCodex1M(p) {
|
|
391
|
-
const flags = {};
|
|
392
|
-
for (const s of CODEX_MODEL_SLOTS) {
|
|
393
|
-
if (p.model1M && Object.hasOwn(p.model1M, s)) flags[s] = p.model1M[s];
|
|
394
|
-
}
|
|
395
|
-
return flags;
|
|
396
|
-
}
|
|
397
|
-
|
|
398
382
|
function createClaudeHalf(p) {
|
|
399
383
|
const half = {};
|
|
400
384
|
for (const [k, v] of Object.entries(p)) {
|
|
385
|
+
// model1M is dropped as well: the window is the model's own now, so a flag left in an old
|
|
386
|
+
// config is read by nothing and would only mislead whoever opens the dashboard.
|
|
401
387
|
if (['inFormat', 'blindfold', 'blindfoldPort', 'blindfoldHost', 'blindfoldPrefix', 'defaultModels', 'model1M', 'publicModels', 'codexRoles'].includes(k)) {
|
|
402
388
|
continue;
|
|
403
389
|
}
|
|
@@ -414,14 +400,6 @@ function createClaudeHalf(p) {
|
|
|
414
400
|
}
|
|
415
401
|
if (p.defaultModels && typeof p.defaultModels === 'object') half.defaultModels = dm;
|
|
416
402
|
|
|
417
|
-
const m1m = {};
|
|
418
|
-
if (p.model1M && typeof p.model1M === 'object') {
|
|
419
|
-
for (const slot of CLAUDE_MODEL_SLOTS) {
|
|
420
|
-
if (Object.hasOwn(p.model1M, slot)) m1m[slot] = p.model1M[slot];
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
if (Object.keys(m1m).length > 0) half.model1M = m1m;
|
|
424
|
-
|
|
425
403
|
return half;
|
|
426
404
|
}
|
|
427
405
|
|
|
@@ -459,27 +437,6 @@ function createCodexHalf(p) {
|
|
|
459
437
|
if (Object.hasOwn(srcDm, 'default')) dm.default = srcDm.default;
|
|
460
438
|
if (Object.keys(dm).length > 0) half.defaultModels = dm;
|
|
461
439
|
|
|
462
|
-
const m1m = {};
|
|
463
|
-
const srcM1m = p.model1M && typeof p.model1M === 'object' ? p.model1M : {};
|
|
464
|
-
for (const slot of CODEX_MODEL_SLOTS) {
|
|
465
|
-
if (Object.hasOwn(srcM1m, slot)) m1m[slot] = srcM1m[slot];
|
|
466
|
-
}
|
|
467
|
-
if (!Object.hasOwn(m1m, 'main') && Object.hasOwn(srcM1m, 'opus')) {
|
|
468
|
-
m1m.main = srcM1m.opus;
|
|
469
|
-
}
|
|
470
|
-
if (!Object.hasOwn(m1m, 'review') && Object.hasOwn(srcM1m, 'sonnet')) {
|
|
471
|
-
m1m.review = srcM1m.sonnet;
|
|
472
|
-
}
|
|
473
|
-
if (!Object.hasOwn(m1m, 'subagent')) {
|
|
474
|
-
for (const k of ['fast', 'fallback', 'haiku', 'fable']) {
|
|
475
|
-
if (Object.hasOwn(srcM1m, k)) {
|
|
476
|
-
m1m.subagent = srcM1m[k];
|
|
477
|
-
break;
|
|
478
|
-
}
|
|
479
|
-
}
|
|
480
|
-
}
|
|
481
|
-
if (Object.keys(m1m).length > 0) half.model1M = m1m;
|
|
482
|
-
|
|
483
440
|
return half;
|
|
484
441
|
}
|
|
485
442
|
|
|
@@ -595,11 +552,9 @@ export function migrateConfigInMemory(rawConfig) {
|
|
|
595
552
|
}
|
|
596
553
|
|
|
597
554
|
const claudeSlots = getClaudeSlots(p);
|
|
598
|
-
const claude1M = getClaude1M(p);
|
|
599
555
|
const codexSlots = getCodexSlots(p);
|
|
600
|
-
const
|
|
601
|
-
const
|
|
602
|
-
const hasCodex = Object.keys(codexSlots).length > 0 || Object.keys(codex1M).length > 0
|
|
556
|
+
const hasClaude = Object.keys(claudeSlots).length > 0;
|
|
557
|
+
const hasCodex = Object.keys(codexSlots).length > 0
|
|
603
558
|
|| (Array.isArray(p.publicModels) && p.publicModels.length > 0)
|
|
604
559
|
|| (p.codexRoles && Object.keys(p.codexRoles).length > 0);
|
|
605
560
|
|
|
@@ -1079,15 +1034,6 @@ export function modelForSlot(profile, slot) {
|
|
|
1079
1034
|
return '';
|
|
1080
1035
|
}
|
|
1081
1036
|
|
|
1082
|
-
export function model1MForSlot(profile, slot) {
|
|
1083
|
-
const flags = profile?.model1M || {};
|
|
1084
|
-
if (Object.hasOwn(flags, slot)) return Boolean(flags[slot]);
|
|
1085
|
-
for (const k of legacyKeysForSlot(slot)) {
|
|
1086
|
-
if (Object.hasOwn(flags, k)) return Boolean(flags[k]);
|
|
1087
|
-
}
|
|
1088
|
-
return false;
|
|
1089
|
-
}
|
|
1090
|
-
|
|
1091
1037
|
// ---------------- Codex-facing model names ----------------
|
|
1092
1038
|
//
|
|
1093
1039
|
// Codex must never be handed a switcher-internal name. It can display whatever it
|
|
@@ -1178,32 +1124,28 @@ function codexCatalogTemplate() {
|
|
|
1178
1124
|
|
|
1179
1125
|
// One catalog entry. /v1/models and the OpenAI-Model handshake both use it, so they cannot drift.
|
|
1180
1126
|
// model-catalog.json itself is no longer written (R9: publicModels stays the gateway's mapping table).
|
|
1181
|
-
|
|
1127
|
+
//
|
|
1128
|
+
// The window comes from the model, never from a setting: `win` is what the profile's own list says
|
|
1129
|
+
// ({ context, compact }), and the entry then carries the same pair the official Codex catalog
|
|
1130
|
+
// publishes — the window to compress at as context_window, the whole one as max_context_window. A
|
|
1131
|
+
// model that lists no window keeps the template's.
|
|
1132
|
+
export function codexModelEntry(name, win) {
|
|
1182
1133
|
// The template copies a real OpenAI model. Its Responses Lite and code modes send the tools in a
|
|
1183
1134
|
// form made for OpenAI's own tools, and an upstream then gets none (LS-5).
|
|
1184
1135
|
const { tool_mode, ...template } = structuredClone(codexCatalogTemplate() || {});
|
|
1185
|
-
|
|
1136
|
+
const out = {
|
|
1186
1137
|
...template,
|
|
1187
1138
|
use_responses_lite: false,
|
|
1188
1139
|
slug: name,
|
|
1189
|
-
display_name: name
|
|
1190
|
-
...(is1M ? { context_window: 1000000, max_context_window: 1000000 } : {})
|
|
1140
|
+
display_name: name
|
|
1191
1141
|
};
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
const windows = new Map();
|
|
1198
|
-
for (const [name, is1M] of pairs) {
|
|
1199
|
-
if (!name) continue;
|
|
1200
|
-
if (!windows.has(name) || !is1M) windows.set(name, Boolean(is1M));
|
|
1142
|
+
const context = Number(win?.context) || 0;
|
|
1143
|
+
const compact = Number(win?.compact) || 0;
|
|
1144
|
+
if (context > 0) {
|
|
1145
|
+
out.max_context_window = context;
|
|
1146
|
+
out.context_window = compact > 0 && compact < context ? compact : context;
|
|
1201
1147
|
}
|
|
1202
|
-
return
|
|
1203
|
-
}
|
|
1204
|
-
|
|
1205
|
-
export function publicModelWindows(profile) {
|
|
1206
|
-
return smallestWindows(CODEX_MODEL_SLOTS.map(slot => [codexPublicModel(profile, slot), model1MForSlot(profile, slot)]));
|
|
1148
|
+
return out;
|
|
1207
1149
|
}
|
|
1208
1150
|
|
|
1209
1151
|
export function parsePort(value) {
|
|
@@ -1342,17 +1284,6 @@ function writeOrRemove(file, content) {
|
|
|
1342
1284
|
}
|
|
1343
1285
|
}
|
|
1344
1286
|
|
|
1345
|
-
// The main session model. Haiku is left out on purpose: a haiku-only 1M profile would otherwise
|
|
1346
|
-
// move the main session to Haiku. claude1MTiers reports every tier, haiku included.
|
|
1347
|
-
function claudeTier1M(profile) {
|
|
1348
|
-
const m = profile?.model1M || {};
|
|
1349
|
-
return m.opus ? 'opus[1m]' : m.sonnet ? 'sonnet[1m]' : m.fable ? 'fable[1m]' : null;
|
|
1350
|
-
}
|
|
1351
|
-
|
|
1352
|
-
function anyTier1M(profile) {
|
|
1353
|
-
return modelSlotsForProfile(profile).some(slot => model1MForSlot(profile, slot));
|
|
1354
|
-
}
|
|
1355
|
-
|
|
1356
1287
|
export function primaryModel(profile) {
|
|
1357
1288
|
const slots = modelSlotsForProfile(profile);
|
|
1358
1289
|
for (const slot of slots) {
|
|
@@ -1362,13 +1293,14 @@ export function primaryModel(profile) {
|
|
|
1362
1293
|
return '';
|
|
1363
1294
|
}
|
|
1364
1295
|
|
|
1365
|
-
// What a person needs at launch: which profile took the traffic, the host it goes to, the
|
|
1366
|
-
// maps to
|
|
1367
|
-
//
|
|
1296
|
+
// What a person needs at launch: which profile took the traffic, the host it goes to, and the
|
|
1297
|
+
// model it maps to. Never the API key: a desktop notification reads this line, and so can any
|
|
1298
|
+
// shell. No window goes with it: the tool sizes the session from the model's own window, which the
|
|
1299
|
+
// profile's list reports (see windows.mjs).
|
|
1368
1300
|
function routeLine(tool, key, profile) {
|
|
1369
1301
|
let host = '';
|
|
1370
1302
|
try { host = new URL(profile.baseURL).host; } catch { /* a profile may carry no address yet */ }
|
|
1371
|
-
return [`${tool} -> ${key}`, host, primaryModel(profile)
|
|
1303
|
+
return [`${tool} -> ${key}`, host, primaryModel(profile)]
|
|
1372
1304
|
.filter(Boolean).join(' | ');
|
|
1373
1305
|
}
|
|
1374
1306
|
|
package/switch
CHANGED
|
File without changes
|
package/switch.cmd
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
@echo off
|
|
2
|
-
node "%~dp0switch.mjs" %*
|
|
1
|
+
@echo off
|
|
2
|
+
node "%~dp0switch.mjs" %*
|
package/switch.mjs
CHANGED
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
ROOT_DIR, STATE_DIR, TOOLS, configPath, hasPlaceholder, claudeSettingsPath, paths, certCoversHost, loadConfig, getConfigLoadError, saveConfig,
|
|
8
8
|
resolvePort, parsePort, findProfileKey, getActiveMap, setTargetProfile, activateProfile, deactivateAll,
|
|
9
9
|
applyLaunchState, clearLaunchState, computeLaunchState,
|
|
10
|
-
modelSlotsForProfile, modelForSlot,
|
|
10
|
+
modelSlotsForProfile, modelForSlot, readAdminToken, adminTokenPath, openLog,
|
|
11
11
|
probeGateway, probeBlindfold, ensureBlindfoldCerts, stopRecordedBlindfold, writeDashboardLauncher,
|
|
12
12
|
contractLabSettings, codexPublicModelsWarning,
|
|
13
13
|
getMigrationError, getMigrationCollision, getLastLoadError,
|
|
@@ -342,7 +342,7 @@ function printProfile(profile) {
|
|
|
342
342
|
console.log(`Upstream: ${show(profile.baseURL || '(not set)')}`);
|
|
343
343
|
for (const slot of modelSlotsForProfile(profile)) {
|
|
344
344
|
const model = modelForSlot(profile, slot);
|
|
345
|
-
if (model) console.log(`${(slot[0].toUpperCase() + slot.slice(1) + ':').padEnd(14)}${show(model)}
|
|
345
|
+
if (model) console.log(`${(slot[0].toUpperCase() + slot.slice(1) + ':').padEnd(14)}${show(model)}`);
|
|
346
346
|
}
|
|
347
347
|
}
|
|
348
348
|
|
|
@@ -480,20 +480,19 @@ describe('dashboard, driven in a real browser', { skip: skipReason() }, () => {
|
|
|
480
480
|
assert.equal(models.haiku, 'my-own-model');
|
|
481
481
|
});
|
|
482
482
|
|
|
483
|
-
it('
|
|
483
|
+
it('shows the window each listed model has, and offers no window switch', async () => {
|
|
484
484
|
await act(() => editProfile('intact-claude'));
|
|
485
485
|
await act(() => tab('models'));
|
|
486
486
|
await page.waitFor(`document.querySelector('#models-status')?.dataset.state === 'ready'`, 'the model list');
|
|
487
487
|
await page.type('#p-sonnet', 'up-alpha'); // window 200000
|
|
488
|
-
assert.
|
|
489
|
-
assert.match(await page.text('#p-sonnet-1m-note'), /200K.*under 1M/);
|
|
488
|
+
assert.match(await page.text('#p-sonnet-note'), /200K context/);
|
|
490
489
|
await page.type('#p-opus', 'up-beta'); // window 1000000
|
|
491
|
-
assert.
|
|
492
|
-
await page.
|
|
493
|
-
assert.equal(await page.evaluate(`document.getElementById('p-opus-1m').checked`), true);
|
|
490
|
+
assert.match(await page.text('#p-opus-note'), /1M context/);
|
|
491
|
+
assert.equal(await page.evaluate(`document.querySelectorAll('#slot-fields input[type=checkbox]').length`), 0, 'no window switch is offered');
|
|
494
492
|
await act(() => saveForm());
|
|
495
|
-
|
|
496
|
-
assert.
|
|
493
|
+
const saved = profiles()['intact-claude'];
|
|
494
|
+
assert.equal(saved.defaultModels.opus, 'up-beta');
|
|
495
|
+
assert.equal(saved.model1M, undefined, 'and nothing window-shaped is saved');
|
|
497
496
|
});
|
|
498
497
|
|
|
499
498
|
it('says why the list did not load and loads it on Retry', async () => {
|
|
@@ -508,7 +507,7 @@ describe('dashboard, driven in a real browser', { skip: skipReason() }, () => {
|
|
|
508
507
|
assert.equal(await modelsStatus(), '3 models');
|
|
509
508
|
});
|
|
510
509
|
|
|
511
|
-
it('clears a model slot
|
|
510
|
+
it('clears a model slot the person emptied, and drops the retired 1M flags', async () => {
|
|
512
511
|
const cfg = fx.config();
|
|
513
512
|
cfg.profiles['intact-claude'].model1M = { opus: true };
|
|
514
513
|
fx.writeConfig(cfg);
|
|
@@ -517,14 +516,12 @@ describe('dashboard, driven in a real browser', { skip: skipReason() }, () => {
|
|
|
517
516
|
await act(() => tab('models'));
|
|
518
517
|
await page.waitFor(`document.querySelector('#models-status')?.dataset.state === 'ready'`, 'the model list');
|
|
519
518
|
await page.type('#p-sonnet', '');
|
|
520
|
-
await page.key('Escape'); // the list of suggestions covers the
|
|
521
|
-
await page.click('label:has(#p-opus-1m)');
|
|
522
|
-
assert.equal(await page.evaluate(`document.getElementById('p-opus-1m').checked`), false);
|
|
519
|
+
await page.key('Escape'); // the list of suggestions covers the row below until it closes
|
|
523
520
|
await act(() => saveForm());
|
|
524
521
|
const saved = profiles()['intact-claude'];
|
|
525
522
|
assert.equal(saved.defaultModels.sonnet, undefined, 'the emptied slot came back');
|
|
526
523
|
assert.equal(saved.defaultModels.opus, 'up-opus');
|
|
527
|
-
assert.
|
|
524
|
+
assert.equal(saved.model1M, undefined, 'the retired flags are not written back');
|
|
528
525
|
});
|
|
529
526
|
|
|
530
527
|
it('shows the slots of the tool that the profile serves', async () => {
|