llm-switcher 1.2.7 → 1.2.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.impeccable/hook.cache.json +1 -0
- package/CHANGELOG.md +45 -0
- package/README.md +74 -4
- package/README.vi.md +75 -4
- package/blindfold/blindfold.mjs +2 -1
- package/docs/TOKEN-OPTIMIZER-INTEROP.md +110 -110
- package/docs/cross-platform.md +3 -1
- package/docs/response-matrix.json +1130 -1130
- package/hook-status.mjs +51 -0
- package/notify-route.ps1 +46 -0
- package/package.json +2 -2
- package/plugin.mjs +204 -0
- package/service.mjs +1 -1
- package/shim.mjs +28 -2
- package/skills/llm-switcher/SKILL.md +93 -93
- package/state.mjs +41 -5
- package/switch +0 -0
- package/switch.cmd +2 -2
- package/switch.mjs +53 -2
- package/tests/blindfold.test.mjs +1 -1
- package/tests/cdp.mjs +21 -2
- package/tests/dashboard.e2e.test.mjs +116 -0
- package/tests/gateway.e2e.test.mjs +1 -1
- package/tests/helpers.mjs +24 -24
- package/tests/hook-status.test.mjs +119 -0
- package/tests/live-optimizer-interop.mjs +205 -205
- package/tests/plugin.test.mjs +132 -0
- package/tests/real-user-sim.test.mjs +2 -2
- package/tests/service.test.mjs +2 -0
- package/tests/shim.test.mjs +112 -0
- package/tests/state.test.mjs +130 -5
- package/ui.html +85 -6
|
@@ -1,205 +1,205 @@
|
|
|
1
|
-
// LIVE test: requires a running gateway (switch on) and a real upstream (costs tokens).
|
|
2
|
-
// Offline test, no network needed: npm test
|
|
3
|
-
const GATEWAY_PORT = Number(process.env.LLM_SWITCHER_PORT) || 3456;
|
|
4
|
-
const BASE_URL = `http://127.0.0.1:${GATEWAY_PORT}`;
|
|
5
|
-
|
|
6
|
-
console.log('================================================================');
|
|
7
|
-
console.log(' LLM SWITCHER — TOKEN OPTIMIZER INTEROPERABILITY TEST SUITE ');
|
|
8
|
-
console.log(' Simulating failure modes from Headroom, RTK, and Ponytail ');
|
|
9
|
-
console.log('================================================================\n');
|
|
10
|
-
|
|
11
|
-
let passed = 0;
|
|
12
|
-
let failed = 0;
|
|
13
|
-
|
|
14
|
-
async function runTest(name, fn) {
|
|
15
|
-
process.stdout.write(`[TEST] ${name}... `);
|
|
16
|
-
const t0 = Date.now();
|
|
17
|
-
try {
|
|
18
|
-
const detail = await fn();
|
|
19
|
-
console.log(`PASS (${Date.now() - t0}ms)`);
|
|
20
|
-
if (detail) console.log(` ↳ ${detail}`);
|
|
21
|
-
passed++;
|
|
22
|
-
} catch (err) {
|
|
23
|
-
console.log(`FAIL (${Date.now() - t0}ms)`);
|
|
24
|
-
console.log(` ↳ ERROR: ${err.message}`);
|
|
25
|
-
failed++;
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
// -----------------------------------------------------------------------------
|
|
30
|
-
// TEST 1: Headroom Failure Mode — Orphaned tool_result
|
|
31
|
-
// When Headroom prunes history to save tokens, it drops the assistant tool_use turn.
|
|
32
|
-
// Anthropic strictly throws: 400 invalid_request_error: 'tool_use_id does not correspond to any tool_use'
|
|
33
|
-
// LLM Switcher Healer Engine: Repairs orphaned result into context text -> 200 OK
|
|
34
|
-
// -----------------------------------------------------------------------------
|
|
35
|
-
await runTest('Headroom Simulation: Orphaned tool_result turn', async () => {
|
|
36
|
-
const payload = {
|
|
37
|
-
model: 'claude-opus-4-6',
|
|
38
|
-
max_tokens: 40,
|
|
39
|
-
messages: [
|
|
40
|
-
{ role: 'user', content: 'Context turn before pruning' },
|
|
41
|
-
{
|
|
42
|
-
role: 'user',
|
|
43
|
-
content: [
|
|
44
|
-
{ type: 'tool_result', tool_use_id: 'headroom_pruned_call_99', content: 'Database query result: 42 rows found' },
|
|
45
|
-
{ type: 'text', text: 'Reply: healed successfully' }
|
|
46
|
-
]
|
|
47
|
-
}
|
|
48
|
-
],
|
|
49
|
-
stream: false
|
|
50
|
-
};
|
|
51
|
-
|
|
52
|
-
const res = await fetch(`${BASE_URL}/v1/messages`, {
|
|
53
|
-
method: 'POST',
|
|
54
|
-
headers: { 'Content-Type': 'application/json', 'anthropic-version': '2023-06-01' },
|
|
55
|
-
body: JSON.stringify(payload)
|
|
56
|
-
});
|
|
57
|
-
|
|
58
|
-
if (res.status !== 200) {
|
|
59
|
-
const err = await res.text();
|
|
60
|
-
throw new Error(`Expected HTTP 200, got ${res.status}: ${err}`);
|
|
61
|
-
}
|
|
62
|
-
const json = await res.json();
|
|
63
|
-
const text = json.content?.map(c => c.text).join('') || '';
|
|
64
|
-
return `Healed orphaned tool_result. Response HTTP 200: "${text.slice(0, 50).replace(/\n/g, ' ')}..."`;
|
|
65
|
-
});
|
|
66
|
-
|
|
67
|
-
// -----------------------------------------------------------------------------
|
|
68
|
-
// TEST 2: Headroom Failure Mode — Consecutive User turns (Roles must alternate)
|
|
69
|
-
// When Headroom collapses history, multiple user turns occur back-to-back.
|
|
70
|
-
// Anthropic strictly throws: 400 invalid_request_error: 'roles must alternate'
|
|
71
|
-
// LLM Switcher Healer Engine: Merges consecutive same-role turns -> 200 OK
|
|
72
|
-
// -----------------------------------------------------------------------------
|
|
73
|
-
await runTest('Headroom Simulation: Consecutive User turns (Role alternation violation)', async () => {
|
|
74
|
-
const payload = {
|
|
75
|
-
model: 'claude-opus-4-6',
|
|
76
|
-
max_tokens: 30,
|
|
77
|
-
messages: [
|
|
78
|
-
{ role: 'user', content: 'User message turn 1' },
|
|
79
|
-
{ role: 'user', content: 'User message turn 2 (no assistant in between)' },
|
|
80
|
-
{ role: 'user', content: 'User message turn 3: reply pong' }
|
|
81
|
-
],
|
|
82
|
-
stream: false
|
|
83
|
-
};
|
|
84
|
-
|
|
85
|
-
const res = await fetch(`${BASE_URL}/v1/messages`, {
|
|
86
|
-
method: 'POST',
|
|
87
|
-
headers: { 'Content-Type': 'application/json', 'anthropic-version': '2023-06-01' },
|
|
88
|
-
body: JSON.stringify(payload)
|
|
89
|
-
});
|
|
90
|
-
|
|
91
|
-
if (res.status !== 200) {
|
|
92
|
-
const err = await res.text();
|
|
93
|
-
throw new Error(`Expected HTTP 200, got ${res.status}: ${err}`);
|
|
94
|
-
}
|
|
95
|
-
const json = await res.json();
|
|
96
|
-
return `Merged consecutive turns seamlessly. Stop reason: ${json.stop_reason}`;
|
|
97
|
-
});
|
|
98
|
-
|
|
99
|
-
// -----------------------------------------------------------------------------
|
|
100
|
-
// TEST 3: Headroom / RTK Failure Mode — Stripped Thinking Parameter
|
|
101
|
-
// Optimizer stripped the 'thinking' object to reduce tokens.
|
|
102
|
-
// LLM Switcher Thinking Guard: Detects reasoning model (ag/claude-opus-4-6-thinking)
|
|
103
|
-
// and restores thinking.budget_tokens automatically -> thinking_delta emitted!
|
|
104
|
-
// -----------------------------------------------------------------------------
|
|
105
|
-
await runTest('Thinking Guard: Restoring stripped thinking parameter on reasoning models', async () => {
|
|
106
|
-
const payload = {
|
|
107
|
-
model: 'claude-opus-4-6',
|
|
108
|
-
max_tokens: 200,
|
|
109
|
-
// Note: NO thinking parameter included (simulating optimizer stripping it)
|
|
110
|
-
messages: [
|
|
111
|
-
{ role: 'user', content: 'Solve step by step: what is 17 * 23? Show reasoning.' }
|
|
112
|
-
],
|
|
113
|
-
stream: true
|
|
114
|
-
};
|
|
115
|
-
|
|
116
|
-
const res = await fetch(`${BASE_URL}/v1/messages`, {
|
|
117
|
-
method: 'POST',
|
|
118
|
-
headers: { 'Content-Type': 'application/json', 'anthropic-version': '2023-06-01' },
|
|
119
|
-
body: JSON.stringify(payload)
|
|
120
|
-
});
|
|
121
|
-
|
|
122
|
-
if (res.status !== 200) {
|
|
123
|
-
throw new Error(`Expected HTTP 200, got ${res.status}`);
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
const text = await res.text();
|
|
127
|
-
const hasThinkingDelta = text.includes('thinking_delta');
|
|
128
|
-
const hasSignatureDelta = text.includes('signature_delta');
|
|
129
|
-
const hasTextDelta = text.includes('text_delta');
|
|
130
|
-
|
|
131
|
-
if (!hasThinkingDelta) {
|
|
132
|
-
throw new Error('Thinking Guard failed: thinking_delta was NOT emitted in stream!');
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
return `Automatically restored thinking: thinking_delta=${hasThinkingDelta}, signature_delta=${hasSignatureDelta}, text_delta=${hasTextDelta}`;
|
|
136
|
-
});
|
|
137
|
-
|
|
138
|
-
// -----------------------------------------------------------------------------
|
|
139
|
-
// TEST 4: RTK & Intermediary Headers Passthrough
|
|
140
|
-
// RTK / tracing tools inject headers: x-rtk-version, traceparent, x-request-id.
|
|
141
|
-
// LLM Switcher: Transparently preserves all tracking headers without rejection.
|
|
142
|
-
// -----------------------------------------------------------------------------
|
|
143
|
-
await runTest('RTK Intermediary: Custom headers and traceparent passthrough', async () => {
|
|
144
|
-
const payload = {
|
|
145
|
-
model: 'claude-opus-4-6',
|
|
146
|
-
max_tokens: 20,
|
|
147
|
-
messages: [{ role: 'user', content: 'ping' }],
|
|
148
|
-
stream: false
|
|
149
|
-
};
|
|
150
|
-
|
|
151
|
-
const res = await fetch(`${BASE_URL}/v1/messages`, {
|
|
152
|
-
method: 'POST',
|
|
153
|
-
headers: {
|
|
154
|
-
'Content-Type': 'application/json',
|
|
155
|
-
'anthropic-version': '2023-06-01',
|
|
156
|
-
'x-rtk-version': '0.37.2',
|
|
157
|
-
'x-optimizer-id': 'rtk-cli-hook',
|
|
158
|
-
'traceparent': '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01'
|
|
159
|
-
},
|
|
160
|
-
body: JSON.stringify(payload)
|
|
161
|
-
});
|
|
162
|
-
|
|
163
|
-
if (res.status !== 200) {
|
|
164
|
-
throw new Error(`Expected HTTP 200, got ${res.status}`);
|
|
165
|
-
}
|
|
166
|
-
return `Headers accepted cleanly with HTTP 200 OK`;
|
|
167
|
-
});
|
|
168
|
-
|
|
169
|
-
// -----------------------------------------------------------------------------
|
|
170
|
-
// TEST 5: OpenAI Chat Healer Mode — Orphaned Tool Result in Chat Completions
|
|
171
|
-
// Same test but over /v1/chat/completions: tool role message without prior tool_calls assistant message.
|
|
172
|
-
// OpenAI API strictly throws: 400 'messages with role tool must be a response to a preceding message with tool_calls'
|
|
173
|
-
// LLM Switcher: Converts orphaned tool to user text message -> 200 OK
|
|
174
|
-
// -----------------------------------------------------------------------------
|
|
175
|
-
await runTest('OpenAI Chat Healer: Orphaned role tool without preceding assistant tool_calls', async () => {
|
|
176
|
-
const payload = {
|
|
177
|
-
model: 'ag/gpt-oss-120b-medium',
|
|
178
|
-
max_tokens: 30,
|
|
179
|
-
messages: [
|
|
180
|
-
{ role: 'user', content: 'Turn 1 context' },
|
|
181
|
-
{ role: 'tool', tool_call_id: 'orphaned_chat_call_77', content: 'Tool execution logs: OK' },
|
|
182
|
-
{ role: 'user', content: 'reply pong' }
|
|
183
|
-
],
|
|
184
|
-
stream: false
|
|
185
|
-
};
|
|
186
|
-
|
|
187
|
-
const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
|
|
188
|
-
method: 'POST',
|
|
189
|
-
headers: { 'Content-Type': 'application/json' },
|
|
190
|
-
body: JSON.stringify(payload)
|
|
191
|
-
});
|
|
192
|
-
|
|
193
|
-
if (res.status !== 200) {
|
|
194
|
-
const err = await res.text();
|
|
195
|
-
throw new Error(`Expected HTTP 200, got ${res.status}: ${err}`);
|
|
196
|
-
}
|
|
197
|
-
const json = await res.json();
|
|
198
|
-
return `Chat Healer rescued orphaned tool role. Stop reason: ${json.choices?.[0]?.finish_reason}`;
|
|
199
|
-
});
|
|
200
|
-
|
|
201
|
-
console.log('\n================================================================');
|
|
202
|
-
console.log(` TEST RESULTS: ${passed} PASSED / ${failed} FAILED `);
|
|
203
|
-
console.log('================================================================\n');
|
|
204
|
-
|
|
205
|
-
process.exit(failed ? 1 : 0);
|
|
1
|
+
// LIVE test: requires a running gateway (switch on) and a real upstream (costs tokens).
|
|
2
|
+
// Offline test, no network needed: npm test
|
|
3
|
+
const GATEWAY_PORT = Number(process.env.LLM_SWITCHER_PORT) || 3456;
|
|
4
|
+
const BASE_URL = `http://127.0.0.1:${GATEWAY_PORT}`;
|
|
5
|
+
|
|
6
|
+
console.log('================================================================');
|
|
7
|
+
console.log(' LLM SWITCHER — TOKEN OPTIMIZER INTEROPERABILITY TEST SUITE ');
|
|
8
|
+
console.log(' Simulating failure modes from Headroom, RTK, and Ponytail ');
|
|
9
|
+
console.log('================================================================\n');
|
|
10
|
+
|
|
11
|
+
let passed = 0;
|
|
12
|
+
let failed = 0;
|
|
13
|
+
|
|
14
|
+
async function runTest(name, fn) {
|
|
15
|
+
process.stdout.write(`[TEST] ${name}... `);
|
|
16
|
+
const t0 = Date.now();
|
|
17
|
+
try {
|
|
18
|
+
const detail = await fn();
|
|
19
|
+
console.log(`PASS (${Date.now() - t0}ms)`);
|
|
20
|
+
if (detail) console.log(` ↳ ${detail}`);
|
|
21
|
+
passed++;
|
|
22
|
+
} catch (err) {
|
|
23
|
+
console.log(`FAIL (${Date.now() - t0}ms)`);
|
|
24
|
+
console.log(` ↳ ERROR: ${err.message}`);
|
|
25
|
+
failed++;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// -----------------------------------------------------------------------------
|
|
30
|
+
// TEST 1: Headroom Failure Mode — Orphaned tool_result
|
|
31
|
+
// When Headroom prunes history to save tokens, it drops the assistant tool_use turn.
|
|
32
|
+
// Anthropic strictly throws: 400 invalid_request_error: 'tool_use_id does not correspond to any tool_use'
|
|
33
|
+
// LLM Switcher Healer Engine: Repairs orphaned result into context text -> 200 OK
|
|
34
|
+
// -----------------------------------------------------------------------------
|
|
35
|
+
await runTest('Headroom Simulation: Orphaned tool_result turn', async () => {
|
|
36
|
+
const payload = {
|
|
37
|
+
model: 'claude-opus-4-6',
|
|
38
|
+
max_tokens: 40,
|
|
39
|
+
messages: [
|
|
40
|
+
{ role: 'user', content: 'Context turn before pruning' },
|
|
41
|
+
{
|
|
42
|
+
role: 'user',
|
|
43
|
+
content: [
|
|
44
|
+
{ type: 'tool_result', tool_use_id: 'headroom_pruned_call_99', content: 'Database query result: 42 rows found' },
|
|
45
|
+
{ type: 'text', text: 'Reply: healed successfully' }
|
|
46
|
+
]
|
|
47
|
+
}
|
|
48
|
+
],
|
|
49
|
+
stream: false
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const res = await fetch(`${BASE_URL}/v1/messages`, {
|
|
53
|
+
method: 'POST',
|
|
54
|
+
headers: { 'Content-Type': 'application/json', 'anthropic-version': '2023-06-01' },
|
|
55
|
+
body: JSON.stringify(payload)
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
if (res.status !== 200) {
|
|
59
|
+
const err = await res.text();
|
|
60
|
+
throw new Error(`Expected HTTP 200, got ${res.status}: ${err}`);
|
|
61
|
+
}
|
|
62
|
+
const json = await res.json();
|
|
63
|
+
const text = json.content?.map(c => c.text).join('') || '';
|
|
64
|
+
return `Healed orphaned tool_result. Response HTTP 200: "${text.slice(0, 50).replace(/\n/g, ' ')}..."`;
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
// -----------------------------------------------------------------------------
|
|
68
|
+
// TEST 2: Headroom Failure Mode — Consecutive User turns (Roles must alternate)
|
|
69
|
+
// When Headroom collapses history, multiple user turns occur back-to-back.
|
|
70
|
+
// Anthropic strictly throws: 400 invalid_request_error: 'roles must alternate'
|
|
71
|
+
// LLM Switcher Healer Engine: Merges consecutive same-role turns -> 200 OK
|
|
72
|
+
// -----------------------------------------------------------------------------
|
|
73
|
+
await runTest('Headroom Simulation: Consecutive User turns (Role alternation violation)', async () => {
|
|
74
|
+
const payload = {
|
|
75
|
+
model: 'claude-opus-4-6',
|
|
76
|
+
max_tokens: 30,
|
|
77
|
+
messages: [
|
|
78
|
+
{ role: 'user', content: 'User message turn 1' },
|
|
79
|
+
{ role: 'user', content: 'User message turn 2 (no assistant in between)' },
|
|
80
|
+
{ role: 'user', content: 'User message turn 3: reply pong' }
|
|
81
|
+
],
|
|
82
|
+
stream: false
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
const res = await fetch(`${BASE_URL}/v1/messages`, {
|
|
86
|
+
method: 'POST',
|
|
87
|
+
headers: { 'Content-Type': 'application/json', 'anthropic-version': '2023-06-01' },
|
|
88
|
+
body: JSON.stringify(payload)
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
if (res.status !== 200) {
|
|
92
|
+
const err = await res.text();
|
|
93
|
+
throw new Error(`Expected HTTP 200, got ${res.status}: ${err}`);
|
|
94
|
+
}
|
|
95
|
+
const json = await res.json();
|
|
96
|
+
return `Merged consecutive turns seamlessly. Stop reason: ${json.stop_reason}`;
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
// -----------------------------------------------------------------------------
|
|
100
|
+
// TEST 3: Headroom / RTK Failure Mode — Stripped Thinking Parameter
|
|
101
|
+
// Optimizer stripped the 'thinking' object to reduce tokens.
|
|
102
|
+
// LLM Switcher Thinking Guard: Detects reasoning model (ag/claude-opus-4-6-thinking)
|
|
103
|
+
// and restores thinking.budget_tokens automatically -> thinking_delta emitted!
|
|
104
|
+
// -----------------------------------------------------------------------------
|
|
105
|
+
await runTest('Thinking Guard: Restoring stripped thinking parameter on reasoning models', async () => {
|
|
106
|
+
const payload = {
|
|
107
|
+
model: 'claude-opus-4-6',
|
|
108
|
+
max_tokens: 200,
|
|
109
|
+
// Note: NO thinking parameter included (simulating optimizer stripping it)
|
|
110
|
+
messages: [
|
|
111
|
+
{ role: 'user', content: 'Solve step by step: what is 17 * 23? Show reasoning.' }
|
|
112
|
+
],
|
|
113
|
+
stream: true
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
const res = await fetch(`${BASE_URL}/v1/messages`, {
|
|
117
|
+
method: 'POST',
|
|
118
|
+
headers: { 'Content-Type': 'application/json', 'anthropic-version': '2023-06-01' },
|
|
119
|
+
body: JSON.stringify(payload)
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
if (res.status !== 200) {
|
|
123
|
+
throw new Error(`Expected HTTP 200, got ${res.status}`);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const text = await res.text();
|
|
127
|
+
const hasThinkingDelta = text.includes('thinking_delta');
|
|
128
|
+
const hasSignatureDelta = text.includes('signature_delta');
|
|
129
|
+
const hasTextDelta = text.includes('text_delta');
|
|
130
|
+
|
|
131
|
+
if (!hasThinkingDelta) {
|
|
132
|
+
throw new Error('Thinking Guard failed: thinking_delta was NOT emitted in stream!');
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
return `Automatically restored thinking: thinking_delta=${hasThinkingDelta}, signature_delta=${hasSignatureDelta}, text_delta=${hasTextDelta}`;
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
// -----------------------------------------------------------------------------
|
|
139
|
+
// TEST 4: RTK & Intermediary Headers Passthrough
|
|
140
|
+
// RTK / tracing tools inject headers: x-rtk-version, traceparent, x-request-id.
|
|
141
|
+
// LLM Switcher: Transparently preserves all tracking headers without rejection.
|
|
142
|
+
// -----------------------------------------------------------------------------
|
|
143
|
+
await runTest('RTK Intermediary: Custom headers and traceparent passthrough', async () => {
|
|
144
|
+
const payload = {
|
|
145
|
+
model: 'claude-opus-4-6',
|
|
146
|
+
max_tokens: 20,
|
|
147
|
+
messages: [{ role: 'user', content: 'ping' }],
|
|
148
|
+
stream: false
|
|
149
|
+
};
|
|
150
|
+
|
|
151
|
+
const res = await fetch(`${BASE_URL}/v1/messages`, {
|
|
152
|
+
method: 'POST',
|
|
153
|
+
headers: {
|
|
154
|
+
'Content-Type': 'application/json',
|
|
155
|
+
'anthropic-version': '2023-06-01',
|
|
156
|
+
'x-rtk-version': '0.37.2',
|
|
157
|
+
'x-optimizer-id': 'rtk-cli-hook',
|
|
158
|
+
'traceparent': '00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01'
|
|
159
|
+
},
|
|
160
|
+
body: JSON.stringify(payload)
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
if (res.status !== 200) {
|
|
164
|
+
throw new Error(`Expected HTTP 200, got ${res.status}`);
|
|
165
|
+
}
|
|
166
|
+
return `Headers accepted cleanly with HTTP 200 OK`;
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
// -----------------------------------------------------------------------------
|
|
170
|
+
// TEST 5: OpenAI Chat Healer Mode — Orphaned Tool Result in Chat Completions
|
|
171
|
+
// Same test but over /v1/chat/completions: tool role message without prior tool_calls assistant message.
|
|
172
|
+
// OpenAI API strictly throws: 400 'messages with role tool must be a response to a preceding message with tool_calls'
|
|
173
|
+
// LLM Switcher: Converts orphaned tool to user text message -> 200 OK
|
|
174
|
+
// -----------------------------------------------------------------------------
|
|
175
|
+
await runTest('OpenAI Chat Healer: Orphaned role tool without preceding assistant tool_calls', async () => {
|
|
176
|
+
const payload = {
|
|
177
|
+
model: 'ag/gpt-oss-120b-medium',
|
|
178
|
+
max_tokens: 30,
|
|
179
|
+
messages: [
|
|
180
|
+
{ role: 'user', content: 'Turn 1 context' },
|
|
181
|
+
{ role: 'tool', tool_call_id: 'orphaned_chat_call_77', content: 'Tool execution logs: OK' },
|
|
182
|
+
{ role: 'user', content: 'reply pong' }
|
|
183
|
+
],
|
|
184
|
+
stream: false
|
|
185
|
+
};
|
|
186
|
+
|
|
187
|
+
const res = await fetch(`${BASE_URL}/v1/chat/completions`, {
|
|
188
|
+
method: 'POST',
|
|
189
|
+
headers: { 'Content-Type': 'application/json' },
|
|
190
|
+
body: JSON.stringify(payload)
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
if (res.status !== 200) {
|
|
194
|
+
const err = await res.text();
|
|
195
|
+
throw new Error(`Expected HTTP 200, got ${res.status}: ${err}`);
|
|
196
|
+
}
|
|
197
|
+
const json = await res.json();
|
|
198
|
+
return `Chat Healer rescued orphaned tool role. Stop reason: ${json.choices?.[0]?.finish_reason}`;
|
|
199
|
+
});
|
|
200
|
+
|
|
201
|
+
console.log('\n================================================================');
|
|
202
|
+
console.log(` TEST RESULTS: ${passed} PASSED / ${failed} FAILED `);
|
|
203
|
+
console.log('================================================================\n');
|
|
204
|
+
|
|
205
|
+
process.exit(failed ? 1 : 0);
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// Installing the launch hook into the two coding tools, without editing any configuration of theirs.
|
|
2
|
+
//
|
|
3
|
+
// Claude Code loads any folder under a skills directory that holds `.claude-plugin/plugin.json` as
|
|
4
|
+
// a plugin, on the next session, with no marketplace and no install step. Codex loads
|
|
5
|
+
// `$CODEX_HOME/hooks.json` by itself. So both tools get the hook from a file of their own, and
|
|
6
|
+
// `settings.json` and `config.toml` are never touched.
|
|
7
|
+
import { test } from 'node:test';
|
|
8
|
+
import assert from 'node:assert/strict';
|
|
9
|
+
import fs from 'node:fs';
|
|
10
|
+
import os from 'node:os';
|
|
11
|
+
import path from 'node:path';
|
|
12
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
13
|
+
|
|
14
|
+
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
15
|
+
const { renderClaudePlugin, renderClaudeHooks, renderCodexHooks, installPlugin, uninstallPlugin, pluginStatus } =
|
|
16
|
+
await import(pathToFileURL(path.join(ROOT, 'plugin.mjs')).href);
|
|
17
|
+
|
|
18
|
+
function dirs(t) {
|
|
19
|
+
const home = fs.mkdtempSync(path.join(os.tmpdir(), 'llmsw-plug-'));
|
|
20
|
+
t.after(() => fs.rmSync(home, { recursive: true, force: true }));
|
|
21
|
+
const claudeSkillsDir = path.join(home, '.claude', 'skills');
|
|
22
|
+
const codexHome = path.join(home, '.codex');
|
|
23
|
+
fs.mkdirSync(claudeSkillsDir, { recursive: true });
|
|
24
|
+
fs.mkdirSync(codexHome, { recursive: true });
|
|
25
|
+
return { home, claudeSkillsDir, codexHome, opts: { claudeSkillsDir, codexHome } };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const readJson = (f) => JSON.parse(fs.readFileSync(f, 'utf8'));
|
|
29
|
+
|
|
30
|
+
test('the Claude plugin manifest and hooks name the hook script and the tool claude', () => {
|
|
31
|
+
const manifest = JSON.parse(renderClaudePlugin());
|
|
32
|
+
assert.equal(typeof manifest.name, 'string');
|
|
33
|
+
assert.ok(manifest.name.length > 0, 'a plugin needs a name, it becomes the skills-dir id');
|
|
34
|
+
assert.ok(manifest.version, 'and a version');
|
|
35
|
+
const hooks = JSON.parse(renderClaudeHooks());
|
|
36
|
+
const entry = hooks.hooks.SessionStart[0].hooks[0];
|
|
37
|
+
assert.equal(entry.type, 'command');
|
|
38
|
+
assert.match(JSON.stringify(entry), /hook-status\.mjs/);
|
|
39
|
+
assert.match(JSON.stringify(entry), /"claude"/, 'the claude plugin asks about claude');
|
|
40
|
+
assert.doesNotMatch(JSON.stringify(entry), /codex/);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test('the Codex hooks file asks about codex and matches a fresh start or a resume', () => {
|
|
44
|
+
const hooks = JSON.parse(renderCodexHooks());
|
|
45
|
+
const group = hooks.hooks.SessionStart[0];
|
|
46
|
+
assert.match(group.matcher, /startup/, 'the first open of a session');
|
|
47
|
+
assert.match(group.matcher, /resume/, 'and a resumed session, which is the bypass this exists for');
|
|
48
|
+
const entry = group.hooks[0];
|
|
49
|
+
assert.equal(entry.type, 'command');
|
|
50
|
+
assert.match(entry.command, /hook-status\.mjs/);
|
|
51
|
+
assert.match(entry.command, /codex/);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test('install writes a plugin for Claude Code and a hooks file for Codex, and repeats cleanly', (t) => {
|
|
55
|
+
const { claudeSkillsDir, codexHome, opts } = dirs(t);
|
|
56
|
+
const first = installPlugin(opts);
|
|
57
|
+
assert.deepEqual(first.failed, [], 'nothing failed');
|
|
58
|
+
assert.deepEqual(first.installed.sort(), ['claude', 'codex']);
|
|
59
|
+
|
|
60
|
+
const manifest = path.join(claudeSkillsDir, 'llm-switcher-status', '.claude-plugin', 'plugin.json');
|
|
61
|
+
assert.ok(fs.existsSync(manifest), 'the manifest is what makes the folder a plugin');
|
|
62
|
+
assert.ok(fs.existsSync(path.join(claudeSkillsDir, 'llm-switcher-status', 'hooks', 'hooks.json')));
|
|
63
|
+
assert.ok(fs.existsSync(path.join(codexHome, 'hooks.json')));
|
|
64
|
+
|
|
65
|
+
// The whole point: neither tool's own configuration file is created or changed.
|
|
66
|
+
assert.equal(fs.existsSync(path.join(codexHome, 'config.toml')), false, 'config.toml is never written');
|
|
67
|
+
|
|
68
|
+
const second = installPlugin(opts);
|
|
69
|
+
assert.deepEqual(second.failed, [], 'a second install is a no-op that still succeeds');
|
|
70
|
+
assert.equal(JSON.parse(fs.readFileSync(path.join(codexHome, 'hooks.json'), 'utf8')).hooks.SessionStart.length, 1,
|
|
71
|
+
'and it does not add a second copy of our entry');
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test('install keeps a Codex hook that belongs to someone else', (t) => {
|
|
75
|
+
const { codexHome, opts } = dirs(t);
|
|
76
|
+
fs.writeFileSync(path.join(codexHome, 'hooks.json'), JSON.stringify({
|
|
77
|
+
description: 'the rule gate of this machine',
|
|
78
|
+
hooks: {
|
|
79
|
+
PreToolUse: [{ matcher: '^Bash$', hooks: [{ type: 'command', command: 'node rule-gate.js' }] }],
|
|
80
|
+
SessionStart: [{ matcher: 'startup', hooks: [{ type: 'command', command: 'node someone-else.js' }] }]
|
|
81
|
+
}
|
|
82
|
+
}, null, 2));
|
|
83
|
+
|
|
84
|
+
installPlugin(opts);
|
|
85
|
+
const after = readJson(path.join(codexHome, 'hooks.json'));
|
|
86
|
+
assert.equal(after.hooks.PreToolUse.length, 1, 'another event is untouched');
|
|
87
|
+
assert.match(JSON.stringify(after.hooks.PreToolUse), /rule-gate/);
|
|
88
|
+
const commands = after.hooks.SessionStart.flatMap(g => g.hooks.map(h => h.command));
|
|
89
|
+
assert.ok(commands.some(c => /someone-else\.js/.test(c)), 'the other SessionStart hook survives');
|
|
90
|
+
assert.ok(commands.some(c => /hook-status\.mjs/.test(c)), 'and ours is added beside it');
|
|
91
|
+
|
|
92
|
+
const removed = uninstallPlugin(opts);
|
|
93
|
+
assert.deepEqual(removed.failed, []);
|
|
94
|
+
const end = readJson(path.join(codexHome, 'hooks.json'));
|
|
95
|
+
const endCommands = JSON.stringify(end);
|
|
96
|
+
assert.match(endCommands, /someone-else\.js/, 'uninstall leaves the other hook in place');
|
|
97
|
+
assert.match(endCommands, /rule-gate/);
|
|
98
|
+
assert.doesNotMatch(endCommands, /hook-status\.mjs/, 'and takes only ours away');
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test('install refuses a Codex hooks file it cannot parse, and changes nothing', (t) => {
|
|
102
|
+
const { codexHome, opts } = dirs(t);
|
|
103
|
+
const f = path.join(codexHome, 'hooks.json');
|
|
104
|
+
fs.writeFileSync(f, '{ not json at all');
|
|
105
|
+
const r = installPlugin(opts);
|
|
106
|
+
assert.equal(fs.readFileSync(f, 'utf8'), '{ not json at all', 'a file that may hold real hooks is never overwritten');
|
|
107
|
+
assert.ok(r.failed.some(x => x.tool === 'codex'), 'and the refusal is reported');
|
|
108
|
+
assert.ok(r.installed.includes('claude'), 'while the other tool still gets its plugin');
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
test('install leaves a folder that is not our plugin alone', (t) => {
|
|
112
|
+
const { claudeSkillsDir, opts } = dirs(t);
|
|
113
|
+
const dir = path.join(claudeSkillsDir, 'llm-switcher-status');
|
|
114
|
+
fs.mkdirSync(path.join(dir, '.claude-plugin'), { recursive: true });
|
|
115
|
+
fs.writeFileSync(path.join(dir, '.claude-plugin', 'plugin.json'), JSON.stringify({ name: 'someone-elses-work' }));
|
|
116
|
+
const r = installPlugin(opts);
|
|
117
|
+
assert.match(fs.readFileSync(path.join(dir, '.claude-plugin', 'plugin.json'), 'utf8'), /someone-elses-work/);
|
|
118
|
+
assert.ok(r.failed.some(x => x.tool === 'claude'));
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test('status reports each tool, and says plainly when nothing is installed', (t) => {
|
|
122
|
+
const { opts } = dirs(t);
|
|
123
|
+
const before = pluginStatus(opts);
|
|
124
|
+
assert.equal(before.claude.installed, false);
|
|
125
|
+
assert.equal(before.codex.installed, false);
|
|
126
|
+
installPlugin(opts);
|
|
127
|
+
const after = pluginStatus(opts);
|
|
128
|
+
assert.equal(after.claude.installed, true);
|
|
129
|
+
assert.equal(after.codex.installed, true);
|
|
130
|
+
assert.match(after.claude.path, /llm-switcher-status/);
|
|
131
|
+
assert.match(after.codex.path, /hooks\.json$/);
|
|
132
|
+
});
|
|
@@ -324,7 +324,7 @@ test('Real User Sim 3: Gemini / Vertex healing of empty & complex tool schemas',
|
|
|
324
324
|
await post('/api/switch', { tool: 'claude', profile: 'chat' });
|
|
325
325
|
});
|
|
326
326
|
|
|
327
|
-
test('Real User Sim 4: Codex WebSocket turns (warmup -> prompt -> apply_patch -> continue)', async () => {
|
|
327
|
+
test('Real User Sim 4: Codex WebSocket turns (warmup -> prompt -> apply_patch -> continue)', { skip: typeof WebSocket === 'undefined' && 'global WebSocket needs Node 22+' }, async () => {
|
|
328
328
|
const ws = new WebSocket(`ws://127.0.0.1:${proxyPort}/v1/responses`);
|
|
329
329
|
const messages = [];
|
|
330
330
|
|
|
@@ -387,7 +387,7 @@ test('Real User Sim 4: Codex WebSocket turns (warmup -> prompt -> apply_patch ->
|
|
|
387
387
|
ws.close();
|
|
388
388
|
});
|
|
389
389
|
|
|
390
|
-
test('Real User Sim 5: High-concurrency dual-tool execution (Claude streaming + Codex WS simultaneously)', async () => {
|
|
390
|
+
test('Real User Sim 5: High-concurrency dual-tool execution (Claude streaming + Codex WS simultaneously)', { skip: typeof WebSocket === 'undefined' && 'global WebSocket needs Node 22+' }, async () => {
|
|
391
391
|
const ws = new WebSocket(`ws://127.0.0.1:${proxyPort}/v1/responses`);
|
|
392
392
|
await new Promise(r => { ws.onopen = r; });
|
|
393
393
|
const wsMessages = [];
|
package/tests/service.test.mjs
CHANGED
|
@@ -16,6 +16,8 @@ const HAS_XMLLINT = (() => { try { execFileSync('xmllint', ['--version'], { stdi
|
|
|
16
16
|
test('serviceEnv passes only the variables that change where the gateway reads and writes', () => {
|
|
17
17
|
const env = serviceEnv({ CLAUDE_CONFIG_DIR: '/c', LLM_SWITCHER_CONFIG: '/cfg.json', LLM_SWITCHER_BLINDFOLD_CERTS: '', HOME: '/h', PATH: '/bin' });
|
|
18
18
|
assert.deepEqual(env, [['CLAUDE_CONFIG_DIR', '/c'], ['LLM_SWITCHER_CONFIG', '/cfg.json']]);
|
|
19
|
+
// The data folder override must reach the service too, or it reads another config.json.
|
|
20
|
+
assert.deepEqual(serviceEnv({ LLM_SWITCHER_HOME: '/data' }), [['LLM_SWITCHER_HOME', '/data']]);
|
|
19
21
|
});
|
|
20
22
|
|
|
21
23
|
test('systemd unit quotes the command and carries the environment', () => {
|