modelmix 5.1.19 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.gitignore +138 -0
- package/README.md +45 -3
- package/demo/benchmark.js +83 -0
- package/demo/package.json +2 -0
- package/demo/prompts/story.txt +35 -0
- package/demo/prompts/template-engine.txt +41 -0
- package/demo/short.js +3 -0
- package/effort.js +4 -1
- package/index.d.ts +9 -0
- package/index.js +33 -5
- package/lib/model-chain.js +2 -2
- package/lib/parse-json-response.js +14 -0
- package/lib/providers/anthropic.js +3 -1
- package/lib/providers/base.js +65 -17
- package/lib/providers/google.js +13 -2
- package/lib/providers/openai-compatible.js +36 -0
- package/lib/providers/openai-options.js +1 -1
- package/lib/providers/openai.js +5 -1
- package/lib/token-usage.js +10 -4
- package/package.json +5 -5
- package/plugins/benchmark/index.d.ts +112 -0
- package/plugins/benchmark/index.js +575 -0
- package/plugins/benchmark/test/benchmark.test.js +518 -0
- package/skills/modelmix/SKILL.md +14 -5
- package/test/deepseek.test.js +273 -1
- package/test/effort.test.js +9 -0
- package/test/fallback.test.js +97 -5
- package/test/google.test.js +55 -0
- package/test/history.test.js +7 -4
- package/test/json.test.js +29 -1
- package/test/public-api.test.js +2 -0
- package/test/tokens.test.js +83 -3
- package/RLM_PLUGIN_SPEC.md +0 -465
- package/demo/package-lock.json +0 -516
package/test/deepseek.test.js
CHANGED
|
@@ -1,7 +1,279 @@
|
|
|
1
1
|
const { expect } = require('chai');
|
|
2
|
-
const
|
|
2
|
+
const nock = require('nock');
|
|
3
|
+
const { ModelMix, MixDeepSeek, MixFireworks, MixOpenRouter } = require('../index.js');
|
|
3
4
|
|
|
4
5
|
describe('DeepSeek Model Registration Tests', () => {
|
|
6
|
+
it('registers V4 Pro 0813 through OpenRouter and preserves caller settings', () => {
|
|
7
|
+
const model = ModelMix.new();
|
|
8
|
+
expect(model.deepseekPro({
|
|
9
|
+
options: { temperature: 0.5 },
|
|
10
|
+
config: { apiKey: 'test-key', effort: 100 }
|
|
11
|
+
})).to.equal(model);
|
|
12
|
+
|
|
13
|
+
expect(model.models).to.have.length(1);
|
|
14
|
+
expect(model.models[0].key).to.equal('deepseek/deepseek-v4-pro-0813');
|
|
15
|
+
expect(model.models[0].provider).to.be.instanceOf(MixOpenRouter);
|
|
16
|
+
expect(model.models[0].provider.options.temperature).to.equal(0.5);
|
|
17
|
+
expect(model.models[0].provider.config.effort).to.equal(100);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
describe('DeepSeek V4 Pro 0813 requests', () => {
|
|
21
|
+
afterEach(() => nock.cleanAll());
|
|
22
|
+
|
|
23
|
+
for (const [effort, level] of [[0, undefined], [20, 'low'], [60, 'high'], [100, 'max'], [-1, undefined]]) {
|
|
24
|
+
it(`sends the exact model ID through chain() at effort ${effort} and accounts for cached input`, async () => {
|
|
25
|
+
const scope = nock('https://openrouter.ai')
|
|
26
|
+
.post('/api/v1/chat/completions', body => {
|
|
27
|
+
expect(body.model).to.equal('deepseek/deepseek-v4-pro-0813');
|
|
28
|
+
expect(body.reasoning_effort).to.equal(level);
|
|
29
|
+
expect(body.thinking).to.deep.equal(effort === -1
|
|
30
|
+
? undefined
|
|
31
|
+
: { type: effort === 0 ? 'disabled' : 'enabled' });
|
|
32
|
+
return true;
|
|
33
|
+
})
|
|
34
|
+
.reply(200, {
|
|
35
|
+
model: 'deepseek/deepseek-v4-pro-0813',
|
|
36
|
+
choices: [{ message: { role: 'assistant', content: 'ok' } }],
|
|
37
|
+
usage: {
|
|
38
|
+
prompt_tokens: 1000,
|
|
39
|
+
completion_tokens: 100,
|
|
40
|
+
prompt_tokens_details: { cached_tokens: 400 }
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
const model = ModelMix.new().chain(`deepseekPro@${effort}`).addText('Hello');
|
|
44
|
+
|
|
45
|
+
expect(await model.message()).to.equal('ok');
|
|
46
|
+
expect(scope.isDone()).to.equal(true);
|
|
47
|
+
expect(model.lastRaw.tokens).to.include({ input: 1000, cached: 400, output: 100 });
|
|
48
|
+
expect(model.lastRaw.tokens.cost).to.be.closeTo(0.000545952, 1e-12);
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it('requires native credentials and accepts DEEPSEEK_API_KEY', () => {
|
|
54
|
+
const originalApiKey = process.env.DEEPSEEK_API_KEY;
|
|
55
|
+
try {
|
|
56
|
+
delete process.env.DEEPSEEK_API_KEY;
|
|
57
|
+
expect(() => new MixDeepSeek()).to.throw(/DEEPSEEK_API_KEY/);
|
|
58
|
+
process.env.DEEPSEEK_API_KEY = 'native-test-key';
|
|
59
|
+
const model = ModelMix.new().deepseekV41Flash({ mix: { deepseek: true, openrouter: false } });
|
|
60
|
+
expect(model.models).to.have.length(1);
|
|
61
|
+
expect(model.models[0].key).to.equal('deepseek-flash');
|
|
62
|
+
expect(model.models[0].provider.config.apiKey).to.equal('native-test-key');
|
|
63
|
+
} finally {
|
|
64
|
+
if (originalApiKey === undefined) delete process.env.DEEPSEEK_API_KEY;
|
|
65
|
+
else process.env.DEEPSEEK_API_KEY = originalApiKey;
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
it('registers DeepSeek V4.1 Flash through OpenRouter and preserves caller settings', () => {
|
|
70
|
+
const model = ModelMix.new();
|
|
71
|
+
expect(model.deepseekV41Flash({
|
|
72
|
+
options: { temperature: 0.5 },
|
|
73
|
+
config: { effort: 100 }
|
|
74
|
+
})).to.equal(model);
|
|
75
|
+
|
|
76
|
+
expect(model.models).to.have.length(1);
|
|
77
|
+
expect(model.models[0].key).to.equal('deepseek/deepseek-v4.1-flash');
|
|
78
|
+
expect(model.models[0].provider).to.be.instanceOf(MixOpenRouter);
|
|
79
|
+
expect(model.models[0].provider.options.temperature).to.equal(0.5);
|
|
80
|
+
expect(model.models[0].provider.config.effort).to.equal(100);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
describe('DeepSeek V4.1 Flash requests', () => {
|
|
84
|
+
afterEach(() => nock.cleanAll());
|
|
85
|
+
|
|
86
|
+
it('falls back from native DeepSeek through Fireworks to OpenRouter', async () => {
|
|
87
|
+
const native = nock('https://api.deepseek.com')
|
|
88
|
+
.post('/chat/completions', body => body.model === 'deepseek-flash')
|
|
89
|
+
.reply(503, { error: { message: 'Unavailable' } });
|
|
90
|
+
const fireworks = nock('https://api.fireworks.ai')
|
|
91
|
+
.post('/inference/v1/chat/completions')
|
|
92
|
+
.reply(503, { error: { message: 'Unavailable' } });
|
|
93
|
+
const openrouter = nock('https://openrouter.ai')
|
|
94
|
+
.post('/api/v1/chat/completions')
|
|
95
|
+
.reply(200, { choices: [{ message: { content: 'fallback' } }] });
|
|
96
|
+
const model = ModelMix.new({ config: { retry: { retries: 0 } } })
|
|
97
|
+
.deepseekV41Flash({
|
|
98
|
+
mix: { deepseek: true, fireworks: true, openrouter: true },
|
|
99
|
+
config: { apiKey: 'test-key' }
|
|
100
|
+
}).addText('Hello');
|
|
101
|
+
|
|
102
|
+
expect(await model.message()).to.equal('fallback');
|
|
103
|
+
expect(native.isDone()).to.equal(true);
|
|
104
|
+
expect(fireworks.isDone()).to.equal(true);
|
|
105
|
+
expect(openrouter.isDone()).to.equal(true);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
for (const [effort, level] of [[0, undefined], [20, 'low'], [60, 'high'], [100, 'max'], [-1, undefined]]) {
|
|
109
|
+
it(`sends native DeepSeek effort ${effort} and reads native cache usage`, async () => {
|
|
110
|
+
const scope = nock('https://api.deepseek.com', {
|
|
111
|
+
reqheaders: { authorization: 'Bearer native-test-key' }
|
|
112
|
+
})
|
|
113
|
+
.post('/chat/completions', body => {
|
|
114
|
+
expect(body.model).to.equal('deepseek-flash');
|
|
115
|
+
expect(body.reasoning_effort).to.equal(level);
|
|
116
|
+
expect(body.thinking).to.deep.equal(effort === -1
|
|
117
|
+
? undefined
|
|
118
|
+
: { type: effort === 0 ? 'disabled' : 'enabled' });
|
|
119
|
+
return true;
|
|
120
|
+
})
|
|
121
|
+
.reply(200, {
|
|
122
|
+
choices: [{ message: { role: 'assistant', content: 'ok', reasoning_content: 'Reasoning' } }],
|
|
123
|
+
usage: { prompt_tokens: 1000, completion_tokens: 100, prompt_cache_hit_tokens: 400 }
|
|
124
|
+
});
|
|
125
|
+
const model = ModelMix.new().deepseekV41Flash({
|
|
126
|
+
mix: { deepseek: true, openrouter: false },
|
|
127
|
+
config: { apiKey: 'native-test-key', effort }
|
|
128
|
+
}).addText('Hello');
|
|
129
|
+
|
|
130
|
+
expect(model.models).to.have.length(1);
|
|
131
|
+
expect(model.models[0].provider).to.be.instanceOf(MixDeepSeek);
|
|
132
|
+
expect(await model.message()).to.equal('ok');
|
|
133
|
+
expect(scope.isDone()).to.equal(true);
|
|
134
|
+
expect(model.lastRaw.tokens).to.include({ input: 1000, cached: 400, output: 100 });
|
|
135
|
+
expect(model.lastRaw.tokens.cost).to.be.closeTo(0.0003024, 1e-12);
|
|
136
|
+
});
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
it('preserves native reasoning across tool continuation and later turns', async () => {
|
|
140
|
+
const assistantMessage = {
|
|
141
|
+
role: 'assistant', content: null, reasoning_content: 'Use the lookup tool.',
|
|
142
|
+
tool_calls: [{ id: 'call_lookup', type: 'function', function: { name: 'lookup', arguments: '{}' } }]
|
|
143
|
+
};
|
|
144
|
+
const scope = nock('https://api.deepseek.com')
|
|
145
|
+
.post('/chat/completions')
|
|
146
|
+
.reply(200, { choices: [{ message: assistantMessage }] })
|
|
147
|
+
.post('/chat/completions', body => {
|
|
148
|
+
expect(body.messages.find(message => message.role === 'assistant')).to.deep.equal(assistantMessage);
|
|
149
|
+
expect(body.messages.find(message => message.role === 'tool')).to.include({ tool_call_id: 'call_lookup' });
|
|
150
|
+
return true;
|
|
151
|
+
})
|
|
152
|
+
.reply(200, { choices: [{ message: { role: 'assistant', content: 'Found', reasoning_content: 'Lookup complete.' } }] })
|
|
153
|
+
.post('/chat/completions', body => {
|
|
154
|
+
expect(body.messages.filter(message => message.role === 'assistant').map(message => message.reasoning_content))
|
|
155
|
+
.to.deep.equal(['Use the lookup tool.', 'Lookup complete.']);
|
|
156
|
+
return true;
|
|
157
|
+
})
|
|
158
|
+
.reply(200, { choices: [{ message: { role: 'assistant', content: 'Done' } }] });
|
|
159
|
+
const model = ModelMix.new().deepseekV41Flash({
|
|
160
|
+
mix: { deepseek: true, openrouter: false },
|
|
161
|
+
config: { apiKey: 'native-test-key' }
|
|
162
|
+
}).addTool({
|
|
163
|
+
name: 'lookup', description: 'Look up a value.',
|
|
164
|
+
inputSchema: { type: 'object', properties: {} }
|
|
165
|
+
}, () => 'value').addText('Look it up');
|
|
166
|
+
|
|
167
|
+
expect(await model.message()).to.equal('Found');
|
|
168
|
+
model.addText('Continue');
|
|
169
|
+
expect(await model.message()).to.equal('Done');
|
|
170
|
+
expect(scope.isDone()).to.equal(true);
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
it('streams native text without including empty reasoning or usage deltas', async () => {
|
|
174
|
+
const chunks = [
|
|
175
|
+
{ choices: [{ delta: { reasoning_content: 'Reasoning' } }] },
|
|
176
|
+
{ choices: [{ delta: { content: 'Hello' } }] },
|
|
177
|
+
{ choices: [], usage: { prompt_tokens: 1000, completion_tokens: 100, prompt_cache_hit_tokens: 400 } }
|
|
178
|
+
];
|
|
179
|
+
const scope = nock('https://api.deepseek.com')
|
|
180
|
+
.post('/chat/completions', body => body.stream === true)
|
|
181
|
+
.reply(200, chunks.map(chunk => `data: ${JSON.stringify(chunk)}\n\n`).join('') + 'data: [DONE]\n\n', {
|
|
182
|
+
'content-type': 'text/event-stream'
|
|
183
|
+
});
|
|
184
|
+
const model = ModelMix.new().deepseekV41Flash({
|
|
185
|
+
mix: { deepseek: true, openrouter: false },
|
|
186
|
+
config: { apiKey: 'native-test-key' }
|
|
187
|
+
}).addText('Hello');
|
|
188
|
+
const deltas = [];
|
|
189
|
+
const result = await model.stream(({ delta }) => deltas.push(delta));
|
|
190
|
+
|
|
191
|
+
expect(result.message).to.equal('Hello');
|
|
192
|
+
expect(deltas.join('')).to.equal('Hello');
|
|
193
|
+
expect(result.tokens.cached).to.equal(400);
|
|
194
|
+
expect(scope.isDone()).to.equal(true);
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
it('uses only Fireworks when selected and accounts for its cached input pricing', async () => {
|
|
198
|
+
const scope = nock('https://api.fireworks.ai')
|
|
199
|
+
.post('/inference/v1/chat/completions', body => {
|
|
200
|
+
expect(body.model).to.equal('accounts/fireworks/models/deepseek-v4p1-flash');
|
|
201
|
+
expect(body.reasoning_effort).to.equal('max');
|
|
202
|
+
expect(body.thinking).to.deep.equal({ type: 'enabled' });
|
|
203
|
+
expect(body.temperature).to.equal(0.5);
|
|
204
|
+
return true;
|
|
205
|
+
})
|
|
206
|
+
.reply(200, {
|
|
207
|
+
choices: [{ message: { content: 'ok' } }],
|
|
208
|
+
usage: {
|
|
209
|
+
prompt_tokens: 1000,
|
|
210
|
+
completion_tokens: 100,
|
|
211
|
+
prompt_tokens_details: { cached_tokens: 400 }
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
const model = ModelMix.new().deepseekV41Flash({
|
|
215
|
+
mix: { fireworks: true, openrouter: false },
|
|
216
|
+
config: { effort: 100 },
|
|
217
|
+
options: { temperature: 0.5 }
|
|
218
|
+
}).addText('Hello');
|
|
219
|
+
|
|
220
|
+
expect(model.models).to.have.length(1);
|
|
221
|
+
expect(model.models[0].provider).to.be.instanceOf(MixFireworks);
|
|
222
|
+
expect(await model.message()).to.equal('ok');
|
|
223
|
+
expect(scope.isDone()).to.equal(true);
|
|
224
|
+
expect(model.lastRaw.tokens).to.include({ input: 1000, cached: 400, output: 100 });
|
|
225
|
+
expect(model.lastRaw.tokens.cost).to.be.closeTo(0.0002008, 1e-12);
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
it('falls back from Fireworks to OpenRouter when both are enabled', async () => {
|
|
229
|
+
const fireworks = nock('https://api.fireworks.ai')
|
|
230
|
+
.post('/inference/v1/chat/completions')
|
|
231
|
+
.reply(503, { error: { message: 'Unavailable' } });
|
|
232
|
+
const openrouter = nock('https://openrouter.ai')
|
|
233
|
+
.post('/api/v1/chat/completions', body => body.model === 'deepseek/deepseek-v4.1-flash')
|
|
234
|
+
.reply(200, { choices: [{ message: { content: 'fallback' } }] });
|
|
235
|
+
const model = ModelMix.new({ config: { retry: { retries: 0 } } })
|
|
236
|
+
.deepseekV41Flash({ mix: { fireworks: true, openrouter: true } })
|
|
237
|
+
.addText('Hello');
|
|
238
|
+
|
|
239
|
+
expect(model.models.map(({ key }) => key)).to.deep.equal([
|
|
240
|
+
'accounts/fireworks/models/deepseek-v4p1-flash',
|
|
241
|
+
'deepseek/deepseek-v4.1-flash'
|
|
242
|
+
]);
|
|
243
|
+
expect(await model.message()).to.equal('fallback');
|
|
244
|
+
expect(fireworks.isDone()).to.equal(true);
|
|
245
|
+
expect(openrouter.isDone()).to.equal(true);
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
for (const [effort, level] of [[0, null], [20, 'low'], [60, 'high'], [100, 'max'], [-1, undefined]]) {
|
|
249
|
+
it(`sends chain effort ${effort} and accounts for cached input`, async () => {
|
|
250
|
+
const scope = nock('https://openrouter.ai')
|
|
251
|
+
.post('/api/v1/chat/completions', body => {
|
|
252
|
+
expect(body.model).to.equal('deepseek/deepseek-v4.1-flash');
|
|
253
|
+
expect(body.reasoning_effort).to.equal(level || undefined);
|
|
254
|
+
expect(body.thinking).to.deep.equal(level === undefined
|
|
255
|
+
? undefined
|
|
256
|
+
: { type: level === null ? 'disabled' : 'enabled' });
|
|
257
|
+
return true;
|
|
258
|
+
})
|
|
259
|
+
.reply(200, {
|
|
260
|
+
choices: [{ message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
|
|
261
|
+
usage: {
|
|
262
|
+
prompt_tokens: 1000,
|
|
263
|
+
completion_tokens: 100,
|
|
264
|
+
prompt_tokens_details: { cached_tokens: 400 }
|
|
265
|
+
}
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
const model = ModelMix.new().chain(`deepseekV41Flash@${effort}`).addText('Hello');
|
|
269
|
+
expect(await model.message()).to.equal('ok');
|
|
270
|
+
expect(scope.isDone()).to.equal(true);
|
|
271
|
+
expect(model.lastRaw.tokens).to.include({ input: 1000, cached: 400, output: 100 });
|
|
272
|
+
expect(model.lastRaw.tokens.cost).to.be.closeTo(0.000156, 1e-12);
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
});
|
|
276
|
+
|
|
5
277
|
it('should register Fireworks DeepSeek V4 Pro by default', () => {
|
|
6
278
|
const model = ModelMix.new();
|
|
7
279
|
model.deepseekV4Pro({ mix: { fireworks: true, openrouter: false } });
|
package/test/effort.test.js
CHANGED
|
@@ -73,6 +73,15 @@ describe('Unified effort scale', () => {
|
|
|
73
73
|
expect(mapEffort('openai', 100)).to.deep.equal({ reasoning_effort: 'xhigh' });
|
|
74
74
|
});
|
|
75
75
|
|
|
76
|
+
it('clamps GPT-6 Astra effort to its supported range', () => {
|
|
77
|
+
for (const key of ['gpt-6-astra', 'openai/gpt-6-astra']) {
|
|
78
|
+
for (const [effort, level] of [[0, 'low'], [39, 'low'], [40, 'medium'], [60, 'high'], [80, 'xhigh'], [99, 'xhigh'], [100, 'max']]) {
|
|
79
|
+
expect(mapEffort('openai', effort, key)).to.deep.equal({ reasoning_effort: level });
|
|
80
|
+
}
|
|
81
|
+
expect(mapEffort('openai', -1, key)).to.equal(null);
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
|
|
76
85
|
it('maps GPT-5.6 maximum unified effort to max', () => {
|
|
77
86
|
expect(mapEffort('openai', 99, 'gpt-5.6-luna')).to.deep.equal({ reasoning_effort: 'xhigh' });
|
|
78
87
|
for (const model of ['gpt-5.6-sol', 'gpt-5.6-terra', 'gpt-5.6-luna']) {
|
package/test/fallback.test.js
CHANGED
|
@@ -13,6 +13,63 @@ const {
|
|
|
13
13
|
} = require('../index.js');
|
|
14
14
|
|
|
15
15
|
describe('Provider Fallback Chain Tests', () => {
|
|
16
|
+
|
|
17
|
+
describe('Interrupted provider responses', () => {
|
|
18
|
+
const timeout = {
|
|
19
|
+
id: 'generation-timeout',
|
|
20
|
+
provider: 'Novita',
|
|
21
|
+
error: { code: 504, message: 'Upstream idle timeout exceeded', metadata: { error_type: 'timeout' } },
|
|
22
|
+
choices: [{ finish_reason: 'error', message: { content: '' } }]
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
it('rejects HTTP 200 error bodies with their status and generation details', async () => {
|
|
26
|
+
sinon.stub(global, 'fetch').resolves(new Response(JSON.stringify(timeout), { status: 200 }));
|
|
27
|
+
const model = ModelMix.new().deepseekV41Flash().addText('test');
|
|
28
|
+
let failure;
|
|
29
|
+
try { await model.raw(); } catch (error) { failure = error; }
|
|
30
|
+
expect(failure).to.include({ statusCode: 504, message: timeout.error.message });
|
|
31
|
+
expect(failure.details).to.include({ id: timeout.id, provider: 'Novita' });
|
|
32
|
+
expect(failure.details.error.metadata.error_type).to.equal('timeout');
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it('retries HTTP 200 provider errors only under the configured policy', async () => {
|
|
36
|
+
const fetch = sinon.stub(global, 'fetch');
|
|
37
|
+
fetch.onFirstCall().resolves(new Response(JSON.stringify(timeout)));
|
|
38
|
+
fetch.onSecondCall().resolves(new Response(JSON.stringify({ choices: [{ finish_reason: 'stop', message: { content: 'recovered' } }] })));
|
|
39
|
+
const model = ModelMix.new({ config: { retry: { enabled: true, retries: 1, baseDelayMs: 0, maxDelayMs: 0 } } }).deepseekV41Flash().addText('test');
|
|
40
|
+
expect((await model.raw()).message).to.equal('recovered');
|
|
41
|
+
expect(fetch.callCount).to.equal(2);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it('falls back when a provider returns only reasoning without completing', async () => {
|
|
45
|
+
const fetch = sinon.stub(global, 'fetch');
|
|
46
|
+
fetch.onFirstCall().resolves(new Response(JSON.stringify({ choices: [{ finish_reason: null, message: { content: '', reasoning: 'partial' } }] })));
|
|
47
|
+
fetch.onSecondCall().resolves(new Response(JSON.stringify({ choices: [{ finish_reason: 'stop', message: { content: 'fallback' } }] })));
|
|
48
|
+
const model = ModelMix.new().deepseekV41Flash().gpt5mini().addText('test');
|
|
49
|
+
expect((await model.raw()).message).to.equal('fallback');
|
|
50
|
+
expect(fetch.callCount).to.equal(2);
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
it('rejects a streaming timeout instead of returning partial output', async () => {
|
|
54
|
+
const chunks = [
|
|
55
|
+
{ choices: [{ delta: { content: 'partial' } }] },
|
|
56
|
+
timeout
|
|
57
|
+
];
|
|
58
|
+
sinon.stub(global, 'fetch').resolves(new Response(chunks.map(chunk => `data: ${JSON.stringify(chunk)}\n\n`).join(''), { headers: { 'content-type': 'text/event-stream' } }));
|
|
59
|
+
const model = ModelMix.new().deepseekV41Flash().addText('test');
|
|
60
|
+
let failure;
|
|
61
|
+
try { await model.stream(() => {}); } catch (error) { failure = error; }
|
|
62
|
+
expect(failure).to.include({ statusCode: 504, message: timeout.error.message });
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it('rejects an abruptly ended stream even after some text arrived', async () => {
|
|
66
|
+
sinon.stub(global, 'fetch').resolves(new Response('data: {"choices":[{"delta":{"content":"partial"},"finish_reason":null}]}\n\n'));
|
|
67
|
+
let failure;
|
|
68
|
+
try { await ModelMix.new().deepseekV41Flash().addText('test').stream(() => {}); } catch (error) { failure = error; }
|
|
69
|
+
expect(failure).to.include({ statusCode: 502 });
|
|
70
|
+
expect(failure.message).to.include('without completing');
|
|
71
|
+
});
|
|
72
|
+
});
|
|
16
73
|
|
|
17
74
|
// Setup test hooks
|
|
18
75
|
if (global.setupTestHooks) {
|
|
@@ -57,6 +114,7 @@ describe('Provider Fallback Chain Tests', () => {
|
|
|
57
114
|
it('should register every supported OpenAI text shortcut with its OpenRouter fallback', () => {
|
|
58
115
|
expect(ModelMix.new().mix.openrouter).to.equal(false);
|
|
59
116
|
const shortcuts = [
|
|
117
|
+
['gpt6astra', 'gpt-6-astra', 'openai/gpt-6-astra', MixOpenAIResponses],
|
|
60
118
|
['gpt5', 'gpt-5', 'openai/gpt-5', MixOpenAI],
|
|
61
119
|
['gpt5mini', 'gpt-5-mini', 'openai/gpt-5-mini', MixOpenAI],
|
|
62
120
|
['gpt5nano', 'gpt-5-nano', 'openai/gpt-5-nano', MixOpenAI],
|
|
@@ -101,13 +159,13 @@ describe('Provider Fallback Chain Tests', () => {
|
|
|
101
159
|
});
|
|
102
160
|
|
|
103
161
|
it('should append OpenRouter GPT fallbacks in chain() only when enabled globally', () => {
|
|
104
|
-
const official = ModelMix.new().chain('
|
|
105
|
-
expect(official.models.map(({ key }) => key)).to.deep.equal(['gpt-
|
|
162
|
+
const official = ModelMix.new().chain('gpt6astra@100');
|
|
163
|
+
expect(official.models.map(({ key }) => key)).to.deep.equal(['gpt-6-astra']);
|
|
106
164
|
|
|
107
|
-
const routed = ModelMix.new({ mix: { openrouter: true } }).chain('
|
|
165
|
+
const routed = ModelMix.new({ mix: { openrouter: true } }).chain('gpt6astra@100');
|
|
108
166
|
expect(routed.models.map(({ key }) => key)).to.deep.equal([
|
|
109
|
-
'gpt-
|
|
110
|
-
'openai/gpt-
|
|
167
|
+
'gpt-6-astra',
|
|
168
|
+
'openai/gpt-6-astra'
|
|
111
169
|
]);
|
|
112
170
|
expect(routed.models.map(({ provider }) => provider.config.effort)).to.deep.equal([100, 100]);
|
|
113
171
|
});
|
|
@@ -140,6 +198,40 @@ describe('Provider Fallback Chain Tests', () => {
|
|
|
140
198
|
expect(openRouterRequest).to.not.have.property('temperature');
|
|
141
199
|
});
|
|
142
200
|
|
|
201
|
+
it('should fallback from the official GPT-6 Astra endpoint to OpenRouter', async () => {
|
|
202
|
+
let openRouterRequest;
|
|
203
|
+
model.effort(100).gpt6astra({ mix: { openrouter: true } }).addText('Hello');
|
|
204
|
+
|
|
205
|
+
nock('https://api.openai.com')
|
|
206
|
+
.post('/v1/responses', body => {
|
|
207
|
+
expect(body.model).to.equal('gpt-6-astra');
|
|
208
|
+
expect(body.reasoning).to.deep.equal({ effort: 'max' });
|
|
209
|
+
expect(body.max_output_tokens).to.equal(8192);
|
|
210
|
+
expect(body).to.not.have.property('temperature');
|
|
211
|
+
return true;
|
|
212
|
+
})
|
|
213
|
+
.reply(503, { error: 'Service unavailable' });
|
|
214
|
+
|
|
215
|
+
nock('https://openrouter.ai')
|
|
216
|
+
.post('/api/v1/chat/completions', body => {
|
|
217
|
+
openRouterRequest = body;
|
|
218
|
+
return true;
|
|
219
|
+
})
|
|
220
|
+
.reply(200, {
|
|
221
|
+
choices: [{
|
|
222
|
+
message: {
|
|
223
|
+
role: 'assistant',
|
|
224
|
+
content: 'Hello from GPT-6 Astra through OpenRouter!'
|
|
225
|
+
}
|
|
226
|
+
}]
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
expect(await model.message()).to.equal('Hello from GPT-6 Astra through OpenRouter!');
|
|
230
|
+
expect(openRouterRequest.model).to.equal('openai/gpt-6-astra');
|
|
231
|
+
expect(openRouterRequest.max_completion_tokens).to.equal(8192);
|
|
232
|
+
expect(openRouterRequest).to.not.have.property('temperature');
|
|
233
|
+
});
|
|
234
|
+
|
|
143
235
|
it('should keep the default fable51 chain on Anthropic', () => {
|
|
144
236
|
model.chain('fable51@80');
|
|
145
237
|
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
const { expect } = require('chai');
|
|
2
|
+
const sinon = require('sinon');
|
|
3
|
+
const { MixGoogle, ModelMix } = require('../index');
|
|
4
|
+
|
|
5
|
+
describe('Google message and JSON contracts', () => {
|
|
6
|
+
afterEach(() => sinon.restore());
|
|
7
|
+
|
|
8
|
+
it('converts string user and assistant messages without changing their inputs', () => {
|
|
9
|
+
const messages = [
|
|
10
|
+
{ role: 'user', content: 'Request' },
|
|
11
|
+
{ role: 'assistant', content: 'Answer' },
|
|
12
|
+
{ role: 'user', content: [{ type: 'text', text: 'Follow-up' }] }
|
|
13
|
+
];
|
|
14
|
+
expect(MixGoogle.convertMessages(messages)).to.deep.equal([
|
|
15
|
+
{ role: 'user', parts: [{ text: 'Request' }] },
|
|
16
|
+
{ role: 'model', parts: [{ text: 'Answer' }] },
|
|
17
|
+
{ role: 'user', parts: [{ text: 'Follow-up' }] }
|
|
18
|
+
]);
|
|
19
|
+
expect(messages[0]).to.deep.equal({ role: 'user', content: 'Request' });
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it('sends child text messages and JSON mode through the real Google adapter', async () => {
|
|
23
|
+
const bodies = [];
|
|
24
|
+
sinon.stub(global, 'fetch').callsFake(async (_url, input) => {
|
|
25
|
+
bodies.push(JSON.parse(input.body));
|
|
26
|
+
return new Response(JSON.stringify({
|
|
27
|
+
candidates: [{ content: { parts: [{ text: '{"ok":true}' }] } }]
|
|
28
|
+
}), { status: 200 });
|
|
29
|
+
});
|
|
30
|
+
const worker = ModelMix.new({ config: { apiKey: 'test-key' } }).gemini38flash();
|
|
31
|
+
const result = await ModelMix.new().use({
|
|
32
|
+
name: 'child-json',
|
|
33
|
+
execute: context => context.invoke({
|
|
34
|
+
model: worker,
|
|
35
|
+
system: 'Return JSON.',
|
|
36
|
+
messages: [{ role: 'user', content: 'Evaluate this response.' }],
|
|
37
|
+
options: { response_format: { type: 'json_object' } },
|
|
38
|
+
plugins: 'none'
|
|
39
|
+
})
|
|
40
|
+
}).addText('Task').json();
|
|
41
|
+
expect(result).to.deep.equal({ ok: true });
|
|
42
|
+
expect(bodies[0].contents).to.deep.equal([
|
|
43
|
+
{ role: 'user', parts: [{ text: 'Evaluate this response.' }] }
|
|
44
|
+
]);
|
|
45
|
+
expect(bodies[0].generationConfig.responseMimeType).to.equal('application/json');
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('combines answer text parts without including thought parts', () => {
|
|
49
|
+
expect(MixGoogle.extractMessage({ candidates: [{ content: { parts: [
|
|
50
|
+
{ thought: true, text: 'Internal analysis' },
|
|
51
|
+
{ text: '{"ok":' },
|
|
52
|
+
{ text: 'true}' }
|
|
53
|
+
] } }] })).to.equal('{"ok":true}');
|
|
54
|
+
});
|
|
55
|
+
});
|
package/test/history.test.js
CHANGED
|
@@ -629,7 +629,7 @@ describe('Conversation History Tests', () => {
|
|
|
629
629
|
expect(model.messages).to.have.length(2);
|
|
630
630
|
});
|
|
631
631
|
|
|
632
|
-
it('should
|
|
632
|
+
it('should reject an empty assistant response without adding it to history', async () => {
|
|
633
633
|
const model = ModelMix.new({
|
|
634
634
|
config: { debug: false, max_history: 10 }
|
|
635
635
|
});
|
|
@@ -643,9 +643,12 @@ describe('Conversation History Tests', () => {
|
|
|
643
643
|
}]
|
|
644
644
|
});
|
|
645
645
|
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
expect(
|
|
646
|
+
let failure;
|
|
647
|
+
try { await model.message(); } catch (error) { failure = error; }
|
|
648
|
+
expect(failure).to.include({ statusCode: 502 });
|
|
649
|
+
expect(failure.message).to.include('no text or tool calls');
|
|
650
|
+
expect(model.messages).to.have.length(1);
|
|
651
|
+
expect(model.messages[0].role).to.equal('user');
|
|
649
652
|
});
|
|
650
653
|
|
|
651
654
|
it('should handle multiple addText before first message()', async () => {
|
package/test/json.test.js
CHANGED
|
@@ -17,6 +17,34 @@ describe('JSON Schema and Structured Output Tests', () => {
|
|
|
17
17
|
sinon.restore();
|
|
18
18
|
});
|
|
19
19
|
|
|
20
|
+
describe('JSON response formatting', () => {
|
|
21
|
+
for (const suffix of ['', '```', '\n```\n']) {
|
|
22
|
+
it(`preserves embedded Markdown with trailing delimiter ${JSON.stringify(suffix)}`, async () => {
|
|
23
|
+
const value = { text: 'Use ```text\ncontent\n``` here.' };
|
|
24
|
+
const message = JSON.stringify(value) + suffix;
|
|
25
|
+
const mix = ModelMix.new();
|
|
26
|
+
const result = { message };
|
|
27
|
+
sinon.stub(mix, 'execute').resolves(result);
|
|
28
|
+
expect(await mix.json()).to.deep.equal(value);
|
|
29
|
+
expect(result.message).to.equal(message);
|
|
30
|
+
});
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
for (const message of ['{"ok":true} explanation```', '{"ok":true}{"other":true}```', '{"ok"":true}```']) {
|
|
34
|
+
it(`rejects invalid content even with a closing delimiter: ${message}`, async () => {
|
|
35
|
+
const mix = ModelMix.new();
|
|
36
|
+
sinon.stub(mix, 'execute').resolves({ message });
|
|
37
|
+
let failure;
|
|
38
|
+
try {
|
|
39
|
+
await mix.json();
|
|
40
|
+
} catch (error) {
|
|
41
|
+
failure = error;
|
|
42
|
+
}
|
|
43
|
+
expect(failure).to.be.instanceOf(SyntaxError);
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
|
|
20
48
|
describe('JSON Schema Generation', () => {
|
|
21
49
|
it('should generate schema for simple object', () => {
|
|
22
50
|
const example = {
|
|
@@ -490,4 +518,4 @@ describe('JSON Schema and Structured Output Tests', () => {
|
|
|
490
518
|
expect(result[2]).to.have.property('name', 'Spain');
|
|
491
519
|
});
|
|
492
520
|
});
|
|
493
|
-
});
|
|
521
|
+
});
|
package/test/public-api.test.js
CHANGED
|
@@ -8,6 +8,7 @@ describe('public module boundary', () => {
|
|
|
8
8
|
'MixAnthropic',
|
|
9
9
|
'MixCerebras',
|
|
10
10
|
'MixCustom',
|
|
11
|
+
'MixDeepSeek',
|
|
11
12
|
'MixFireworks',
|
|
12
13
|
'MixGoogle',
|
|
13
14
|
'MixGrok',
|
|
@@ -56,6 +57,7 @@ describe('public module boundary', () => {
|
|
|
56
57
|
['Gemini', config => new api.MixGoogle({ config })],
|
|
57
58
|
['MiniMax', config => new api.MixMiniMax({ config })],
|
|
58
59
|
['MiMo', config => new api.MixMiMo({ config })],
|
|
60
|
+
['DeepSeek', config => new api.MixDeepSeek({ config })],
|
|
59
61
|
['Perplexity', config => new api.MixPerplexity({ config })],
|
|
60
62
|
['Grok', config => new api.MixGrok({ config })],
|
|
61
63
|
['Lambda', config => new api.MixLambda({ config })],
|