modelmix 4.7.4 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +43 -0
- package/README.md +283 -92
- package/demo/demo.js +4 -4
- package/demo/prompt.md +2 -2
- package/http-client.js +15 -2
- package/index.d.ts +102 -12
- package/index.js +965 -275
- package/package.json +4 -1
- package/skills/modelmix/SKILL.md +109 -19
- package/test/README.md +10 -9
- package/test/anthropic.test.js +146 -9
- package/test/effort.test.js +2 -2
- package/test/fallback.test.js +192 -2
- package/test/fixtures/account-details.txt +4 -0
- package/test/fixtures/choice-options.txt +6 -0
- package/test/fixtures/choice-template.txt +2 -0
- package/test/fixtures/dynamic-include.txt +2 -0
- package/test/fixtures/system-rules.txt +1 -0
- package/test/fixtures/system-template.txt +2 -0
- package/test/fixtures/template.txt +4 -11
- package/test/fixtures/tree.ejs +6 -0
- package/test/grok.test.js +28 -1
- package/test/history.test.js +2 -2
- package/test/live.mcp.js +20 -20
- package/test/live.test.js +11 -11
- package/test/templates.test.js +621 -280
- package/test/tokens.test.js +409 -16
package/test/templates.test.js
CHANGED
|
@@ -2,12 +2,13 @@ const { expect } = require('chai');
|
|
|
2
2
|
const sinon = require('sinon');
|
|
3
3
|
const nock = require('nock');
|
|
4
4
|
const fs = require('fs');
|
|
5
|
+
const os = require('os');
|
|
5
6
|
const path = require('path');
|
|
6
7
|
const { ModelMix } = require('../index.js');
|
|
7
8
|
|
|
8
|
-
describe('Template and File Operations Tests', () => {
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
describe('EJS Template and File Operations Tests', () => {
|
|
10
|
+
const fixturesPath = path.join(__dirname, 'fixtures');
|
|
11
|
+
|
|
11
12
|
if (global.setupTestHooks) {
|
|
12
13
|
global.setupTestHooks();
|
|
13
14
|
}
|
|
@@ -17,367 +18,707 @@ describe('Template and File Operations Tests', () => {
|
|
|
17
18
|
sinon.restore();
|
|
18
19
|
});
|
|
19
20
|
|
|
20
|
-
|
|
21
|
-
|
|
21
|
+
function mockOpenAI(assertRequest, responseText = 'Template processed successfully') {
|
|
22
|
+
nock('https://api.openai.com')
|
|
23
|
+
.post('/v1/responses')
|
|
24
|
+
.reply(function (uri, body) {
|
|
25
|
+
assertRequest(body);
|
|
26
|
+
return [200, testUtils.createMockResponse('openai-responses', responseText)];
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function userTexts(body) {
|
|
31
|
+
return body.input
|
|
32
|
+
.filter(message => message.role === 'user')
|
|
33
|
+
.flatMap(message => message.content)
|
|
34
|
+
.filter(content => content.type === 'input_text')
|
|
35
|
+
.map(content => content.text);
|
|
36
|
+
}
|
|
22
37
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
38
|
+
describe('EJS rendering', () => {
|
|
39
|
+
it('renders inline variables with plain data keys', async () => {
|
|
40
|
+
const model = ModelMix.new()
|
|
41
|
+
.gpt51()
|
|
42
|
+
.assign({ name: 'Alice', age: 30, city: 'New York' })
|
|
43
|
+
.addText('Hello <%- name %>, you are <%- age %> years old and live in <%- city %>.');
|
|
44
|
+
|
|
45
|
+
mockOpenAI(body => {
|
|
46
|
+
expect(userTexts(body)).to.deep.equal([
|
|
47
|
+
'Hello Alice, you are 30 years old and live in New York.'
|
|
48
|
+
]);
|
|
26
49
|
});
|
|
50
|
+
|
|
51
|
+
await model.message();
|
|
27
52
|
});
|
|
28
53
|
|
|
29
|
-
it('
|
|
30
|
-
model.
|
|
31
|
-
.
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
'{{city}}': 'New York'
|
|
35
|
-
})
|
|
36
|
-
.addText('Hello {{name}}, you are {{age}} years old and live in {{city}}.');
|
|
54
|
+
it('assigns one template data key', async () => {
|
|
55
|
+
const model = ModelMix.new()
|
|
56
|
+
.gpt51()
|
|
57
|
+
.assignKey('name', 'Martin')
|
|
58
|
+
.addText('Hello <%- name %>.');
|
|
37
59
|
|
|
38
|
-
|
|
39
|
-
.
|
|
40
|
-
|
|
41
|
-
const userMsg = body.input.find(m => m.role === 'user');
|
|
42
|
-
expect(userMsg.content[0].text).to.equal('Hello Alice, you are 30 years old and live in New York.');
|
|
43
|
-
return [200, testUtils.createMockResponse('openai-responses', 'Template processed successfully')];
|
|
44
|
-
});
|
|
60
|
+
mockOpenAI(body => {
|
|
61
|
+
expect(userTexts(body)).to.deep.equal(['Hello Martin.']);
|
|
62
|
+
});
|
|
45
63
|
|
|
46
|
-
|
|
47
|
-
expect(response).to.include('Template processed successfully');
|
|
64
|
+
await model.message();
|
|
48
65
|
});
|
|
49
66
|
|
|
50
|
-
it('
|
|
51
|
-
model.
|
|
52
|
-
.
|
|
53
|
-
.
|
|
54
|
-
|
|
55
|
-
|
|
67
|
+
it('supports nested data, conditionals, and loops', async () => {
|
|
68
|
+
const model = ModelMix.new()
|
|
69
|
+
.gpt51()
|
|
70
|
+
.assign({
|
|
71
|
+
user: {
|
|
72
|
+
name: 'Charlie',
|
|
73
|
+
active: true,
|
|
74
|
+
roles: ['admin', 'reviewer']
|
|
75
|
+
}
|
|
76
|
+
})
|
|
77
|
+
.addText('<% if (user.active) { %><%- user.name %>: <% user.roles.forEach((role, index) => { %><%= index ? ", " : "" %><%- role %><% }) %><% } %>');
|
|
56
78
|
|
|
57
|
-
|
|
58
|
-
.
|
|
59
|
-
|
|
60
|
-
const userMsg = body.input.find(m => m.role === 'user');
|
|
61
|
-
expect(userMsg.content[0].text).to.equal('Hello Bob, welcome to our platform!');
|
|
62
|
-
return [200, testUtils.createMockResponse('openai-responses', 'Multiple templates replaced')];
|
|
63
|
-
});
|
|
79
|
+
mockOpenAI(body => {
|
|
80
|
+
expect(userTexts(body)).to.deep.equal(['Charlie: admin, reviewer']);
|
|
81
|
+
});
|
|
64
82
|
|
|
65
|
-
|
|
66
|
-
expect(response).to.include('Multiple templates replaced');
|
|
83
|
+
await model.message();
|
|
67
84
|
});
|
|
68
85
|
|
|
69
|
-
it('
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
86
|
+
it('keeps raw and XML-escaped output distinct', async () => {
|
|
87
|
+
const value = 'Hello & "World" <test>';
|
|
88
|
+
const model = ModelMix.new()
|
|
89
|
+
.gpt51()
|
|
90
|
+
.assign({ value })
|
|
91
|
+
.addText('Escaped: <%= value %>\nRaw: <%- value %>');
|
|
92
|
+
|
|
93
|
+
mockOpenAI(body => {
|
|
94
|
+
expect(userTexts(body)).to.deep.equal([
|
|
95
|
+
'Escaped: Hello & "World" <test>\nRaw: Hello & "World" <test>'
|
|
96
|
+
]);
|
|
97
|
+
});
|
|
78
98
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
})
|
|
99
|
+
await model.message();
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
it('does not execute EJS received through template data', async () => {
|
|
103
|
+
const model = ModelMix.new()
|
|
104
|
+
.gpt51()
|
|
105
|
+
.assign({ payload: '<%- secret %>', secret: 'must-not-render' })
|
|
106
|
+
.addText('Payload: <%- payload %>');
|
|
86
107
|
|
|
87
|
-
|
|
88
|
-
|
|
108
|
+
mockOpenAI(body => {
|
|
109
|
+
expect(userTexts(body)).to.deep.equal(['Payload: <%- secret %>']);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
await model.message();
|
|
89
113
|
});
|
|
90
114
|
|
|
91
|
-
it('
|
|
92
|
-
model.
|
|
93
|
-
.
|
|
94
|
-
.addText(
|
|
115
|
+
it('selects uniformly when choice options omit weights', async () => {
|
|
116
|
+
const model = ModelMix.new()
|
|
117
|
+
.gpt51()
|
|
118
|
+
.addText(`<% choice %>
|
|
119
|
+
<% option %>
|
|
120
|
+
Use emojis.
|
|
121
|
+
<% option %>
|
|
122
|
+
Use few emojis.
|
|
123
|
+
<% option %>
|
|
124
|
+
Do not use emojis.
|
|
125
|
+
<% /choice %>`);
|
|
126
|
+
sinon.stub(model, '_choiceRandom').returns(0.5);
|
|
127
|
+
|
|
128
|
+
mockOpenAI(body => {
|
|
129
|
+
expect(userTexts(body)[0].trim()).to.equal('Use few emojis.');
|
|
130
|
+
});
|
|
95
131
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
})
|
|
132
|
+
await model.message();
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it('selects weighted options using relative weights', async () => {
|
|
136
|
+
const model = ModelMix.new()
|
|
137
|
+
.gpt51()
|
|
138
|
+
.assign({ language: 'Spanish' })
|
|
139
|
+
.addText(`<% choice %>
|
|
140
|
+
<% option 20 %>
|
|
141
|
+
Use emojis in <%- language %>.
|
|
142
|
+
<% option 40 %>
|
|
143
|
+
Use few emojis in <%- language %>.
|
|
144
|
+
<% option 40 %>
|
|
145
|
+
Do not use emojis in <%- language %>.
|
|
146
|
+
<% /choice %>`);
|
|
147
|
+
sinon.stub(model, '_choiceRandom').returns(0.2);
|
|
148
|
+
|
|
149
|
+
mockOpenAI(body => {
|
|
150
|
+
expect(userTexts(body)[0].trim()).to.equal('Use few emojis in Spanish.');
|
|
151
|
+
});
|
|
103
152
|
|
|
104
|
-
|
|
105
|
-
expect(response).to.include('Partial template replacement');
|
|
153
|
+
await model.message();
|
|
106
154
|
});
|
|
107
155
|
|
|
108
|
-
it('
|
|
109
|
-
model.
|
|
110
|
-
.
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
156
|
+
it('supports nested choices', async () => {
|
|
157
|
+
const model = ModelMix.new()
|
|
158
|
+
.gpt51()
|
|
159
|
+
.addText(`<% choice %>
|
|
160
|
+
<% option %>
|
|
161
|
+
Tone:
|
|
162
|
+
<% choice %>
|
|
163
|
+
<% option %>
|
|
164
|
+
formal
|
|
165
|
+
<% option %>
|
|
166
|
+
casual
|
|
167
|
+
<% /choice %>
|
|
168
|
+
<% option %>
|
|
169
|
+
No tone instruction.
|
|
170
|
+
<% /choice %>`);
|
|
171
|
+
const random = sinon.stub(model, '_choiceRandom');
|
|
172
|
+
random.onFirstCall().returns(0.1);
|
|
173
|
+
random.onSecondCall().returns(0.9);
|
|
174
|
+
|
|
175
|
+
mockOpenAI(body => {
|
|
176
|
+
expect(userTexts(body)[0].trim()).to.equal('Tone:\ncasual');
|
|
177
|
+
});
|
|
117
178
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
const userMsg = body.input.find(m => m.role === 'user');
|
|
122
|
-
expect(userMsg.content[0].text).to.equal('Empty: , Special: Hello & "World" <test>, Number: 42, Boolean: true');
|
|
123
|
-
return [200, testUtils.createMockResponse('openai-responses', 'Special characters handled')];
|
|
124
|
-
});
|
|
179
|
+
await model.message();
|
|
180
|
+
expect(random.callCount).to.equal(2);
|
|
181
|
+
});
|
|
125
182
|
|
|
126
|
-
|
|
127
|
-
|
|
183
|
+
it('rerolls choices on each new request', async () => {
|
|
184
|
+
const template = `<% choice %>
|
|
185
|
+
<% option %>
|
|
186
|
+
first
|
|
187
|
+
<% option %>
|
|
188
|
+
second
|
|
189
|
+
<% /choice %>`;
|
|
190
|
+
const model = ModelMix.new().gpt51().addText(template);
|
|
191
|
+
const random = sinon.stub(model, '_choiceRandom');
|
|
192
|
+
random.onFirstCall().returns(0.1);
|
|
193
|
+
random.onSecondCall().returns(0.9);
|
|
194
|
+
|
|
195
|
+
mockOpenAI(body => {
|
|
196
|
+
expect(userTexts(body)[0].trim()).to.equal('first');
|
|
197
|
+
}, 'First response');
|
|
198
|
+
await model.message();
|
|
199
|
+
|
|
200
|
+
model.addText(template);
|
|
201
|
+
mockOpenAI(body => {
|
|
202
|
+
expect(userTexts(body)[0].trim()).to.equal('second');
|
|
203
|
+
}, 'Second response');
|
|
204
|
+
await model.message();
|
|
205
|
+
|
|
206
|
+
expect(random.callCount).to.equal(2);
|
|
128
207
|
});
|
|
129
|
-
});
|
|
130
208
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
209
|
+
it('fails before the request when a variable is missing', async () => {
|
|
210
|
+
const model = ModelMix.new()
|
|
211
|
+
.gpt51()
|
|
212
|
+
.assign({ name: 'David' })
|
|
213
|
+
.addText('Hello <%- name %>, status: <%- status %>');
|
|
214
|
+
|
|
215
|
+
let error;
|
|
216
|
+
try {
|
|
217
|
+
await model.message();
|
|
218
|
+
} catch (caught) {
|
|
219
|
+
error = caught;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
expect(error).to.be.instanceOf(Error);
|
|
223
|
+
expect(error.message).to.include('Failed to render message template');
|
|
224
|
+
expect(error.message).to.include('status is not defined');
|
|
225
|
+
});
|
|
134
226
|
|
|
135
|
-
|
|
136
|
-
model = ModelMix.new(
|
|
137
|
-
|
|
138
|
-
|
|
227
|
+
it('rejects invalid template data immediately', () => {
|
|
228
|
+
const model = ModelMix.new().gpt51();
|
|
229
|
+
|
|
230
|
+
expect(() => model.assign(null)).to.throw(TypeError, 'Template data must be a plain non-null object.');
|
|
231
|
+
expect(() => model.assign(undefined)).to.throw(TypeError, 'Template data must be a plain non-null object.');
|
|
232
|
+
expect(() => model.assign([])).to.throw(TypeError, 'Template data must be a plain non-null object.');
|
|
233
|
+
expect(() => ModelMix.new({ config: { templateData: null } })).to.throw(
|
|
234
|
+
TypeError,
|
|
235
|
+
'Template data must be a plain non-null object.'
|
|
236
|
+
);
|
|
237
|
+
expect(() => model.assign({ $mix: 'reserved' })).to.throw(
|
|
238
|
+
TypeError,
|
|
239
|
+
'Template data key "$mix" is reserved.'
|
|
240
|
+
);
|
|
241
|
+
expect(() => model.assignKey('', 'value')).to.throw(
|
|
242
|
+
TypeError,
|
|
243
|
+
'Template data key must be a non-empty string.'
|
|
244
|
+
);
|
|
245
|
+
expect(() => model.assignKey('$mix', 'value')).to.throw(
|
|
246
|
+
TypeError,
|
|
247
|
+
'Template data key "$mix" is reserved.'
|
|
248
|
+
);
|
|
139
249
|
});
|
|
140
250
|
|
|
141
|
-
it('
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
}
|
|
153
|
-
|
|
251
|
+
it('reports malformed choice directives with their source line', async () => {
|
|
252
|
+
const cases = [
|
|
253
|
+
{
|
|
254
|
+
source: '<% choice %>\n<% option %>\none\n<% option 2 %>\ntwo\n<% /choice %>',
|
|
255
|
+
message: 'Choice options must either all have weights or all omit them',
|
|
256
|
+
line: 4
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
source: '<% choice %>\n<% option 0 %>\none\n<% /choice %>',
|
|
260
|
+
message: 'Choice weight must be a positive finite number',
|
|
261
|
+
line: 2
|
|
262
|
+
},
|
|
263
|
+
{
|
|
264
|
+
source: '<% option %>\none',
|
|
265
|
+
message: 'Option directive must be inside a choice',
|
|
266
|
+
line: 1
|
|
267
|
+
},
|
|
268
|
+
{
|
|
269
|
+
source: '<% choice %>\ntext\n<% option %>\none\n<% /choice %>',
|
|
270
|
+
message: 'Choice content must be inside an option',
|
|
271
|
+
line: 2
|
|
272
|
+
},
|
|
273
|
+
{
|
|
274
|
+
source: '<% choice %>\n<% option %>\none',
|
|
275
|
+
message: 'Unclosed choice directive',
|
|
276
|
+
line: 1
|
|
277
|
+
}
|
|
278
|
+
];
|
|
279
|
+
|
|
280
|
+
for (const testCase of cases) {
|
|
281
|
+
const model = ModelMix.new().gpt51().addText(testCase.source);
|
|
282
|
+
let error;
|
|
283
|
+
try {
|
|
284
|
+
await model.message();
|
|
285
|
+
} catch (caught) {
|
|
286
|
+
error = caught;
|
|
287
|
+
}
|
|
288
|
+
expect(error).to.be.instanceOf(Error);
|
|
289
|
+
expect(error.message).to.include(testCase.message);
|
|
290
|
+
expect(error.message).to.include(`message template at line ${testCase.line}`);
|
|
291
|
+
}
|
|
292
|
+
});
|
|
154
293
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
.
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
294
|
+
it('rerolls earlier choices after a later template fails to render', async () => {
|
|
295
|
+
const model = ModelMix.new()
|
|
296
|
+
.gpt51()
|
|
297
|
+
.addText(`<% choice %>
|
|
298
|
+
<% option %>
|
|
299
|
+
A
|
|
300
|
+
<% option %>
|
|
301
|
+
B
|
|
302
|
+
<% /choice %>`)
|
|
303
|
+
.addText('<%- missing %>');
|
|
304
|
+
const random = sinon.stub(model, '_choiceRandom');
|
|
305
|
+
random.onFirstCall().returns(0.1);
|
|
306
|
+
random.onSecondCall().returns(0.9);
|
|
307
|
+
|
|
308
|
+
let error;
|
|
309
|
+
try {
|
|
310
|
+
await model.message();
|
|
311
|
+
} catch (caught) {
|
|
312
|
+
error = caught;
|
|
313
|
+
}
|
|
314
|
+
expect(error).to.be.instanceOf(Error);
|
|
315
|
+
expect(model.messages[0].content[0].text).to.include('<% choice %>');
|
|
316
|
+
|
|
317
|
+
model.assign({ missing: 'ready' });
|
|
318
|
+
mockOpenAI(body => {
|
|
319
|
+
const text = userTexts(body).join('\n').trim();
|
|
320
|
+
expect(text).to.include('B');
|
|
321
|
+
expect(text).to.not.include('A');
|
|
322
|
+
expect(text).to.include('ready');
|
|
323
|
+
});
|
|
324
|
+
await model.message();
|
|
167
325
|
|
|
168
|
-
|
|
169
|
-
expect(response).to.include('Template file processed');
|
|
326
|
+
expect(random.callCount).to.equal(2);
|
|
170
327
|
});
|
|
171
328
|
|
|
172
|
-
it('
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
329
|
+
it('rerolls choices after a request fails', async () => {
|
|
330
|
+
const template = `<% choice %>
|
|
331
|
+
<% option %>
|
|
332
|
+
A
|
|
333
|
+
<% option %>
|
|
334
|
+
B
|
|
335
|
+
<% /choice %>`;
|
|
336
|
+
const model = ModelMix.new().gpt51().addText(template);
|
|
337
|
+
const random = sinon.stub(model, '_choiceRandom');
|
|
338
|
+
random.onFirstCall().returns(0.1);
|
|
339
|
+
random.onSecondCall().returns(0.9);
|
|
176
340
|
|
|
177
341
|
nock('https://api.openai.com')
|
|
178
342
|
.post('/v1/responses')
|
|
179
|
-
.reply(
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
343
|
+
.reply(500, { error: 'temporary failure' });
|
|
344
|
+
let error;
|
|
345
|
+
try {
|
|
346
|
+
await model.message();
|
|
347
|
+
} catch (caught) {
|
|
348
|
+
error = caught;
|
|
349
|
+
}
|
|
350
|
+
expect(error).to.exist;
|
|
351
|
+
expect(model.messages[0].content[0].text).to.equal(template);
|
|
352
|
+
|
|
353
|
+
mockOpenAI(body => {
|
|
354
|
+
expect(userTexts(body)[0].trim()).to.equal('B');
|
|
355
|
+
});
|
|
356
|
+
await model.message();
|
|
191
357
|
|
|
192
|
-
|
|
193
|
-
expect(response).to.include('JSON data processed');
|
|
358
|
+
expect(random.callCount).to.equal(2);
|
|
194
359
|
});
|
|
195
360
|
|
|
196
|
-
it('
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
361
|
+
it('does not let a failed concurrent request overwrite a successful choice', async () => {
|
|
362
|
+
const template = `<% choice %>
|
|
363
|
+
<% option %>
|
|
364
|
+
A
|
|
365
|
+
<% option %>
|
|
366
|
+
B
|
|
367
|
+
<% /choice %>`;
|
|
368
|
+
const model = ModelMix.new({
|
|
369
|
+
config: {
|
|
370
|
+
max_history: 10,
|
|
371
|
+
bottleneck: { maxConcurrent: 2, minTime: 0 }
|
|
372
|
+
}
|
|
373
|
+
}).gpt51().addText(template);
|
|
374
|
+
const content = model.messages[0].content[0];
|
|
375
|
+
const random = sinon.stub(model, '_choiceRandom');
|
|
376
|
+
random.onFirstCall().returns(0.1);
|
|
377
|
+
random.onSecondCall().returns(0.9);
|
|
200
378
|
|
|
201
379
|
nock('https://api.openai.com')
|
|
202
380
|
.post('/v1/responses')
|
|
203
|
-
.
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
381
|
+
.delay(100)
|
|
382
|
+
.reply(500, { error: 'delayed failure' });
|
|
383
|
+
nock('https://api.openai.com')
|
|
384
|
+
.post('/v1/responses')
|
|
385
|
+
.reply(200, testUtils.createMockResponse('openai-responses', 'Success'));
|
|
386
|
+
|
|
387
|
+
const results = await Promise.allSettled([model.message(), model.message()]);
|
|
208
388
|
|
|
209
|
-
|
|
210
|
-
expect(
|
|
389
|
+
expect(results.map(result => result.status)).to.deep.equal(['rejected', 'fulfilled']);
|
|
390
|
+
expect(content.text.trim()).to.equal('B');
|
|
391
|
+
expect(model.messageTemplates.has(content)).to.equal(false);
|
|
392
|
+
expect(random.callCount).to.equal(2);
|
|
211
393
|
});
|
|
394
|
+
});
|
|
212
395
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
.
|
|
217
|
-
.
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
396
|
+
describe('File templates and data', () => {
|
|
397
|
+
it('renders a file template with a relative include', async () => {
|
|
398
|
+
const model = ModelMix.new()
|
|
399
|
+
.gpt51()
|
|
400
|
+
.assign({
|
|
401
|
+
name: 'Eve',
|
|
402
|
+
platform: 'ModelMix',
|
|
403
|
+
username: 'eve_user',
|
|
404
|
+
role: 'developer',
|
|
405
|
+
createdDate: '2026-08-07',
|
|
406
|
+
website: 'https://modelmix.dev',
|
|
407
|
+
company: 'AI Solutions',
|
|
408
|
+
showAccount: true
|
|
225
409
|
})
|
|
226
|
-
.
|
|
410
|
+
.addTextFromFile(path.join(fixturesPath, 'template.txt'));
|
|
411
|
+
|
|
412
|
+
mockOpenAI(body => {
|
|
413
|
+
const content = userTexts(body)[0];
|
|
414
|
+
expect(content).to.include('Hello Eve, welcome to ModelMix!');
|
|
415
|
+
expect(content).to.include('Username: eve_user');
|
|
416
|
+
expect(content).to.include('Role: developer');
|
|
417
|
+
expect(content).to.include('Created: 2026-08-07');
|
|
418
|
+
expect(content).to.include('The AI Solutions Team');
|
|
419
|
+
});
|
|
227
420
|
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
421
|
+
await model.message();
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
it('resolves a dynamic include path relative to its template', async () => {
|
|
425
|
+
const model = ModelMix.new()
|
|
426
|
+
.gpt51()
|
|
427
|
+
.assign({ rulesFile: 'system-rules.txt', language: 'Spanish' })
|
|
428
|
+
.addTextFromFile(path.join(fixturesPath, 'dynamic-include.txt'));
|
|
429
|
+
|
|
430
|
+
mockOpenAI(body => {
|
|
431
|
+
expect(userTexts(body)[0].trim()).to.equal('Rules:\nAlways respond in Spanish.');
|
|
432
|
+
});
|
|
239
433
|
|
|
240
|
-
|
|
241
|
-
expect(response).to.include('Multiple files processed');
|
|
434
|
+
await model.message();
|
|
242
435
|
});
|
|
243
436
|
|
|
244
|
-
it('
|
|
245
|
-
const
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
.replace({
|
|
250
|
-
'{{name}}': 'Grace',
|
|
251
|
-
'{{platform}}': 'AbsolutePath',
|
|
252
|
-
'{{username}}': 'grace_abs',
|
|
253
|
-
'{{role}}': 'admin',
|
|
254
|
-
'{{created_date}}': '2023-12-20',
|
|
255
|
-
'{{website}}': 'https://absolute.com',
|
|
256
|
-
'{{company}}': 'Absolute Corp'
|
|
257
|
-
})
|
|
258
|
-
.addText('Absolute path content: {{absolute}}');
|
|
437
|
+
it('processes choice directives inside relative includes', async () => {
|
|
438
|
+
const model = ModelMix.new()
|
|
439
|
+
.gpt51()
|
|
440
|
+
.addTextFromFile(path.join(fixturesPath, 'choice-template.txt'));
|
|
441
|
+
sinon.stub(model, '_choiceRandom').returns(0.75);
|
|
259
442
|
|
|
260
|
-
|
|
261
|
-
.
|
|
262
|
-
|
|
263
|
-
const userMsg = body.input.find(m => m.role === 'user');
|
|
264
|
-
const content = userMsg.content[0].text;
|
|
265
|
-
expect(content).to.include('Hello Grace, welcome to AbsolutePath!');
|
|
266
|
-
expect(content).to.include('The Absolute Corp Team');
|
|
267
|
-
return [200, testUtils.createMockResponse('openai-responses', 'Absolute path works')];
|
|
268
|
-
});
|
|
443
|
+
mockOpenAI(body => {
|
|
444
|
+
expect(userTexts(body)[0].trim()).to.equal('Style:\nBe concise.');
|
|
445
|
+
});
|
|
269
446
|
|
|
270
|
-
|
|
271
|
-
expect(response).to.include('Absolute path works');
|
|
447
|
+
await model.message();
|
|
272
448
|
});
|
|
273
|
-
});
|
|
274
449
|
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
450
|
+
it('supports recursive includes with an explicit depth limit', async () => {
|
|
451
|
+
const model = ModelMix.new()
|
|
452
|
+
.gpt51()
|
|
453
|
+
.assign({
|
|
454
|
+
node: {
|
|
455
|
+
text: 'Root',
|
|
456
|
+
children: [{
|
|
457
|
+
text: 'Child',
|
|
458
|
+
children: [{
|
|
459
|
+
text: 'Grandchild',
|
|
460
|
+
children: [{ text: 'Too deep', children: [] }]
|
|
461
|
+
}]
|
|
462
|
+
}]
|
|
463
|
+
},
|
|
464
|
+
depth: 0,
|
|
465
|
+
maxDepth: 2
|
|
466
|
+
})
|
|
467
|
+
.addTextFromFile(path.join(fixturesPath, 'tree.ejs'));
|
|
468
|
+
|
|
469
|
+
mockOpenAI(body => {
|
|
470
|
+
const content = userTexts(body)[0];
|
|
471
|
+
expect(content).to.include('Root');
|
|
472
|
+
expect(content).to.include('Child');
|
|
473
|
+
expect(content).to.include('Grandchild');
|
|
474
|
+
expect(content).to.not.include('Too deep');
|
|
475
|
+
});
|
|
476
|
+
|
|
477
|
+
await model.message();
|
|
478
|
+
});
|
|
278
479
|
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
480
|
+
it('preserves a system template filename through new instances', async () => {
|
|
481
|
+
const base = ModelMix.new()
|
|
482
|
+
.setSystemFromFile(path.join(fixturesPath, 'system-template.txt'))
|
|
483
|
+
.assign({ role: 'data analyst', language: 'Spanish' });
|
|
484
|
+
const model = base.new().gpt51().addText('Analyze this.');
|
|
485
|
+
|
|
486
|
+
mockOpenAI(body => {
|
|
487
|
+
const system = body.input.find(message => message.role === 'developer');
|
|
488
|
+
expect(system.content[0].text).to.include('You are a data analyst.');
|
|
489
|
+
expect(system.content[0].text).to.include('Always respond in Spanish.');
|
|
282
490
|
});
|
|
491
|
+
|
|
492
|
+
await model.message();
|
|
283
493
|
});
|
|
284
494
|
|
|
285
|
-
it('
|
|
286
|
-
model.
|
|
287
|
-
.
|
|
288
|
-
.
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
495
|
+
it('renders assigned files through EJS includes, including their relative includes', async () => {
|
|
496
|
+
const model = ModelMix.new()
|
|
497
|
+
.gpt51()
|
|
498
|
+
.assign({
|
|
499
|
+
name: 'Eve',
|
|
500
|
+
platform: 'ModelMix',
|
|
501
|
+
username: 'eve_user',
|
|
502
|
+
role: 'developer',
|
|
503
|
+
createdDate: '2026-08-07',
|
|
504
|
+
website: 'https://modelmix.dev',
|
|
505
|
+
company: 'AI Solutions',
|
|
506
|
+
showAccount: true
|
|
292
507
|
})
|
|
293
|
-
.
|
|
508
|
+
.assignKeyFromFile('templateSource', path.join(fixturesPath, 'template.txt'))
|
|
509
|
+
.addText('Source:\n<%- templateSource %>');
|
|
510
|
+
|
|
511
|
+
mockOpenAI(body => {
|
|
512
|
+
const content = userTexts(body)[0];
|
|
513
|
+
expect(content).to.include('Hello Eve, welcome to ModelMix!');
|
|
514
|
+
expect(content).to.include('Username: eve_user');
|
|
515
|
+
expect(content).to.include('The AI Solutions Team');
|
|
516
|
+
expect(content).to.not.include('<%-');
|
|
517
|
+
});
|
|
294
518
|
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
519
|
+
await model.message();
|
|
520
|
+
});
|
|
521
|
+
|
|
522
|
+
it('inherits assigned files through new instances', async () => {
|
|
523
|
+
const base = ModelMix.new()
|
|
524
|
+
.assign({ language: 'Spanish' })
|
|
525
|
+
.assignKeyFromFile('rules', path.join(fixturesPath, 'system-rules.txt'));
|
|
526
|
+
const model = base.new().gpt51().addText('Rules:\n<%- rules %>');
|
|
527
|
+
|
|
528
|
+
mockOpenAI(body => {
|
|
529
|
+
expect(userTexts(body)[0].trim()).to.equal('Rules:\nAlways respond in Spanish.');
|
|
530
|
+
});
|
|
531
|
+
|
|
532
|
+
await model.message();
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
it('uses the latest assignment when a plain value and a file share a key', async () => {
|
|
536
|
+
const plainValue = ModelMix.new()
|
|
537
|
+
.gpt51()
|
|
538
|
+
.assign({ language: 'Spanish' })
|
|
539
|
+
.assignKeyFromFile('rules', path.join(fixturesPath, 'system-rules.txt'))
|
|
540
|
+
.assignKey('rules', 'Use the plain value.')
|
|
541
|
+
.addText('<%- rules %>');
|
|
542
|
+
|
|
543
|
+
mockOpenAI(body => {
|
|
544
|
+
expect(userTexts(body)).to.deep.equal(['Use the plain value.']);
|
|
545
|
+
}, 'Plain response');
|
|
546
|
+
await plainValue.message();
|
|
547
|
+
|
|
548
|
+
const fileValue = ModelMix.new()
|
|
549
|
+
.gpt51()
|
|
550
|
+
.assign({ language: 'Spanish', rules: 'Ignore this value.' })
|
|
551
|
+
.assignKeyFromFile('rules', path.join(fixturesPath, 'system-rules.txt'))
|
|
552
|
+
.addText('<%- rules %>');
|
|
553
|
+
|
|
554
|
+
mockOpenAI(body => {
|
|
555
|
+
expect(userTexts(body)[0].trim()).to.equal('Always respond in Spanish.');
|
|
556
|
+
}, 'File response');
|
|
557
|
+
await fileValue.message();
|
|
558
|
+
});
|
|
559
|
+
|
|
560
|
+
it('injects JSON file contents without XML escaping', async () => {
|
|
561
|
+
const model = ModelMix.new()
|
|
562
|
+
.gpt51()
|
|
563
|
+
.assignKeyFromFile('data', path.join(fixturesPath, 'data.json'))
|
|
564
|
+
.addText('Process this data:\n<%- data %>');
|
|
565
|
+
|
|
566
|
+
mockOpenAI(body => {
|
|
567
|
+
const content = userTexts(body)[0];
|
|
568
|
+
expect(content).to.include('Alice Smith');
|
|
569
|
+
expect(content).to.include('alice@example.com');
|
|
570
|
+
expect(content).to.include('"theme": "dark"');
|
|
571
|
+
});
|
|
305
572
|
|
|
306
|
-
|
|
307
|
-
expect(response).to.include('Complex template integration successful');
|
|
573
|
+
await model.message();
|
|
308
574
|
});
|
|
309
575
|
|
|
310
|
-
it('
|
|
311
|
-
const
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
576
|
+
it('reloads an assigned file for each request', async () => {
|
|
577
|
+
const temporaryDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'modelmix-template-'));
|
|
578
|
+
const assignedFile = path.join(temporaryDirectory, 'assigned.ejs');
|
|
579
|
+
|
|
580
|
+
try {
|
|
581
|
+
fs.writeFileSync(assignedFile, 'Version one for <%- name %>.');
|
|
582
|
+
const model = ModelMix.new()
|
|
583
|
+
.gpt51()
|
|
584
|
+
.assign({ name: 'Eve' })
|
|
585
|
+
.assignKeyFromFile('content', assignedFile)
|
|
586
|
+
.addText('<%- content %>');
|
|
587
|
+
|
|
588
|
+
mockOpenAI(body => {
|
|
589
|
+
expect(userTexts(body)).to.deep.equal(['Version one for Eve.']);
|
|
590
|
+
}, 'First response');
|
|
591
|
+
await model.message();
|
|
592
|
+
|
|
593
|
+
fs.writeFileSync(assignedFile, 'Version two for <%- name %>.');
|
|
594
|
+
model.assign({ name: 'Ada' }).addText('<%- content %>');
|
|
595
|
+
mockOpenAI(body => {
|
|
596
|
+
expect(userTexts(body)).to.deep.equal(['Version two for Ada.']);
|
|
597
|
+
}, 'Second response');
|
|
598
|
+
await model.message();
|
|
599
|
+
} finally {
|
|
600
|
+
fs.rmSync(temporaryDirectory, { recursive: true, force: true });
|
|
601
|
+
}
|
|
602
|
+
});
|
|
603
|
+
|
|
604
|
+
it('throws immediately when a template or data file is missing', () => {
|
|
605
|
+
const model = ModelMix.new().gpt51();
|
|
606
|
+
const missingPath = path.join(fixturesPath, 'nonexistent.txt');
|
|
607
|
+
|
|
608
|
+
expect(() => model.addTextFromFile(missingPath)).to.throw(`File not found: ${missingPath}`);
|
|
609
|
+
expect(() => model.assignKeyFromFile('missing', missingPath)).to.throw(`File not found: ${missingPath}`);
|
|
610
|
+
expect(() => model.assignKeyFromFile('', missingPath)).to.throw(
|
|
611
|
+
TypeError,
|
|
612
|
+
'Template data key must be a non-empty string.'
|
|
613
|
+
);
|
|
614
|
+
expect(() => model.assignKeyFromFile('$mix', missingPath)).to.throw(
|
|
615
|
+
TypeError,
|
|
616
|
+
'Template data key "$mix" is reserved.'
|
|
617
|
+
);
|
|
618
|
+
});
|
|
619
|
+
});
|
|
317
620
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
621
|
+
describe('Execution integration', () => {
|
|
622
|
+
it('renders system and message templates for JSON output', async () => {
|
|
623
|
+
const schema = { summary: 'Analysis summary', userCount: 0 };
|
|
624
|
+
const model = ModelMix.new()
|
|
625
|
+
.gpt51()
|
|
626
|
+
.setSystem('You are a <%- role %>.')
|
|
627
|
+
.assign({ role: 'data analyst', instruction: 'Count active users' })
|
|
628
|
+
.assignKeyFromFile('data', path.join(fixturesPath, 'data.json'))
|
|
629
|
+
.addText('<%- instruction %> from this data: <%- data %>');
|
|
322
630
|
|
|
323
631
|
nock('https://api.openai.com')
|
|
324
632
|
.post('/v1/responses')
|
|
325
633
|
.reply(function (uri, body) {
|
|
326
|
-
const
|
|
327
|
-
expect(
|
|
328
|
-
expect(
|
|
634
|
+
const system = body.input.find(message => message.role === 'developer');
|
|
635
|
+
expect(system.content[0].text).to.include('You are a data analyst.');
|
|
636
|
+
expect(system.content[0].text).to.include('Output JSON Schema');
|
|
637
|
+
expect(userTexts(body)[0]).to.include('Count active users');
|
|
329
638
|
return [200, {
|
|
330
639
|
output: [{
|
|
331
640
|
type: 'message',
|
|
332
|
-
content: [{
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
roles: ['admin', 'user', 'moderator']
|
|
337
|
-
}) }]
|
|
641
|
+
content: [{
|
|
642
|
+
type: 'output_text',
|
|
643
|
+
text: JSON.stringify({ summary: 'Complete', userCount: 3 })
|
|
644
|
+
}]
|
|
338
645
|
}],
|
|
339
646
|
usage: { input_tokens: 10, output_tokens: 5, total_tokens: 15 }
|
|
340
647
|
}];
|
|
341
648
|
});
|
|
342
649
|
|
|
343
650
|
const result = await model.json(schema);
|
|
344
|
-
expect(result
|
|
345
|
-
expect(result.user_count).to.equal(3);
|
|
346
|
-
expect(result.active_users).to.equal(2);
|
|
347
|
-
expect(result.roles).to.deep.equal(['admin', 'user', 'moderator']);
|
|
651
|
+
expect(result).to.deep.equal({ summary: 'Complete', userCount: 3 });
|
|
348
652
|
});
|
|
349
|
-
});
|
|
350
|
-
|
|
351
|
-
describe('Error Handling', () => {
|
|
352
|
-
let model;
|
|
353
653
|
|
|
354
|
-
|
|
355
|
-
model = ModelMix.new(
|
|
356
|
-
|
|
357
|
-
|
|
654
|
+
it('renders the system before adding block instructions', async () => {
|
|
655
|
+
const model = ModelMix.new()
|
|
656
|
+
.gpt51()
|
|
657
|
+
.setSystem('Act as <%- role %>.')
|
|
658
|
+
.assign({ role: 'reviewer' })
|
|
659
|
+
.addText('Review this.');
|
|
660
|
+
|
|
661
|
+
mockOpenAI(body => {
|
|
662
|
+
const system = body.input.find(message => message.role === 'developer');
|
|
663
|
+
expect(system.content[0].text).to.equal(
|
|
664
|
+
'Act as reviewer.\nReturn the result of the task between triple backtick block code tags ```'
|
|
665
|
+
);
|
|
666
|
+
}, '```\napproved\n```');
|
|
667
|
+
|
|
668
|
+
expect(await model.block()).to.equal('approved');
|
|
358
669
|
});
|
|
359
670
|
|
|
360
|
-
it('
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
671
|
+
it('keeps rendered history snapshots when template data changes', async () => {
|
|
672
|
+
const model = ModelMix.new({ config: { max_history: 10 } })
|
|
673
|
+
.gpt51()
|
|
674
|
+
.assign({ name: 'Alice' })
|
|
675
|
+
.addText('Hello <%- name %>.');
|
|
676
|
+
|
|
677
|
+
mockOpenAI(body => {
|
|
678
|
+
expect(userTexts(body)).to.deep.equal(['Hello Alice.']);
|
|
679
|
+
}, 'First response');
|
|
680
|
+
await model.message();
|
|
681
|
+
|
|
682
|
+
model.assign({ name: 'Bob' }).addText('Hello <%- name %>.');
|
|
683
|
+
mockOpenAI(body => {
|
|
684
|
+
expect(userTexts(body)).to.deep.equal(['Hello Alice.', 'Hello Bob.']);
|
|
685
|
+
}, 'Second response');
|
|
686
|
+
await model.message();
|
|
368
687
|
});
|
|
369
688
|
|
|
370
|
-
it('
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
.
|
|
689
|
+
it('keeps a system choice stable across provider fallback', async () => {
|
|
690
|
+
const systems = [];
|
|
691
|
+
const model = ModelMix.new()
|
|
692
|
+
.gpt51()
|
|
693
|
+
.sonnet46()
|
|
694
|
+
.setSystem(`<% choice %>
|
|
695
|
+
<% option %>
|
|
696
|
+
First system.
|
|
697
|
+
<% option %>
|
|
698
|
+
Second system.
|
|
699
|
+
<% /choice %>`)
|
|
700
|
+
.addText('Hello');
|
|
701
|
+
const random = sinon.stub(model, '_choiceRandom').returns(0.9);
|
|
374
702
|
|
|
375
703
|
nock('https://api.openai.com')
|
|
376
704
|
.post('/v1/responses')
|
|
377
|
-
.reply(
|
|
705
|
+
.reply(function (uri, body) {
|
|
706
|
+
systems.push(body.input.find(message => message.role === 'developer').content[0].text.trim());
|
|
707
|
+
return [500, { error: 'temporary failure' }];
|
|
708
|
+
});
|
|
709
|
+
nock('https://api.anthropic.com')
|
|
710
|
+
.post('/v1/messages')
|
|
711
|
+
.reply(function (uri, body) {
|
|
712
|
+
systems.push(body.system.trim());
|
|
713
|
+
return [200, {
|
|
714
|
+
content: [{ type: 'text', text: 'Fallback response' }],
|
|
715
|
+
usage: { input_tokens: 10, output_tokens: 5 }
|
|
716
|
+
}];
|
|
717
|
+
});
|
|
378
718
|
|
|
379
|
-
|
|
380
|
-
expect(
|
|
719
|
+
expect(await model.message()).to.equal('Fallback response');
|
|
720
|
+
expect(systems).to.deep.equal(['Second system.', 'Second system.']);
|
|
721
|
+
expect(random.callCount).to.equal(1);
|
|
381
722
|
});
|
|
382
723
|
});
|
|
383
|
-
});
|
|
724
|
+
});
|