@layer-ai/core 2.0.11 → 2.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/routes/tests/test-chat-structured-output.d.ts +7 -0
- package/dist/routes/tests/test-chat-structured-output.d.ts.map +1 -0
- package/dist/routes/tests/test-chat-structured-output.js +320 -0
- package/dist/routes/v1/gates.d.ts.map +1 -1
- package/dist/routes/v1/gates.js +11 -2
- package/dist/routes/v3/chat.d.ts +2 -0
- package/dist/routes/v3/chat.d.ts.map +1 -1
- package/dist/routes/v3/chat.js +8 -9
- package/dist/services/providers/tests/test-json-response-format-anthropic.d.ts +7 -0
- package/dist/services/providers/tests/test-json-response-format-anthropic.d.ts.map +1 -0
- package/dist/services/providers/tests/test-json-response-format-anthropic.js +179 -0
- package/dist/services/providers/tests/test-json-response-format-google.d.ts +7 -0
- package/dist/services/providers/tests/test-json-response-format-google.d.ts.map +1 -0
- package/dist/services/providers/tests/test-json-response-format-google.js +179 -0
- package/dist/services/providers/tests/test-json-response-format-mistral.d.ts +7 -0
- package/dist/services/providers/tests/test-json-response-format-mistral.d.ts.map +1 -0
- package/dist/services/providers/tests/test-json-response-format-mistral.js +179 -0
- package/dist/services/providers/tests/test-json-response-format-openai.d.ts +7 -0
- package/dist/services/providers/tests/test-json-response-format-openai.d.ts.map +1 -0
- package/dist/services/providers/tests/test-json-response-format-openai.js +175 -0
- package/dist/services/providers/tests/test-openai-adapter.js +76 -0
- package/package.json +10 -10
- package/LICENSE +0 -21
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-chat-structured-output.d.ts","sourceRoot":"","sources":["../../../src/routes/tests/test-chat-structured-output.ts"],"names":[],"mappings":";AACA;;;GAGG"}
|
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
#!/usr/bin/env npx tsx
|
|
2
|
+
/**
|
|
3
|
+
* Test structured output at the chat route level
|
|
4
|
+
* Tests that gate configuration flows correctly through resolveFinalRequest
|
|
5
|
+
*/
|
|
6
|
+
// We'll test the resolveFinalRequest function directly
|
|
7
|
+
// This simulates what happens in the chat route
|
|
8
|
+
console.log('='.repeat(80));
|
|
9
|
+
console.log('STRUCTURED OUTPUT - CHAT ROUTE LEVEL TESTS');
|
|
10
|
+
console.log('='.repeat(80));
|
|
11
|
+
console.log('');
|
|
12
|
+
// Mock gate config
|
|
13
|
+
const baseGateConfig = {
|
|
14
|
+
id: 'test-gate-id',
|
|
15
|
+
name: 'Test Gate',
|
|
16
|
+
model: 'gpt-4o',
|
|
17
|
+
taskType: 'chat',
|
|
18
|
+
temperature: 0.7,
|
|
19
|
+
maxTokens: 100,
|
|
20
|
+
topP: 0.9,
|
|
21
|
+
systemPrompt: undefined,
|
|
22
|
+
routingStrategy: 'single',
|
|
23
|
+
responseFormatEnabled: false,
|
|
24
|
+
responseFormatType: 'text',
|
|
25
|
+
responseFormatSchema: undefined,
|
|
26
|
+
// Add other required fields for Gate
|
|
27
|
+
userId: 'test-user',
|
|
28
|
+
createdAt: new Date(),
|
|
29
|
+
updatedAt: new Date(),
|
|
30
|
+
};
|
|
31
|
+
async function testOpenAIJsonObjectMode() {
|
|
32
|
+
console.log('Test 1: OpenAI - JSON Object Mode (Native)');
|
|
33
|
+
console.log('-'.repeat(80));
|
|
34
|
+
const gateConfig = {
|
|
35
|
+
...baseGateConfig,
|
|
36
|
+
model: 'gpt-4o',
|
|
37
|
+
responseFormatEnabled: true,
|
|
38
|
+
responseFormatType: 'json_object',
|
|
39
|
+
};
|
|
40
|
+
const incomingRequest = {
|
|
41
|
+
gateId: 'test-gate',
|
|
42
|
+
type: 'chat',
|
|
43
|
+
data: {
|
|
44
|
+
messages: [
|
|
45
|
+
{ role: 'user', content: 'Generate a user profile' }
|
|
46
|
+
],
|
|
47
|
+
}
|
|
48
|
+
};
|
|
49
|
+
try {
|
|
50
|
+
// Import the function we need to test
|
|
51
|
+
const { resolveFinalRequest } = await import('../v3/chat.js');
|
|
52
|
+
const resolvedRequest = resolveFinalRequest(gateConfig, incomingRequest);
|
|
53
|
+
console.log('✓ Request resolved');
|
|
54
|
+
console.log('');
|
|
55
|
+
console.log('Resolved request model:', resolvedRequest.model);
|
|
56
|
+
console.log('Resolved request responseFormat:', resolvedRequest.data.responseFormat);
|
|
57
|
+
console.log('');
|
|
58
|
+
// Validate
|
|
59
|
+
if (resolvedRequest.data.responseFormat !== 'json_object') {
|
|
60
|
+
throw new Error(`Expected responseFormat 'json_object', got: ${resolvedRequest.data.responseFormat}`);
|
|
61
|
+
}
|
|
62
|
+
console.log('✅ OpenAI JSON Object Mode: PASSED');
|
|
63
|
+
console.log('');
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
console.log('❌ OpenAI JSON Object Mode: FAILED');
|
|
68
|
+
console.log('Error:', error.message);
|
|
69
|
+
console.log('');
|
|
70
|
+
return false;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
async function testOpenAIJsonSchemaMode() {
|
|
74
|
+
console.log('Test 2: OpenAI - JSON Schema Mode (Native)');
|
|
75
|
+
console.log('-'.repeat(80));
|
|
76
|
+
const schema = {
|
|
77
|
+
type: 'json_schema',
|
|
78
|
+
json_schema: {
|
|
79
|
+
name: 'user_profile',
|
|
80
|
+
strict: true,
|
|
81
|
+
schema: {
|
|
82
|
+
type: 'object',
|
|
83
|
+
properties: {
|
|
84
|
+
name: { type: 'string' },
|
|
85
|
+
age: { type: 'number' },
|
|
86
|
+
city: { type: 'string' },
|
|
87
|
+
},
|
|
88
|
+
required: ['name', 'age', 'city'],
|
|
89
|
+
additionalProperties: false,
|
|
90
|
+
},
|
|
91
|
+
},
|
|
92
|
+
};
|
|
93
|
+
const gateConfig = {
|
|
94
|
+
...baseGateConfig,
|
|
95
|
+
model: 'gpt-4o',
|
|
96
|
+
responseFormatEnabled: true,
|
|
97
|
+
responseFormatType: 'json_schema',
|
|
98
|
+
responseFormatSchema: schema,
|
|
99
|
+
};
|
|
100
|
+
const incomingRequest = {
|
|
101
|
+
gateId: 'test-gate',
|
|
102
|
+
type: 'chat',
|
|
103
|
+
data: {
|
|
104
|
+
messages: [
|
|
105
|
+
{ role: 'user', content: 'Create a profile for John Doe, age 30, from New York' }
|
|
106
|
+
],
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
try {
|
|
110
|
+
const { resolveFinalRequest } = await import('../v3/chat.js');
|
|
111
|
+
const resolvedRequest = resolveFinalRequest(gateConfig, incomingRequest);
|
|
112
|
+
console.log('✓ Request resolved');
|
|
113
|
+
console.log('');
|
|
114
|
+
console.log('Resolved request model:', resolvedRequest.model);
|
|
115
|
+
console.log('Resolved request responseFormat type:', resolvedRequest.data.responseFormat?.type);
|
|
116
|
+
console.log('');
|
|
117
|
+
// Validate
|
|
118
|
+
const responseFormat = resolvedRequest.data.responseFormat;
|
|
119
|
+
if (!responseFormat || responseFormat.type !== 'json_schema') {
|
|
120
|
+
throw new Error(`Expected responseFormat.type 'json_schema', got: ${responseFormat?.type}`);
|
|
121
|
+
}
|
|
122
|
+
if (!responseFormat.json_schema) {
|
|
123
|
+
throw new Error('Expected json_schema property in responseFormat');
|
|
124
|
+
}
|
|
125
|
+
console.log('✅ OpenAI JSON Schema Mode: PASSED');
|
|
126
|
+
console.log('');
|
|
127
|
+
return true;
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
console.log('❌ OpenAI JSON Schema Mode: FAILED');
|
|
131
|
+
console.log('Error:', error.message);
|
|
132
|
+
console.log('');
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
async function testAnthropicBetaMode() {
|
|
137
|
+
console.log('Test 3: Anthropic - JSON Object Mode (Beta - Prompt Injection)');
|
|
138
|
+
console.log('-'.repeat(80));
|
|
139
|
+
const gateConfig = {
|
|
140
|
+
...baseGateConfig,
|
|
141
|
+
model: 'claude-3-7-sonnet-20250219',
|
|
142
|
+
responseFormatEnabled: true,
|
|
143
|
+
responseFormatType: 'json_object',
|
|
144
|
+
};
|
|
145
|
+
const incomingRequest = {
|
|
146
|
+
gateId: 'test-gate',
|
|
147
|
+
type: 'chat',
|
|
148
|
+
data: {
|
|
149
|
+
messages: [
|
|
150
|
+
{ role: 'user', content: 'Generate a user profile' }
|
|
151
|
+
],
|
|
152
|
+
}
|
|
153
|
+
};
|
|
154
|
+
try {
|
|
155
|
+
const { resolveFinalRequest } = await import('../v3/chat.js');
|
|
156
|
+
const resolvedRequest = resolveFinalRequest(gateConfig, incomingRequest);
|
|
157
|
+
console.log('✓ Request resolved');
|
|
158
|
+
console.log('');
|
|
159
|
+
console.log('Resolved request model:', resolvedRequest.model);
|
|
160
|
+
console.log('System prompt length:', resolvedRequest.data.systemPrompt?.length || 0);
|
|
161
|
+
console.log('');
|
|
162
|
+
// Validate - should have JSON instructions in system prompt
|
|
163
|
+
const systemPrompt = resolvedRequest.data.systemPrompt || '';
|
|
164
|
+
if (!systemPrompt.includes('JSON')) {
|
|
165
|
+
throw new Error('Expected JSON instructions in system prompt for beta mode');
|
|
166
|
+
}
|
|
167
|
+
// Should NOT have responseFormat set (beta mode uses prompt injection)
|
|
168
|
+
if (resolvedRequest.data.responseFormat && resolvedRequest.data.responseFormat !== 'text') {
|
|
169
|
+
throw new Error('Beta mode should not set responseFormat');
|
|
170
|
+
}
|
|
171
|
+
console.log('✓ System prompt contains JSON instructions');
|
|
172
|
+
console.log('System prompt excerpt:');
|
|
173
|
+
console.log(systemPrompt.substring(0, 150) + '...');
|
|
174
|
+
console.log('');
|
|
175
|
+
console.log('✅ Anthropic Beta Mode: PASSED');
|
|
176
|
+
console.log('');
|
|
177
|
+
return true;
|
|
178
|
+
}
|
|
179
|
+
catch (error) {
|
|
180
|
+
console.log('❌ Anthropic Beta Mode: FAILED');
|
|
181
|
+
console.log('Error:', error.message);
|
|
182
|
+
console.log('');
|
|
183
|
+
return false;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
async function testGoogleBetaMode() {
|
|
187
|
+
console.log('Test 4: Google - JSON Schema Mode (Beta - Prompt Injection)');
|
|
188
|
+
console.log('-'.repeat(80));
|
|
189
|
+
const schema = {
|
|
190
|
+
type: 'json_schema',
|
|
191
|
+
json_schema: {
|
|
192
|
+
name: 'user_profile',
|
|
193
|
+
strict: true,
|
|
194
|
+
schema: {
|
|
195
|
+
type: 'object',
|
|
196
|
+
properties: {
|
|
197
|
+
name: { type: 'string' },
|
|
198
|
+
age: { type: 'number' },
|
|
199
|
+
},
|
|
200
|
+
required: ['name', 'age'],
|
|
201
|
+
additionalProperties: false,
|
|
202
|
+
},
|
|
203
|
+
},
|
|
204
|
+
};
|
|
205
|
+
const gateConfig = {
|
|
206
|
+
...baseGateConfig,
|
|
207
|
+
model: 'gemini-2.0-flash',
|
|
208
|
+
responseFormatEnabled: true,
|
|
209
|
+
responseFormatType: 'json_schema',
|
|
210
|
+
responseFormatSchema: schema,
|
|
211
|
+
};
|
|
212
|
+
const incomingRequest = {
|
|
213
|
+
gateId: 'test-gate',
|
|
214
|
+
type: 'chat',
|
|
215
|
+
data: {
|
|
216
|
+
messages: [
|
|
217
|
+
{ role: 'user', content: 'Create a profile' }
|
|
218
|
+
],
|
|
219
|
+
}
|
|
220
|
+
};
|
|
221
|
+
try {
|
|
222
|
+
const { resolveFinalRequest } = await import('../v3/chat.js');
|
|
223
|
+
const resolvedRequest = resolveFinalRequest(gateConfig, incomingRequest);
|
|
224
|
+
console.log('✓ Request resolved');
|
|
225
|
+
console.log('');
|
|
226
|
+
console.log('Resolved request model:', resolvedRequest.model);
|
|
227
|
+
console.log('System prompt length:', resolvedRequest.data.systemPrompt?.length || 0);
|
|
228
|
+
console.log('');
|
|
229
|
+
// Validate - should have schema in system prompt
|
|
230
|
+
const systemPrompt = resolvedRequest.data.systemPrompt || '';
|
|
231
|
+
if (!systemPrompt.includes('JSON')) {
|
|
232
|
+
throw new Error('Expected JSON instructions in system prompt');
|
|
233
|
+
}
|
|
234
|
+
if (!systemPrompt.includes('"name"') || !systemPrompt.includes('"age"')) {
|
|
235
|
+
throw new Error('Expected schema properties in system prompt');
|
|
236
|
+
}
|
|
237
|
+
console.log('✓ System prompt contains schema instructions');
|
|
238
|
+
console.log('✅ Google Beta Mode: PASSED');
|
|
239
|
+
console.log('');
|
|
240
|
+
return true;
|
|
241
|
+
}
|
|
242
|
+
catch (error) {
|
|
243
|
+
console.log('❌ Google Beta Mode: FAILED');
|
|
244
|
+
console.log('Error:', error.message);
|
|
245
|
+
console.log('');
|
|
246
|
+
return false;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
async function testTextModeNoChanges() {
|
|
250
|
+
console.log('Test 5: Text Mode - No Modifications');
|
|
251
|
+
console.log('-'.repeat(80));
|
|
252
|
+
const gateConfig = {
|
|
253
|
+
...baseGateConfig,
|
|
254
|
+
responseFormatEnabled: false,
|
|
255
|
+
responseFormatType: 'text',
|
|
256
|
+
};
|
|
257
|
+
const incomingRequest = {
|
|
258
|
+
gateId: 'test-gate',
|
|
259
|
+
type: 'chat',
|
|
260
|
+
data: {
|
|
261
|
+
messages: [
|
|
262
|
+
{ role: 'user', content: 'Say hello' }
|
|
263
|
+
],
|
|
264
|
+
}
|
|
265
|
+
};
|
|
266
|
+
try {
|
|
267
|
+
const { resolveFinalRequest } = await import('../v3/chat.js');
|
|
268
|
+
const resolvedRequest = resolveFinalRequest(gateConfig, incomingRequest);
|
|
269
|
+
console.log('✓ Request resolved');
|
|
270
|
+
console.log('');
|
|
271
|
+
// Validate - should NOT have responseFormat or JSON instructions
|
|
272
|
+
if (resolvedRequest.data.responseFormat && resolvedRequest.data.responseFormat !== 'text') {
|
|
273
|
+
throw new Error(`Expected no responseFormat, got: ${resolvedRequest.data.responseFormat}`);
|
|
274
|
+
}
|
|
275
|
+
const systemPrompt = resolvedRequest.data.systemPrompt || '';
|
|
276
|
+
if (systemPrompt.includes('JSON')) {
|
|
277
|
+
throw new Error('Should not have JSON instructions for text mode');
|
|
278
|
+
}
|
|
279
|
+
console.log('✓ No response format applied');
|
|
280
|
+
console.log('✅ Text Mode: PASSED');
|
|
281
|
+
console.log('');
|
|
282
|
+
return true;
|
|
283
|
+
}
|
|
284
|
+
catch (error) {
|
|
285
|
+
console.log('❌ Text Mode: FAILED');
|
|
286
|
+
console.log('Error:', error.message);
|
|
287
|
+
console.log('');
|
|
288
|
+
return false;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
async function runTests() {
|
|
292
|
+
const results = [];
|
|
293
|
+
results.push(await testOpenAIJsonObjectMode());
|
|
294
|
+
results.push(await testOpenAIJsonSchemaMode());
|
|
295
|
+
results.push(await testAnthropicBetaMode());
|
|
296
|
+
results.push(await testGoogleBetaMode());
|
|
297
|
+
results.push(await testTextModeNoChanges());
|
|
298
|
+
console.log('='.repeat(80));
|
|
299
|
+
console.log('RESULTS SUMMARY');
|
|
300
|
+
console.log('='.repeat(80));
|
|
301
|
+
console.log('');
|
|
302
|
+
const passed = results.filter(r => r).length;
|
|
303
|
+
const failed = results.filter(r => !r).length;
|
|
304
|
+
console.log(`Total Tests: ${results.length}`);
|
|
305
|
+
console.log(`Passed: ${passed}`);
|
|
306
|
+
console.log(`Failed: ${failed}`);
|
|
307
|
+
console.log('');
|
|
308
|
+
if (failed > 0) {
|
|
309
|
+
console.log('❌ Some tests failed');
|
|
310
|
+
process.exit(1);
|
|
311
|
+
}
|
|
312
|
+
else {
|
|
313
|
+
console.log('✅ All tests passed!');
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
runTests().catch(error => {
|
|
317
|
+
console.error('Test suite failed:', error);
|
|
318
|
+
process.exit(1);
|
|
319
|
+
});
|
|
320
|
+
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gates.d.ts","sourceRoot":"","sources":["../../../src/routes/v1/gates.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,SAAS,CAAC;AASpD,QAAA,MAAM,MAAM,EAAE,UAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"gates.d.ts","sourceRoot":"","sources":["../../../src/routes/v1/gates.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,SAAS,CAAC;AASpD,QAAA,MAAM,MAAM,EAAE,UAAqB,CAAC;AAuhBpC,eAAe,MAAM,CAAC"}
|
package/dist/routes/v1/gates.js
CHANGED
|
@@ -15,7 +15,7 @@ router.post('/', async (req, res) => {
|
|
|
15
15
|
return;
|
|
16
16
|
}
|
|
17
17
|
try {
|
|
18
|
-
const { name, description, taskType, model, systemPrompt, allowOverrides, temperature, maxTokens, topP, tags, routingStrategy, fallbackModels, costWeight, latencyWeight, qualityWeight, reanalysisPeriod, taskAnalysis } = req.body;
|
|
18
|
+
const { name, description, taskType, model, systemPrompt, allowOverrides, temperature, maxTokens, topP, tags, routingStrategy, fallbackModels, costWeight, latencyWeight, qualityWeight, reanalysisPeriod, taskAnalysis, responseFormatEnabled, responseFormatType, responseFormatSchema } = req.body;
|
|
19
19
|
if (!name || !model) {
|
|
20
20
|
res.status(400).json({ error: 'bad_request', message: 'Missing required fields: name and model' });
|
|
21
21
|
return;
|
|
@@ -47,6 +47,9 @@ router.post('/', async (req, res) => {
|
|
|
47
47
|
qualityWeight,
|
|
48
48
|
reanalysisPeriod,
|
|
49
49
|
taskAnalysis,
|
|
50
|
+
responseFormatEnabled,
|
|
51
|
+
responseFormatType,
|
|
52
|
+
responseFormatSchema,
|
|
50
53
|
});
|
|
51
54
|
res.status(201).json(gate);
|
|
52
55
|
}
|
|
@@ -193,7 +196,7 @@ router.patch('/:id', async (req, res) => {
|
|
|
193
196
|
return;
|
|
194
197
|
}
|
|
195
198
|
try {
|
|
196
|
-
const { name, description, taskType, model, systemPrompt, allowOverrides, temperature, maxTokens, topP, tags, routingStrategy, fallbackModels, costWeight, latencyWeight, qualityWeight, analysisMethod, reanalysisPeriod, taskAnalysis, autoApplyRecommendations } = req.body;
|
|
199
|
+
const { name, description, taskType, model, systemPrompt, allowOverrides, temperature, maxTokens, topP, tags, routingStrategy, fallbackModels, costWeight, latencyWeight, qualityWeight, analysisMethod, reanalysisPeriod, taskAnalysis, autoApplyRecommendations, responseFormatEnabled, responseFormatType, responseFormatSchema } = req.body;
|
|
197
200
|
const existing = await db.getGateById(req.params.id);
|
|
198
201
|
if (!existing) {
|
|
199
202
|
res.status(404).json({ error: 'not_found', message: 'Gate not found' });
|
|
@@ -225,6 +228,9 @@ router.patch('/:id', async (req, res) => {
|
|
|
225
228
|
analysisMethod,
|
|
226
229
|
reanalysisPeriod,
|
|
227
230
|
autoApplyRecommendations,
|
|
231
|
+
responseFormatEnabled,
|
|
232
|
+
responseFormatType,
|
|
233
|
+
responseFormatSchema,
|
|
228
234
|
});
|
|
229
235
|
const updated = await db.updateGate(req.params.id, {
|
|
230
236
|
name,
|
|
@@ -246,6 +252,9 @@ router.patch('/:id', async (req, res) => {
|
|
|
246
252
|
reanalysisPeriod,
|
|
247
253
|
taskAnalysis,
|
|
248
254
|
autoApplyRecommendations,
|
|
255
|
+
responseFormatEnabled,
|
|
256
|
+
responseFormatType,
|
|
257
|
+
responseFormatSchema,
|
|
249
258
|
});
|
|
250
259
|
// Only create history snapshot if significant changes were detected
|
|
251
260
|
if (updated && changedFields.length > 0) {
|
package/dist/routes/v3/chat.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { Router as RouterType } from 'express';
|
|
2
|
+
import type { LayerRequest, Gate } from '@layer-ai/sdk';
|
|
2
3
|
declare const router: RouterType;
|
|
4
|
+
export declare function resolveFinalRequest(gateConfig: Gate, request: LayerRequest): LayerRequest;
|
|
3
5
|
export default router;
|
|
4
6
|
//# sourceMappingURL=chat.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../../src/routes/v3/chat.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"chat.d.ts","sourceRoot":"","sources":["../../../src/routes/v3/chat.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,SAAS,CAAC;AAIpD,OAAO,KAAK,EAAE,YAAY,EAAiB,IAAI,EAA+C,MAAM,eAAe,CAAC;AAGpH,QAAA,MAAM,MAAM,EAAE,UAAqB,CAAC;AAiBpC,wBAAgB,mBAAmB,CACjC,UAAU,EAAE,IAAI,EAChB,OAAO,EAAE,YAAY,GACpB,YAAY,CAkFd;AA6MD,eAAe,MAAM,CAAC"}
|
package/dist/routes/v3/chat.js
CHANGED
|
@@ -12,7 +12,7 @@ function isOverrideAllowed(allowOverrides, field) {
|
|
|
12
12
|
return false;
|
|
13
13
|
return allowOverrides[field] ?? false;
|
|
14
14
|
}
|
|
15
|
-
function resolveFinalRequest(gateConfig, request) {
|
|
15
|
+
export function resolveFinalRequest(gateConfig, request) {
|
|
16
16
|
let finalModel = gateConfig.model;
|
|
17
17
|
if (request.model && isOverrideAllowed(gateConfig.allowOverrides, OverrideField.Model)) {
|
|
18
18
|
try {
|
|
@@ -51,10 +51,8 @@ function resolveFinalRequest(gateConfig, request) {
|
|
|
51
51
|
if (modelProvider === PROVIDER.OPENAI) {
|
|
52
52
|
// OpenAI: Use native response_format support
|
|
53
53
|
if (gateConfig.responseFormatType === 'json_schema' && gateConfig.responseFormatSchema) {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
json_schema: gateConfig.responseFormatSchema,
|
|
57
|
-
};
|
|
54
|
+
// Schema is already in the correct format from DB
|
|
55
|
+
chatData.responseFormat = gateConfig.responseFormatSchema;
|
|
58
56
|
}
|
|
59
57
|
else {
|
|
60
58
|
chatData.responseFormat = gateConfig.responseFormatType; // 'text' or 'json_object'
|
|
@@ -65,12 +63,13 @@ function resolveFinalRequest(gateConfig, request) {
|
|
|
65
63
|
if (gateConfig.responseFormatType !== 'text') {
|
|
66
64
|
let schemaInstructions = '';
|
|
67
65
|
if (gateConfig.responseFormatType === 'json_schema' && gateConfig.responseFormatSchema) {
|
|
68
|
-
//
|
|
69
|
-
|
|
66
|
+
// Extract just the schema portion if it's the full OpenAI format
|
|
67
|
+
const schemaObj = gateConfig.responseFormatSchema;
|
|
68
|
+
const actualSchema = schemaObj.json_schema?.schema || schemaObj;
|
|
69
|
+
schemaInstructions = `\n\nYou MUST respond with ONLY a valid JSON object matching this schema. Do not include ANY text before or after the JSON. Do not wrap it in markdown code blocks. Do not add explanations. ONLY output the raw JSON object.\n\nRequired JSON Schema:\n${JSON.stringify(actualSchema, null, 2)}`;
|
|
70
70
|
}
|
|
71
71
|
else if (gateConfig.responseFormatType === 'json_object') {
|
|
72
|
-
|
|
73
|
-
schemaInstructions = `\n\n[STRUCTURED OUTPUT - BETA] You MUST respond with valid JSON only. Do not include any explanatory text, markdown formatting, or code blocks. Output should be a raw JSON object.`;
|
|
72
|
+
schemaInstructions = `\n\nYou MUST respond with ONLY a valid JSON object. Do not include ANY text before or after the JSON. Do not wrap it in markdown code blocks. Do not add explanations. ONLY output the raw JSON object.`;
|
|
74
73
|
}
|
|
75
74
|
// Append to existing system prompt or create new one
|
|
76
75
|
chatData.systemPrompt = (chatData.systemPrompt || '') + schemaInstructions;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-json-response-format-anthropic.d.ts","sourceRoot":"","sources":["../../../../src/services/providers/tests/test-json-response-format-anthropic.ts"],"names":[],"mappings":";AACA;;;GAGG"}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
#!/usr/bin/env npx tsx
|
|
2
|
+
/**
|
|
3
|
+
* Focused tests for JSON response format structure - Anthropic Provider
|
|
4
|
+
* Tests structured output with prompt engineering (beta mode)
|
|
5
|
+
*/
|
|
6
|
+
import { AnthropicAdapter } from '../anthropic-adapter.js';
|
|
7
|
+
const adapter = new AnthropicAdapter();
|
|
8
|
+
console.log('='.repeat(80));
|
|
9
|
+
console.log('STRUCTURED OUTPUT - ANTHROPIC ADAPTER TESTS (BETA MODE)');
|
|
10
|
+
console.log('='.repeat(80));
|
|
11
|
+
console.log('');
|
|
12
|
+
// Helper to extract JSON from markdown or plain text
|
|
13
|
+
function extractJSON(content) {
|
|
14
|
+
const codeBlockMatch = content.match(/```(?:json)?\s*\n?([\s\S]*?)\n?```/);
|
|
15
|
+
if (codeBlockMatch) {
|
|
16
|
+
return JSON.parse(codeBlockMatch[1].trim());
|
|
17
|
+
}
|
|
18
|
+
return JSON.parse(content.trim());
|
|
19
|
+
}
|
|
20
|
+
async function testJsonObjectModeBeta() {
|
|
21
|
+
console.log('Test 1: JSON Object Mode (Beta - Prompt Engineering)');
|
|
22
|
+
console.log('-'.repeat(80));
|
|
23
|
+
const request = {
|
|
24
|
+
gateId: 'test-gate',
|
|
25
|
+
model: 'claude-3-7-sonnet-20250219',
|
|
26
|
+
type: 'chat',
|
|
27
|
+
data: {
|
|
28
|
+
messages: [
|
|
29
|
+
{ role: 'user', content: 'Generate a user profile with name, age, and city fields' }
|
|
30
|
+
],
|
|
31
|
+
systemPrompt: 'You MUST respond with ONLY a valid JSON object. Do not include ANY text before or after the JSON. Do not wrap it in markdown code blocks. Do not add explanations. ONLY output the raw JSON object.',
|
|
32
|
+
maxTokens: 150,
|
|
33
|
+
}
|
|
34
|
+
};
|
|
35
|
+
try {
|
|
36
|
+
const response = await adapter.call(request);
|
|
37
|
+
console.log('✓ Request completed');
|
|
38
|
+
console.log('');
|
|
39
|
+
console.log('Raw response content:');
|
|
40
|
+
console.log(response.content);
|
|
41
|
+
console.log('');
|
|
42
|
+
// Try to parse as JSON (with markdown extraction fallback)
|
|
43
|
+
const parsed = extractJSON(response.content || '');
|
|
44
|
+
console.log('✓ Successfully parsed as JSON');
|
|
45
|
+
console.log('');
|
|
46
|
+
console.log('Parsed structure:');
|
|
47
|
+
console.log(JSON.stringify(parsed, null, 2));
|
|
48
|
+
console.log('');
|
|
49
|
+
console.log('✅ JSON Object Mode (Beta): PASSED');
|
|
50
|
+
console.log('');
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
console.log('❌ JSON Object Mode (Beta): FAILED');
|
|
55
|
+
console.log('Error:', error.message);
|
|
56
|
+
console.log('');
|
|
57
|
+
return false;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
async function testJsonSchemaModeBeta() {
|
|
61
|
+
console.log('Test 2: JSON Schema Mode (Beta - Prompt Engineering)');
|
|
62
|
+
console.log('-'.repeat(80));
|
|
63
|
+
const schema = {
|
|
64
|
+
type: 'object',
|
|
65
|
+
properties: {
|
|
66
|
+
name: { type: 'string' },
|
|
67
|
+
age: { type: 'number' },
|
|
68
|
+
city: { type: 'string' },
|
|
69
|
+
},
|
|
70
|
+
required: ['name', 'age', 'city'],
|
|
71
|
+
additionalProperties: false,
|
|
72
|
+
};
|
|
73
|
+
const systemPrompt = `You MUST respond with ONLY a valid JSON object matching this schema. Do not include ANY text before or after the JSON. Do not wrap it in markdown code blocks. Do not add explanations. ONLY output the raw JSON object.
|
|
74
|
+
|
|
75
|
+
Required JSON Schema:
|
|
76
|
+
${JSON.stringify(schema, null, 2)}`;
|
|
77
|
+
const request = {
|
|
78
|
+
gateId: 'test-gate',
|
|
79
|
+
model: 'claude-3-7-sonnet-20250219',
|
|
80
|
+
type: 'chat',
|
|
81
|
+
data: {
|
|
82
|
+
messages: [
|
|
83
|
+
{ role: 'user', content: 'Create data for: name=Jane Smith, age=25, city=London' }
|
|
84
|
+
],
|
|
85
|
+
systemPrompt,
|
|
86
|
+
maxTokens: 150,
|
|
87
|
+
}
|
|
88
|
+
};
|
|
89
|
+
try {
|
|
90
|
+
const response = await adapter.call(request);
|
|
91
|
+
console.log('✓ Request completed');
|
|
92
|
+
console.log('');
|
|
93
|
+
console.log('Raw response content:');
|
|
94
|
+
console.log(response.content);
|
|
95
|
+
console.log('');
|
|
96
|
+
// Try to parse as JSON (with markdown extraction fallback)
|
|
97
|
+
const parsed = extractJSON(response.content || '');
|
|
98
|
+
console.log('✓ Successfully parsed as JSON');
|
|
99
|
+
console.log('');
|
|
100
|
+
console.log('Parsed structure:');
|
|
101
|
+
console.log(JSON.stringify(parsed, null, 2));
|
|
102
|
+
console.log('');
|
|
103
|
+
// Validate schema fields
|
|
104
|
+
const hasRequiredFields = parsed.name && typeof parsed.age === 'number' && parsed.city;
|
|
105
|
+
if (!hasRequiredFields) {
|
|
106
|
+
throw new Error('Missing required fields from schema');
|
|
107
|
+
}
|
|
108
|
+
console.log('✓ Schema validation passed');
|
|
109
|
+
console.log('');
|
|
110
|
+
console.log('✅ JSON Schema Mode (Beta): PASSED');
|
|
111
|
+
console.log('');
|
|
112
|
+
return true;
|
|
113
|
+
}
|
|
114
|
+
catch (error) {
|
|
115
|
+
console.log('❌ JSON Schema Mode (Beta): FAILED');
|
|
116
|
+
console.log('Error:', error.message);
|
|
117
|
+
console.log('');
|
|
118
|
+
return false;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
async function testTextMode() {
|
|
122
|
+
console.log('Test 3: Text Mode (Control Test)');
|
|
123
|
+
console.log('-'.repeat(80));
|
|
124
|
+
const request = {
|
|
125
|
+
gateId: 'test-gate',
|
|
126
|
+
model: 'claude-3-7-sonnet-20250219',
|
|
127
|
+
type: 'chat',
|
|
128
|
+
data: {
|
|
129
|
+
messages: [
|
|
130
|
+
{ role: 'user', content: 'Say hello' }
|
|
131
|
+
],
|
|
132
|
+
maxTokens: 50,
|
|
133
|
+
}
|
|
134
|
+
};
|
|
135
|
+
try {
|
|
136
|
+
const response = await adapter.call(request);
|
|
137
|
+
console.log('✓ Request completed');
|
|
138
|
+
console.log('');
|
|
139
|
+
console.log('Response content:');
|
|
140
|
+
console.log(response.content);
|
|
141
|
+
console.log('');
|
|
142
|
+
console.log('✅ Text Mode: PASSED');
|
|
143
|
+
console.log('');
|
|
144
|
+
return true;
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
console.log('❌ Text Mode: FAILED');
|
|
148
|
+
console.log('Error:', error.message);
|
|
149
|
+
console.log('');
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
async function runTests() {
|
|
154
|
+
const results = [];
|
|
155
|
+
results.push(await testJsonObjectModeBeta());
|
|
156
|
+
results.push(await testJsonSchemaModeBeta());
|
|
157
|
+
results.push(await testTextMode());
|
|
158
|
+
console.log('='.repeat(80));
|
|
159
|
+
console.log('RESULTS SUMMARY');
|
|
160
|
+
console.log('='.repeat(80));
|
|
161
|
+
console.log('');
|
|
162
|
+
const passed = results.filter(r => r).length;
|
|
163
|
+
const failed = results.filter(r => !r).length;
|
|
164
|
+
console.log(`Total Tests: ${results.length}`);
|
|
165
|
+
console.log(`Passed: ${passed}`);
|
|
166
|
+
console.log(`Failed: ${failed}`);
|
|
167
|
+
console.log('');
|
|
168
|
+
if (failed > 0) {
|
|
169
|
+
console.log('❌ Some tests failed');
|
|
170
|
+
process.exit(1);
|
|
171
|
+
}
|
|
172
|
+
else {
|
|
173
|
+
console.log('✅ All tests passed!');
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
runTests().catch(error => {
|
|
177
|
+
console.error('Test suite failed:', error);
|
|
178
|
+
process.exit(1);
|
|
179
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-json-response-format-google.d.ts","sourceRoot":"","sources":["../../../../src/services/providers/tests/test-json-response-format-google.ts"],"names":[],"mappings":";AACA;;;GAGG"}
|