@inference-gateway/sdk 0.7.3 → 0.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (8) hide show
  1. package/CHANGELOG.md +59 -0
  2. package/README.md +7 -5
  3. package/dist/src/client.js +7 -0
  4. package/dist/src/types/generated/index.d.ts +122 -66
  5. package/dist/src/types/generated/index.js +27 -9
  6. package/package.json +13 -14
  7. package/dist/tests/client.test.d.ts +0 -1
  8. package/dist/tests/client.test.js +0 -662
package/dist/tests/client.test.js DELETED
@@ -1,662 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- const client_1 = require("@/client");
4
- const generated_1 = require("@/types/generated");
5
- const web_1 = require("node:stream/web");
6
- const node_util_1 = require("node:util");
7
- describe('InferenceGatewayClient', () => {
8
- let client;
9
- const mockFetch = jest.fn();
10
- beforeEach(() => {
11
- client = new client_1.InferenceGatewayClient({
12
- baseURL: 'http://localhost:8080/v1',
13
- fetch: mockFetch,
14
- });
15
- });
16
- afterEach(() => {
17
- jest.clearAllMocks();
18
- });
19
- describe('listModels', () => {
20
- it('should fetch available models', async () => {
21
- const mockResponse = {
22
- object: 'list',
23
- data: [
24
- {
25
- id: 'gpt-4o',
26
- object: 'model',
27
- created: 1686935002,
28
- owned_by: 'openai',
29
- served_by: generated_1.Provider.openai,
30
- },
31
- {
32
- id: 'llama-3.3-70b-versatile',
33
- object: 'model',
34
- created: 1723651281,
35
- owned_by: 'groq',
36
- served_by: generated_1.Provider.groq,
37
- },
38
- ],
39
- };
40
- mockFetch.mockResolvedValueOnce({
41
- ok: true,
42
- json: () => Promise.resolve(mockResponse),
43
- });
44
- const result = await client.listModels();
45
- expect(result).toEqual(mockResponse);
46
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/models', expect.objectContaining({
47
- method: 'GET',
48
- headers: expect.any(Headers),
49
- }));
50
- });
51
- it('should fetch models for a specific provider', async () => {
52
- const mockResponse = {
53
- object: 'list',
54
- data: [
55
- {
56
- id: 'gpt-4o',
57
- object: 'model',
58
- created: 1686935002,
59
- owned_by: 'openai',
60
- served_by: generated_1.Provider.openai,
61
- },
62
- ],
63
- };
64
- mockFetch.mockResolvedValueOnce({
65
- ok: true,
66
- json: () => Promise.resolve(mockResponse),
67
- });
68
- const result = await client.listModels(generated_1.Provider.openai);
69
- expect(result).toEqual(mockResponse);
70
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/models?provider=openai', expect.objectContaining({
71
- method: 'GET',
72
- headers: expect.any(Headers),
73
- }));
74
- });
75
- it('should throw error when request fails', async () => {
76
- const errorMessage = 'Provider not found';
77
- mockFetch.mockResolvedValueOnce({
78
- ok: false,
79
- status: 404,
80
- json: () => Promise.resolve({ error: errorMessage }),
81
- });
82
- await expect(client.listModels(generated_1.Provider.openai)).rejects.toThrow(errorMessage);
83
- });
84
- });
85
- describe('listTools', () => {
86
- it('should fetch available MCP tools', async () => {
87
- const mockResponse = {
88
- object: 'list',
89
- data: [
90
- {
91
- name: 'read_file',
92
- description: 'Read content from a file',
93
- server: 'http://mcp-filesystem-server:8083/mcp',
94
- },
95
- {
96
- name: 'write_file',
97
- description: 'Write content to a file',
98
- server: 'http://mcp-filesystem-server:8083/mcp',
99
- },
100
- ],
101
- };
102
- mockFetch.mockResolvedValueOnce({
103
- ok: true,
104
- json: () => Promise.resolve(mockResponse),
105
- });
106
- const result = await client.listTools();
107
- expect(result).toEqual(mockResponse);
108
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/mcp/tools', expect.objectContaining({
109
- method: 'GET',
110
- headers: expect.any(Headers),
111
- }));
112
- });
113
- it('should throw error when MCP is not exposed', async () => {
114
- const errorMessage = 'MCP not exposed';
115
- mockFetch.mockResolvedValueOnce({
116
- ok: false,
117
- status: 403,
118
- json: () => Promise.resolve({ error: errorMessage }),
119
- });
120
- await expect(client.listTools()).rejects.toThrow(errorMessage);
121
- });
122
- it('should throw error when unauthorized', async () => {
123
- const errorMessage = 'Unauthorized';
124
- mockFetch.mockResolvedValueOnce({
125
- ok: false,
126
- status: 401,
127
- json: () => Promise.resolve({ error: errorMessage }),
128
- });
129
- await expect(client.listTools()).rejects.toThrow(errorMessage);
130
- });
131
- });
132
- describe('createChatCompletion', () => {
133
- it('should create a chat completion', async () => {
134
- const mockRequest = {
135
- model: 'gpt-4o',
136
- messages: [
137
- { role: generated_1.MessageRole.system, content: 'You are a helpful assistant' },
138
- { role: generated_1.MessageRole.user, content: 'Hello' },
139
- ],
140
- };
141
- const mockResponse = {
142
- id: 'chatcmpl-123',
143
- object: 'chat.completion',
144
- created: 1677652288,
145
- model: 'gpt-4o',
146
- choices: [
147
- {
148
- index: 0,
149
- message: {
150
- role: generated_1.MessageRole.assistant,
151
- content: 'Hello! How can I help you today?',
152
- },
153
- finish_reason: generated_1.ChatCompletionChoiceFinish_reason.stop,
154
- },
155
- ],
156
- usage: {
157
- prompt_tokens: 10,
158
- completion_tokens: 8,
159
- total_tokens: 18,
160
- },
161
- };
162
- mockFetch.mockResolvedValueOnce({
163
- ok: true,
164
- json: () => Promise.resolve(mockResponse),
165
- });
166
- const result = await client.createChatCompletion(mockRequest);
167
- expect(result).toEqual(mockResponse);
168
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/chat/completions', expect.objectContaining({
169
- method: 'POST',
170
- body: JSON.stringify({ ...mockRequest, stream: false }),
171
- }));
172
- });
173
- it('should create a chat completion with a specific provider', async () => {
174
- const mockRequest = {
175
- model: 'claude-3-opus-20240229',
176
- messages: [{ role: generated_1.MessageRole.user, content: 'Hello' }],
177
- };
178
- const mockResponse = {
179
- id: 'chatcmpl-456',
180
- object: 'chat.completion',
181
- created: 1677652288,
182
- model: 'claude-3-opus-20240229',
183
- choices: [
184
- {
185
- index: 0,
186
- message: {
187
- role: generated_1.MessageRole.assistant,
188
- content: 'Hello! How can I assist you today?',
189
- },
190
- finish_reason: generated_1.ChatCompletionChoiceFinish_reason.stop,
191
- },
192
- ],
193
- usage: {
194
- prompt_tokens: 5,
195
- completion_tokens: 8,
196
- total_tokens: 13,
197
- },
198
- };
199
- mockFetch.mockResolvedValueOnce({
200
- ok: true,
201
- json: () => Promise.resolve(mockResponse),
202
- });
203
- const result = await client.createChatCompletion(mockRequest, generated_1.Provider.anthropic);
204
- expect(result).toEqual(mockResponse);
205
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/chat/completions?provider=anthropic', expect.objectContaining({
206
- method: 'POST',
207
- body: JSON.stringify({ ...mockRequest, stream: false }),
208
- }));
209
- });
210
- });
211
- describe('streamChatCompletion', () => {
212
- it('should handle streaming chat completions', async () => {
213
- const mockRequest = {
214
- model: 'gpt-4o',
215
- messages: [{ role: generated_1.MessageRole.user, content: 'Hello' }],
216
- };
217
- const mockStream = new web_1.TransformStream();
218
- const writer = mockStream.writable.getWriter();
219
- const encoder = new node_util_1.TextEncoder();
220
- mockFetch.mockResolvedValueOnce({
221
- ok: true,
222
- body: mockStream.readable,
223
- });
224
- const callbacks = {
225
- onOpen: jest.fn(),
226
- onChunk: jest.fn(),
227
- onContent: jest.fn(),
228
- onFinish: jest.fn(),
229
- onError: jest.fn(),
230
- };
231
- const streamPromise = client.streamChatCompletion(mockRequest, callbacks);
232
- await writer.write(encoder.encode('data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n' +
233
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}\n\n' +
234
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}]}\n\n' +
235
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n' +
236
- 'data: [DONE]\n\n'));
237
- await writer.close();
238
- await streamPromise;
239
- expect(callbacks.onOpen).toHaveBeenCalledTimes(1);
240
- expect(callbacks.onChunk).toHaveBeenCalledTimes(4);
241
- expect(callbacks.onContent).toHaveBeenCalledWith('Hello');
242
- expect(callbacks.onContent).toHaveBeenCalledWith('!');
243
- expect(callbacks.onFinish).toHaveBeenCalledTimes(1);
244
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/chat/completions', expect.objectContaining({
245
- method: 'POST',
246
- body: JSON.stringify({
247
- ...mockRequest,
248
- stream: true,
249
- stream_options: {
250
- include_usage: true,
251
- },
252
- }),
253
- }));
254
- });
255
- it('should handle streaming chat completions reasoning and content', async () => {
256
- const mockRequest = {
257
- model: 'gpt-4o',
258
- messages: [{ role: generated_1.MessageRole.user, content: 'Hello' }],
259
- };
260
- const mockStream = new web_1.TransformStream();
261
- const writer = mockStream.writable.getWriter();
262
- const encoder = new node_util_1.TextEncoder();
263
- mockFetch.mockResolvedValueOnce({
264
- ok: true,
265
- body: mockStream.readable,
266
- });
267
- const callbacks = {
268
- onOpen: jest.fn(),
269
- onChunk: jest.fn(),
270
- onReasoning: jest.fn(),
271
- onContent: jest.fn(),
272
- onFinish: jest.fn(),
273
- };
274
- const streamPromise = client.streamChatCompletion(mockRequest, callbacks);
275
- await writer.write(encoder.encode('data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n' +
276
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"content":"","reasoning_content":"This"},"finish_reason":null}]}\n\n' +
277
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"content":"","reasoning_content":" is"},"finish_reason":null}]}\n\n' +
278
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"content":"","reasoning_content":" a"},"finish_reason":"stop"}]}\n\n' +
279
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"content":"","reasoning_content":" reasoning"},"finish_reason":"stop"}]}\n\n' +
280
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"content":"","reasoning_content":" content"},"finish_reason":"stop"}]}\n\n' +
281
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}\n\n' +
282
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}]}\n\n' +
283
- 'data: [DONE]\n\n'));
284
- await writer.close();
285
- await streamPromise;
286
- expect(callbacks.onOpen).toHaveBeenCalledTimes(1);
287
- expect(callbacks.onChunk).toHaveBeenCalledTimes(8);
288
- expect(callbacks.onReasoning).toHaveBeenCalledTimes(5);
289
- expect(callbacks.onReasoning).toHaveBeenCalledWith('This');
290
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' is');
291
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' a');
292
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' reasoning');
293
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' content');
294
- expect(callbacks.onContent).toHaveBeenCalledTimes(2);
295
- expect(callbacks.onContent).toHaveBeenCalledWith('Hello');
296
- expect(callbacks.onContent).toHaveBeenCalledWith('!');
297
- expect(callbacks.onFinish).toHaveBeenCalledTimes(1);
298
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/chat/completions', expect.objectContaining({
299
- method: 'POST',
300
- body: JSON.stringify({
301
- ...mockRequest,
302
- stream: true,
303
- stream_options: {
304
- include_usage: true,
305
- },
306
- }),
307
- }));
308
- });
309
- it('should handle tool calls in streaming chat completions', async () => {
310
- const mockRequest = {
311
- model: 'gpt-4o',
312
- messages: [
313
- {
314
- role: generated_1.MessageRole.user,
315
- content: 'What is the weather in San Francisco?',
316
- },
317
- ],
318
- tools: [
319
- {
320
- type: generated_1.ChatCompletionToolType.function,
321
- function: {
322
- name: 'get_weather',
323
- strict: true,
324
- },
325
- },
326
- ],
327
- };
328
- const mockStream = new web_1.TransformStream();
329
- const writer = mockStream.writable.getWriter();
330
- const encoder = new node_util_1.TextEncoder();
331
- mockFetch.mockResolvedValueOnce({
332
- ok: true,
333
- body: mockStream.readable,
334
- });
335
- const callbacks = {
336
- onOpen: jest.fn(),
337
- onChunk: jest.fn(),
338
- onTool: jest.fn(),
339
- onFinish: jest.fn(),
340
- };
341
- const streamPromise = client.streamChatCompletion(mockRequest, callbacks);
342
- // Simulate SSE events with tool calls
343
- await writer.write(encoder.encode('data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n' +
344
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"call_123","type":"function","function":{"name":"get_weather"}}]},"finish_reason":null}]}\n\n' +
345
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\\"location\\""}}]},"finish_reason":null}]}\n\n' +
346
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":":\\"San Francisco, CA\\""}}]},"finish_reason":null}]}\n\n' +
347
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]},"finish_reason":null}]}\n\n' +
348
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{},"finish_reason":"tool_calls"}]}\n\n' +
349
- 'data: [DONE]\n\n'));
350
- await writer.close();
351
- await streamPromise;
352
- expect(callbacks.onOpen).toHaveBeenCalledTimes(1);
353
- expect(callbacks.onChunk).toHaveBeenCalledTimes(6);
354
- expect(callbacks.onTool).toHaveBeenCalledTimes(1);
355
- expect(callbacks.onTool).toHaveBeenCalledWith({
356
- id: 'call_123',
357
- type: 'function',
358
- function: {
359
- name: 'get_weather',
360
- arguments: '{"location":"San Francisco, CA"}',
361
- },
362
- });
363
- expect(callbacks.onFinish).toHaveBeenCalledTimes(1);
364
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/chat/completions', expect.objectContaining({
365
- method: 'POST',
366
- body: JSON.stringify({
367
- ...mockRequest,
368
- stream: true,
369
- stream_options: {
370
- include_usage: true,
371
- },
372
- }),
373
- }));
374
- });
375
- it('should handle errors in streaming chat completions', async () => {
376
- const mockRequest = {
377
- model: 'gpt-4o',
378
- messages: [{ role: generated_1.MessageRole.user, content: 'Hello' }],
379
- };
380
- mockFetch.mockResolvedValueOnce({
381
- ok: false,
382
- status: 400,
383
- json: () => Promise.resolve({ error: 'Bad Request' }),
384
- });
385
- const callbacks = {
386
- onError: jest.fn(),
387
- };
388
- await expect(client.streamChatCompletion(mockRequest, callbacks)).rejects.toThrow('Bad Request');
389
- expect(callbacks.onError).toHaveBeenCalledTimes(1);
390
- });
391
- it('should handle streaming chat completions with usage metrics', async () => {
392
- const mockRequest = {
393
- model: 'gpt-4o',
394
- messages: [{ role: generated_1.MessageRole.user, content: 'Hello' }],
395
- };
396
- const mockStream = new web_1.TransformStream();
397
- const writer = mockStream.writable.getWriter();
398
- const encoder = new node_util_1.TextEncoder();
399
- mockFetch.mockResolvedValueOnce({
400
- ok: true,
401
- body: mockStream.readable,
402
- });
403
- const callbacks = {
404
- onOpen: jest.fn(),
405
- onChunk: jest.fn(),
406
- onContent: jest.fn(),
407
- onUsageMetrics: jest.fn(),
408
- onFinish: jest.fn(),
409
- onError: jest.fn(),
410
- };
411
- const streamPromise = client.streamChatCompletion(mockRequest, callbacks);
412
- await writer.write(encoder.encode('data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n' +
413
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}\n\n' +
414
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}]}\n\n' +
415
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n' +
416
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"gpt-4o","choices":[],"usage":{"prompt_tokens":10,"completion_tokens":8,"total_tokens":18}}\n\n' +
417
- 'data: [DONE]\n\n'));
418
- await writer.close();
419
- await streamPromise;
420
- expect(callbacks.onOpen).toHaveBeenCalledTimes(1);
421
- expect(callbacks.onChunk).toHaveBeenCalledTimes(5);
422
- expect(callbacks.onContent).toHaveBeenCalledWith('Hello');
423
- expect(callbacks.onContent).toHaveBeenCalledWith('!');
424
- expect(callbacks.onUsageMetrics).toHaveBeenCalledTimes(1);
425
- expect(callbacks.onUsageMetrics).toHaveBeenCalledWith({
426
- prompt_tokens: 10,
427
- completion_tokens: 8,
428
- total_tokens: 18,
429
- });
430
- expect(callbacks.onFinish).toHaveBeenCalledTimes(1);
431
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/chat/completions', expect.objectContaining({
432
- method: 'POST',
433
- body: JSON.stringify({
434
- ...mockRequest,
435
- stream: true,
436
- stream_options: {
437
- include_usage: true,
438
- },
439
- }),
440
- }));
441
- });
442
- it('should handle streaming chat completions with reasoning field', async () => {
443
- const mockRequest = {
444
- model: 'groq/deepseek-distilled-llama-3.1-70b',
445
- messages: [{ role: generated_1.MessageRole.user, content: 'Hello' }],
446
- };
447
- const mockStream = new web_1.TransformStream();
448
- const writer = mockStream.writable.getWriter();
449
- const encoder = new node_util_1.TextEncoder();
450
- mockFetch.mockResolvedValueOnce({
451
- ok: true,
452
- body: mockStream.readable,
453
- });
454
- const callbacks = {
455
- onOpen: jest.fn(),
456
- onChunk: jest.fn(),
457
- onReasoning: jest.fn(),
458
- onContent: jest.fn(),
459
- onFinish: jest.fn(),
460
- };
461
- const streamPromise = client.streamChatCompletion(mockRequest, callbacks);
462
- await writer.write(encoder.encode('data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"groq/deepseek-distilled-llama-3.1-70b","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n' +
463
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"groq/deepseek-distilled-llama-3.1-70b","choices":[{"index":0,"delta":{"content":"","reasoning":"Let me"},"finish_reason":null}]}\n\n' +
464
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"groq/deepseek-distilled-llama-3.1-70b","choices":[{"index":0,"delta":{"content":"","reasoning":" think"},"finish_reason":null}]}\n\n' +
465
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"groq/deepseek-distilled-llama-3.1-70b","choices":[{"index":0,"delta":{"content":"","reasoning":" about"},"finish_reason":"stop"}]}\n\n' +
466
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"groq/deepseek-distilled-llama-3.1-70b","choices":[{"index":0,"delta":{"content":"","reasoning":" this"},"finish_reason":"stop"}]}\n\n' +
467
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"groq/deepseek-distilled-llama-3.1-70b","choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}\n\n' +
468
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"groq/deepseek-distilled-llama-3.1-70b","choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}]}\n\n' +
469
- 'data: [DONE]\n\n'));
470
- await writer.close();
471
- await streamPromise;
472
- expect(callbacks.onOpen).toHaveBeenCalledTimes(1);
473
- expect(callbacks.onChunk).toHaveBeenCalledTimes(7);
474
- expect(callbacks.onReasoning).toHaveBeenCalledTimes(4);
475
- expect(callbacks.onReasoning).toHaveBeenCalledWith('Let me');
476
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' think');
477
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' about');
478
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' this');
479
- expect(callbacks.onContent).toHaveBeenCalledTimes(2);
480
- expect(callbacks.onContent).toHaveBeenCalledWith('Hello');
481
- expect(callbacks.onContent).toHaveBeenCalledWith('!');
482
- expect(callbacks.onFinish).toHaveBeenCalledTimes(1);
483
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/chat/completions', expect.objectContaining({
484
- method: 'POST',
485
- body: JSON.stringify({
486
- ...mockRequest,
487
- stream: true,
488
- stream_options: {
489
- include_usage: true,
490
- },
491
- }),
492
- }));
493
- });
494
- it('should handle streaming chat completions with reasoning_content (DeepSeek)', async () => {
495
- const mockRequest = {
496
- model: 'deepseek/deepseek-reasoner',
497
- messages: [{ role: generated_1.MessageRole.user, content: 'Hello' }],
498
- };
499
- const mockStream = new web_1.TransformStream();
500
- const writer = mockStream.writable.getWriter();
501
- const encoder = new node_util_1.TextEncoder();
502
- mockFetch.mockResolvedValueOnce({
503
- ok: true,
504
- body: mockStream.readable,
505
- });
506
- const callbacks = {
507
- onOpen: jest.fn(),
508
- onChunk: jest.fn(),
509
- onReasoning: jest.fn(),
510
- onContent: jest.fn(),
511
- onFinish: jest.fn(),
512
- };
513
- const streamPromise = client.streamChatCompletion(mockRequest, callbacks);
514
- await writer.write(encoder.encode('data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"deepseek/deepseek-reasoner","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n' +
515
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"deepseek/deepseek-reasoner","choices":[{"index":0,"delta":{"content":"","reasoning_content":"This"},"finish_reason":null}]}\n\n' +
516
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"deepseek/deepseek-reasoner","choices":[{"index":0,"delta":{"content":"","reasoning_content":" is"},"finish_reason":null}]}\n\n' +
517
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"deepseek/deepseek-reasoner","choices":[{"index":0,"delta":{"content":"","reasoning_content":" a"},"finish_reason":"stop"}]}\n\n' +
518
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"deepseek/deepseek-reasoner","choices":[{"index":0,"delta":{"content":"","reasoning_content":" reasoning"},"finish_reason":"stop"}]}\n\n' +
519
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"deepseek/deepseek-reasoner","choices":[{"index":0,"delta":{"content":"","reasoning_content":" content"},"finish_reason":"stop"}]}\n\n' +
520
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"deepseek/deepseek-reasoner","choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}\n\n' +
521
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"deepseek/deepseek-reasoner","choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}]}\n\n' +
522
- 'data: [DONE]\n\n'));
523
- await writer.close();
524
- await streamPromise;
525
- expect(callbacks.onOpen).toHaveBeenCalledTimes(1);
526
- expect(callbacks.onChunk).toHaveBeenCalledTimes(8);
527
- expect(callbacks.onReasoning).toHaveBeenCalledTimes(5);
528
- expect(callbacks.onReasoning).toHaveBeenCalledWith('This');
529
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' is');
530
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' a');
531
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' reasoning');
532
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' content');
533
- expect(callbacks.onContent).toHaveBeenCalledTimes(2);
534
- expect(callbacks.onContent).toHaveBeenCalledWith('Hello');
535
- expect(callbacks.onContent).toHaveBeenCalledWith('!');
536
- expect(callbacks.onFinish).toHaveBeenCalledTimes(1);
537
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/chat/completions', expect.objectContaining({
538
- method: 'POST',
539
- body: JSON.stringify({
540
- ...mockRequest,
541
- stream: true,
542
- stream_options: {
543
- include_usage: true,
544
- },
545
- }),
546
- }));
547
- });
548
- it('should handle streaming chat completions with reasoning field (Groq)', async () => {
549
- const mockRequest = {
550
- model: 'llama-3.1-70b-versatile',
551
- messages: [{ role: generated_1.MessageRole.user, content: 'Hello' }],
552
- };
553
- const mockStream = new web_1.TransformStream();
554
- const writer = mockStream.writable.getWriter();
555
- const encoder = new node_util_1.TextEncoder();
556
- mockFetch.mockResolvedValueOnce({
557
- ok: true,
558
- body: mockStream.readable,
559
- });
560
- const callbacks = {
561
- onOpen: jest.fn(),
562
- onChunk: jest.fn(),
563
- onReasoning: jest.fn(),
564
- onContent: jest.fn(),
565
- onFinish: jest.fn(),
566
- };
567
- const streamPromise = client.streamChatCompletion(mockRequest, callbacks);
568
- await writer.write(encoder.encode('data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"llama-3.1-70b-versatile","choices":[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}\n\n' +
569
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"llama-3.1-70b-versatile","choices":[{"index":0,"delta":{"content":"","reasoning":"Let me"},"finish_reason":null}]}\n\n' +
570
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"llama-3.1-70b-versatile","choices":[{"index":0,"delta":{"content":"","reasoning":" think"},"finish_reason":null}]}\n\n' +
571
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"llama-3.1-70b-versatile","choices":[{"index":0,"delta":{"content":"","reasoning":" about"},"finish_reason":"stop"}]}\n\n' +
572
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"llama-3.1-70b-versatile","choices":[{"index":0,"delta":{"content":"","reasoning":" this"},"finish_reason":"stop"}]}\n\n' +
573
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"llama-3.1-70b-versatile","choices":[{"index":0,"delta":{"content":"Hello"},"finish_reason":null}]}\n\n' +
574
- 'data: {"id":"chatcmpl-123","object":"chat.completion.chunk","created":1677652288,"model":"llama-3.1-70b-versatile","choices":[{"index":0,"delta":{"content":"!"},"finish_reason":null}]}\n\n' +
575
- 'data: [DONE]\n\n'));
576
- await writer.close();
577
- await streamPromise;
578
- expect(callbacks.onOpen).toHaveBeenCalledTimes(1);
579
- expect(callbacks.onChunk).toHaveBeenCalledTimes(7);
580
- expect(callbacks.onReasoning).toHaveBeenCalledTimes(4);
581
- expect(callbacks.onReasoning).toHaveBeenCalledWith('Let me');
582
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' think');
583
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' about');
584
- expect(callbacks.onReasoning).toHaveBeenCalledWith(' this');
585
- expect(callbacks.onContent).toHaveBeenCalledTimes(2);
586
- expect(callbacks.onContent).toHaveBeenCalledWith('Hello');
587
- expect(callbacks.onContent).toHaveBeenCalledWith('!');
588
- expect(callbacks.onFinish).toHaveBeenCalledTimes(1);
589
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/chat/completions', expect.objectContaining({
590
- method: 'POST',
591
- body: JSON.stringify({
592
- ...mockRequest,
593
- stream: true,
594
- stream_options: {
595
- include_usage: true,
596
- },
597
- }),
598
- }));
599
- });
600
- });
601
- describe('proxy', () => {
602
- it('should proxy requests to a specific provider', async () => {
603
- const mockResponse = { result: 'success' };
604
- mockFetch.mockResolvedValueOnce({
605
- ok: true,
606
- json: () => Promise.resolve(mockResponse),
607
- });
608
- const result = await client.proxy(generated_1.Provider.openai, 'embeddings', {
609
- method: 'POST',
610
- body: JSON.stringify({
611
- model: 'text-embedding-ada-002',
612
- input: 'Hello world',
613
- }),
614
- });
615
- expect(result).toEqual(mockResponse);
616
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/proxy/openai/embeddings', expect.objectContaining({
617
- method: 'POST',
618
- body: JSON.stringify({
619
- model: 'text-embedding-ada-002',
620
- input: 'Hello world',
621
- }),
622
- }));
623
- });
624
- });
625
- describe('healthCheck', () => {
626
- it('should return true when API is healthy', async () => {
627
- mockFetch.mockResolvedValueOnce({
628
- ok: true,
629
- });
630
- const result = await client.healthCheck();
631
- expect(result).toBe(true);
632
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/health');
633
- });
634
- it('should return false when API is unhealthy', async () => {
635
- mockFetch.mockRejectedValueOnce(new Error('API error'));
636
- const result = await client.healthCheck();
637
- expect(result).toBe(false);
638
- });
639
- });
640
- describe('withOptions', () => {
641
- it('should create a new client with merged options', () => {
642
- const originalClient = new client_1.InferenceGatewayClient({
643
- baseURL: 'http://localhost:8080/v1',
644
- apiKey: 'test-key',
645
- fetch: mockFetch,
646
- });
647
- const newClient = originalClient.withOptions({
648
- defaultHeaders: { 'X-Custom-Header': 'value' },
649
- });
650
- expect(newClient).toBeInstanceOf(client_1.InferenceGatewayClient);
651
- expect(newClient).not.toBe(originalClient);
652
- mockFetch.mockResolvedValueOnce({
653
- ok: true,
654
- json: () => Promise.resolve({}),
655
- });
656
- newClient.listModels();
657
- expect(mockFetch).toHaveBeenCalledWith('http://localhost:8080/v1/models', expect.objectContaining({
658
- headers: expect.any(Headers),
659
- }));
660
- });
661
- });
662
- });