@inference-gateway/sdk 0.2.0 → 0.3.0

package/CHANGELOG.md

@@ -1,3 +1,14 @@
+## [0.3.0](https://github.com/inference-gateway/typescript-sdk/compare/v0.2.0...v0.3.0) (2025-02-02)
+
+### ✨ Features
+
+* add streaming content functionality to InferenceGatewayClient and update README ([ba41d2d](https://github.com/inference-gateway/typescript-sdk/commit/ba41d2dc136b83372820af2aefa63969932e16f0))
+
+### 📚 Documentation
+
+* **fix:** Update examples in README.md ([4e972fc](https://github.com/inference-gateway/typescript-sdk/commit/4e972fc2c577f41b0b443f1c87cde7561717b577))
+* Update OpenAPI spec - download it from Inference-gateway ([9816b15](https://github.com/inference-gateway/typescript-sdk/commit/9816b151db6b48b04723f93b988daf83239a09df))
+
 ## [0.2.0](https://github.com/inference-gateway/typescript-sdk/compare/v0.1.6...v0.2.0) (2025-01-28)
 
 ### ✨ Features

package/README.md

@@ -9,6 +9,7 @@ An SDK written in Typescript for the [Inference Gateway](https://github.com/eden
     - [Listing All Models](#listing-all-models)
     - [List Models by Provider](#list-models-by-provider)
     - [Generating Content](#generating-content)
+    - [Streaming Content](#streaming-content)
     - [Health Check](#health-check)
   - [Contributing](#contributing)
   - [License](#license)
@@ -37,7 +38,7 @@ async function main() {
     models.forEach((providerModels) => {
       console.log(`Provider: ${providerModels.provider}`);
       providerModels.models.forEach((model) => {
-        console.log(`Model: ${model.id}`);
+        console.log(`Model: ${model.name}`);
       });
     });
 
@@ -76,7 +77,7 @@ try {
   models.forEach((providerModels) => {
     console.log(`Provider: ${providerModels.provider}`);
     providerModels.models.forEach((model) => {
-      console.log(`Model: ${model.id}`);
+      console.log(`Model: ${model.name}`);
     });
   });
 } catch (error) {
@@ -136,6 +137,32 @@ const client = new InferenceGatewayClient('http://localhost:8080');
 }
 ```
 
+### Streaming Content
+
+To stream content using a model, use the `streamContent` method:
+
+```typescript
+const client = new InferenceGatewayClient('http://localhost:8080');
+
+await client.generateContentStream(
+  {
+    provider: Provider.Groq,
+    model: 'deepseek-r1-distill-llama-70b',
+    messages: [
+      {
+        role: MessageRole.User,
+        content: 'Tell me a story',
+      },
+    ],
+  },
+  {
+    onMessageStart: (role) => console.log('Message started:', role),
+    onContentDelta: (content) => process.stdout.write(content),
+    onStreamEnd: () => console.log('\nStream completed'),
+  }
+);
+```
+
 ### Health Check
 
 To check if the Inference Gateway is running, use the `healthCheck` method:

package/dist/src/client.d.ts

@@ -1,4 +1,4 @@
-import { GenerateContentRequest, GenerateContentResponse, Provider, ProviderModels } from './types';
+import { GenerateContentOptions, GenerateContentRequest, GenerateContentResponse, Provider, ProviderModels } from './types';
 export declare class InferenceGatewayClient {
     private baseUrl;
     private authToken?;
@@ -7,5 +7,6 @@ export declare class InferenceGatewayClient {
     listModels(): Promise<ProviderModels[]>;
     listModelsByProvider(provider: Provider): Promise<ProviderModels>;
     generateContent(params: GenerateContentRequest): Promise<GenerateContentResponse>;
+    generateContentStream(params: GenerateContentRequest, options?: GenerateContentOptions): Promise<void>;
     healthCheck(): Promise<boolean>;
 }

package/dist/src/client.js

@@ -41,6 +41,64 @@ class InferenceGatewayClient {
             }),
         });
     }
+    async generateContentStream(params, options) {
+        const response = await fetch(`${this.baseUrl}/llms/${params.provider}/generate`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                ...(this.authToken && { Authorization: `Bearer ${this.authToken}` }),
+            },
+            body: JSON.stringify({
+                model: params.model,
+                messages: params.messages,
+                stream: true,
+                ssevents: true,
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.json();
+            throw new Error(error.error || `HTTP error! status: ${response.status}`);
+        }
+        const reader = response.body?.getReader();
+        if (!reader)
+            throw new Error('Response body is not readable');
+        const decoder = new TextDecoder();
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            const events = decoder.decode(value).split('\n\n');
+            for (const event of events) {
+                if (!event.trim())
+                    continue;
+                const [eventType, ...data] = event.split('\n');
+                const eventData = JSON.parse(data.join('\n').replace('data: ', ''));
+                switch (eventType.replace('event: ', '')) {
+                    case 'message-start':
+                        options?.onMessageStart?.(eventData.role);
+                        break;
+                    case 'stream-start':
+                        options?.onStreamStart?.();
+                        break;
+                    case 'content-start':
+                        options?.onContentStart?.();
+                        break;
+                    case 'content-delta':
+                        options?.onContentDelta?.(eventData.content);
+                        break;
+                    case 'content-end':
+                        options?.onContentEnd?.();
+                        break;
+                    case 'message-end':
+                        options?.onMessageEnd?.();
+                        break;
+                    case 'stream-end':
+                        options?.onStreamEnd?.();
+                        break;
+                }
+            }
+        }
+    }
     async healthCheck() {
         try {
             await this.request('/health');

package/dist/src/types/index.d.ts

@@ -36,3 +36,12 @@ export interface GenerateContentResponse {
         content: string;
     };
 }
+export interface GenerateContentOptions {
+    onMessageStart?: (role: string) => void;
+    onStreamStart?: () => void;
+    onContentStart?: () => void;
+    onContentDelta?: (content: string) => void;
+    onContentEnd?: () => void;
+    onMessageEnd?: () => void;
+    onStreamEnd?: () => void;
+}

package/dist/tests/client.test.js

@@ -122,4 +122,109 @@ describe('InferenceGatewayClient', () => {
             await expect(client.listModels()).rejects.toThrow(errorMessage);
         });
     });
+    describe('generateContentStream', () => {
+        it('should handle SSE events correctly', async () => {
+            const mockRequest = {
+                provider: types_1.Provider.Ollama,
+                model: 'llama2',
+                messages: [
+                    { role: types_1.MessageRole.System, content: 'You are a helpful assistant' },
+                    { role: types_1.MessageRole.User, content: 'Hello' },
+                ],
+            };
+            const mockStream = new TransformStream();
+            const writer = mockStream.writable.getWriter();
+            const encoder = new TextEncoder();
+            global.fetch.mockResolvedValueOnce({
+                ok: true,
+                body: mockStream.readable,
+            });
+            const callbacks = {
+                onMessageStart: jest.fn(),
+                onStreamStart: jest.fn(),
+                onContentStart: jest.fn(),
+                onContentDelta: jest.fn(),
+                onContentEnd: jest.fn(),
+                onMessageEnd: jest.fn(),
+                onStreamEnd: jest.fn(),
+            };
+            const streamPromise = client.generateContentStream(mockRequest, callbacks);
+            await writer.write(encoder.encode('event: message-start\ndata: {"role": "assistant"}\n\n' +
+                'event: stream-start\ndata: {}\n\n' +
+                'event: content-start\ndata: {}\n\n' +
+                'event: content-delta\ndata: {"content": "Hello"}\n\n' +
+                'event: content-delta\ndata: {"content": " there!"}\n\n' +
+                'event: content-end\ndata: {}\n\n' +
+                'event: message-end\ndata: {}\n\n' +
+                'event: stream-end\ndata: {}\n\n'));
+            await writer.close();
+            await streamPromise;
+            expect(callbacks.onMessageStart).toHaveBeenCalledWith('assistant');
+            expect(callbacks.onStreamStart).toHaveBeenCalledTimes(1);
+            expect(callbacks.onContentStart).toHaveBeenCalledTimes(1);
+            expect(callbacks.onContentDelta).toHaveBeenCalledWith('Hello');
+            expect(callbacks.onContentDelta).toHaveBeenCalledWith(' there!');
+            expect(callbacks.onContentEnd).toHaveBeenCalledTimes(1);
+            expect(callbacks.onMessageEnd).toHaveBeenCalledTimes(1);
+            expect(callbacks.onStreamEnd).toHaveBeenCalledTimes(1);
+            expect(global.fetch).toHaveBeenCalledWith(`${mockBaseUrl}/llms/${mockRequest.provider}/generate`, expect.objectContaining({
+                method: 'POST',
+                body: JSON.stringify({
+                    model: mockRequest.model,
+                    messages: mockRequest.messages,
+                    stream: true,
+                    ssevents: true,
+                }),
+            }));
+        });
+        it('should handle errors in the stream response', async () => {
+            const mockRequest = {
+                provider: types_1.Provider.Ollama,
+                model: 'llama2',
+                messages: [{ role: types_1.MessageRole.User, content: 'Hello' }],
+            };
+            global.fetch.mockResolvedValueOnce({
+                ok: false,
+                status: 400,
+                json: () => Promise.resolve({ error: 'Bad Request' }),
+            });
+            await expect(client.generateContentStream(mockRequest, {})).rejects.toThrow('Bad Request');
+        });
+        it('should handle non-readable response body', async () => {
+            const mockRequest = {
+                provider: types_1.Provider.Ollama,
+                model: 'llama2',
+                messages: [{ role: types_1.MessageRole.User, content: 'Hello' }],
+            };
+            global.fetch.mockResolvedValueOnce({
+                ok: true,
+                body: null,
+            });
+            await expect(client.generateContentStream(mockRequest, {})).rejects.toThrow('Response body is not readable');
+        });
+        it('should handle empty events in the stream', async () => {
+            const mockRequest = {
+                provider: types_1.Provider.Ollama,
+                model: 'llama2',
+                messages: [{ role: types_1.MessageRole.User, content: 'Hello' }],
+            };
+            const mockStream = new TransformStream();
+            const writer = mockStream.writable.getWriter();
+            const encoder = new TextEncoder();
+            global.fetch.mockResolvedValueOnce({
+                ok: true,
+                body: mockStream.readable,
+            });
+            const callbacks = {
+                onContentDelta: jest.fn(),
+            };
+            const streamPromise = client.generateContentStream(mockRequest, callbacks);
+            await writer.write(encoder.encode('\n\n'));
+            await writer.write(encoder.encode('event: content-delta\ndata: {"content": "Hello"}\n\n'));
+            await writer.close();
+            await streamPromise;
+            expect(callbacks.onContentDelta).toHaveBeenCalledTimes(1);
+            expect(callbacks.onContentDelta).toHaveBeenCalledWith('Hello');
+        });
+    });
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@inference-gateway/sdk",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "An SDK written in Typescript for the [Inference Gateway](https://github.com/inference-gateway/inference-gateway).",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",