@inference-gateway/sdk 0.1.6 → 0.3.0

package/CHANGELOG.md

@@ -1,3 +1,26 @@
+## [0.3.0](https://github.com/inference-gateway/typescript-sdk/compare/v0.2.0...v0.3.0) (2025-02-02)
+
+### ✨ Features
+
+* add streaming content functionality to InferenceGatewayClient and update README ([ba41d2d](https://github.com/inference-gateway/typescript-sdk/commit/ba41d2dc136b83372820af2aefa63969932e16f0))
+
+### 📚 Documentation
+
+* **fix:** Update examples in README.md ([4e972fc](https://github.com/inference-gateway/typescript-sdk/commit/4e972fc2c577f41b0b443f1c87cde7561717b577))
+* Update OpenAPI spec - download it from Inference-gateway ([9816b15](https://github.com/inference-gateway/typescript-sdk/commit/9816b151db6b48b04723f93b988daf83239a09df))
+
+## [0.2.0](https://github.com/inference-gateway/typescript-sdk/compare/v0.1.6...v0.2.0) (2025-01-28)
+
+### ✨ Features
+
+* add listModelsByProvider method and update README with new model listing features ([a8d7cd9](https://github.com/inference-gateway/typescript-sdk/commit/a8d7cd9e9332f6455271f4d8f2832631b46d2c3d))
+
+### 📚 Documentation
+
+* add Contributing section to README with reference to CONTRIBUTING.md ([322baae](https://github.com/inference-gateway/typescript-sdk/commit/322baae9110f270615597e647835ed22e4fdbc65))
+* add CONTRIBUTING.md with guidelines for contributing to the project ([d36b08f](https://github.com/inference-gateway/typescript-sdk/commit/d36b08f1647500795d279dcd5612d5a81c9c4a74))
+* **openapi:** Download the latest openapi spec from inference-gateway ([733ee1e](https://github.com/inference-gateway/typescript-sdk/commit/733ee1e57d9fc6669bb2ec0197db1c2c772a0283))
+
 ## [0.1.6](https://github.com/inference-gateway/typescript-sdk/compare/v0.1.5...v0.1.6) (2025-01-23)
 
 ### 🐛 Bug Fixes

package/README.md

@@ -6,9 +6,12 @@ An SDK written in Typescript for the [Inference Gateway](https://github.com/eden
   - [Installation](#installation)
   - [Usage](#usage)
     - [Creating a Client](#creating-a-client)
-    - [Listing Models](#listing-models)
+    - [Listing All Models](#listing-all-models)
+    - [List Models by Provider](#list-models-by-provider)
     - [Generating Content](#generating-content)
+    - [Streaming Content](#streaming-content)
     - [Health Check](#health-check)
+  - [Contributing](#contributing)
   - [License](#license)
 
 ## Installation
@@ -35,20 +38,21 @@ async function main() {
     models.forEach((providerModels) => {
       console.log(`Provider: ${providerModels.provider}`);
       providerModels.models.forEach((model) => {
-        console.log(`Model: ${model.id}`);
+        console.log(`Model: ${model.name}`);
       });
     });
 
+    // Generate content
     const response = await client.generateContent({
       provider: Provider.Ollama,
       model: 'llama2',
       messages: [
         {
-          role: 'system',
+          role: MessageRole.System,
           content: 'You are a helpful llama',
         },
         {
-          role: 'user',
+          role: MessageRole.User,
           content: 'Tell me a joke',
         },
       ],
@@ -63,9 +67,9 @@ async function main() {
 main();
 ```
 
-### Listing Models
+### Listing All Models
 
-To list available models, use the `listModels` method:
+To list all available models from all providers, use the `listModels` method:
 
 ```typescript
 try {
@@ -73,7 +77,7 @@ try {
   models.forEach((providerModels) => {
     console.log(`Provider: ${providerModels.provider}`);
     providerModels.models.forEach((model) => {
-      console.log(`Model: ${model.id}`);
+      console.log(`Model: ${model.name}`);
     });
   });
 } catch (error) {
@@ -81,22 +85,46 @@ try {
 }
 ```
 
+### List Models by Provider
+
+To list all available models from a specific provider, use the `listModelsByProvider` method:
+
+```typescript
+try {
+  const providerModels = await client.listModelsByProvider(Provider.OpenAI);
+  console.log(`Provider: ${providerModels.provider}`);
+  providerModels.models.forEach((model) => {
+    console.log(`Model: ${model.name}`);
+  });
+} catch (error) {
+  console.error('Error:', error);
+}
+```
+
 ### Generating Content
 
 To generate content using a model, use the `generateContent` method:
 
 ```typescript
-try {
+import {
+  InferenceGatewayClient,
+  Message,
+  MessageRole,
+  Provider,
+} from '@inference-gateway/sdk';
+
+const client = new InferenceGatewayClient('http://localhost:8080');
+
   const response = await client.generateContent({
     provider: Provider.Ollama,
     model: 'llama2',
     messages: [
       {
-        role: 'system',
+        role: MessageRole.System,
         content: 'You are a helpful llama',
       },
       {
-        role: 'user',
+        role: MessageRole.User,
         content: 'Tell me a joke',
       },
     ],
@@ -109,6 +137,32 @@ try {
 }
 ```
 
+### Streaming Content
+
+To stream content using a model, use the `streamContent` method:
+
+```typescript
+const client = new InferenceGatewayClient('http://localhost:8080');
+
+await client.generateContentStream(
+  {
+    provider: Provider.Groq,
+    model: 'deepseek-r1-distill-llama-70b',
+    messages: [
+      {
+        role: MessageRole.User,
+        content: 'Tell me a story',
+      },
+    ],
+  },
+  {
+    onMessageStart: (role) => console.log('Message started:', role),
+    onContentDelta: (content) => process.stdout.write(content),
+    onStreamEnd: () => console.log('\nStream completed'),
+  }
+);
+```
+
 ### Health Check
 
 To check if the Inference Gateway is running, use the `healthCheck` method:
@@ -122,6 +176,10 @@ try {
 }
 ```
 
+## Contributing
+
+Please refer to the [CONTRIBUTING.md](CONTRIBUTING.md) file for information about how to get involved. We welcome issues, questions, and pull requests.
+
 ## License
 
 This SDK is distributed under the MIT License, see [LICENSE](LICENSE) for more information.

package/dist/src/client.d.ts

@@ -1,10 +1,12 @@
-import { GenerateContentRequest, GenerateContentResponse, ProviderModels } from './types';
+import { GenerateContentOptions, GenerateContentRequest, GenerateContentResponse, Provider, ProviderModels } from './types';
 export declare class InferenceGatewayClient {
     private baseUrl;
     private authToken?;
     constructor(baseUrl: string, authToken?: string);
     private request;
     listModels(): Promise<ProviderModels[]>;
+    listModelsByProvider(provider: Provider): Promise<ProviderModels>;
     generateContent(params: GenerateContentRequest): Promise<GenerateContentResponse>;
+    generateContentStream(params: GenerateContentRequest, options?: GenerateContentOptions): Promise<void>;
     healthCheck(): Promise<boolean>;
 }

package/dist/src/client.js

@@ -29,6 +29,9 @@ class InferenceGatewayClient {
     async listModels() {
         return this.request('/llms');
     }
+    async listModelsByProvider(provider) {
+        return this.request(`/llms/${provider}`);
+    }
     async generateContent(params) {
         return this.request(`/llms/${params.provider}/generate`, {
             method: 'POST',
@@ -38,6 +41,64 @@ class InferenceGatewayClient {
             }),
         });
     }
+    async generateContentStream(params, options) {
+        const response = await fetch(`${this.baseUrl}/llms/${params.provider}/generate`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                ...(this.authToken && { Authorization: `Bearer ${this.authToken}` }),
+            },
+            body: JSON.stringify({
+                model: params.model,
+                messages: params.messages,
+                stream: true,
+                ssevents: true,
+            }),
+        });
+        if (!response.ok) {
+            const error = await response.json();
+            throw new Error(error.error || `HTTP error! status: ${response.status}`);
+        }
+        const reader = response.body?.getReader();
+        if (!reader)
+            throw new Error('Response body is not readable');
+        const decoder = new TextDecoder();
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            const events = decoder.decode(value).split('\n\n');
+            for (const event of events) {
+                if (!event.trim())
+                    continue;
+                const [eventType, ...data] = event.split('\n');
+                const eventData = JSON.parse(data.join('\n').replace('data: ', ''));
+                switch (eventType.replace('event: ', '')) {
+                    case 'message-start':
+                        options?.onMessageStart?.(eventData.role);
+                        break;
+                    case 'stream-start':
+                        options?.onStreamStart?.();
+                        break;
+                    case 'content-start':
+                        options?.onContentStart?.();
+                        break;
+                    case 'content-delta':
+                        options?.onContentDelta?.(eventData.content);
+                        break;
+                    case 'content-end':
+                        options?.onContentEnd?.();
+                        break;
+                    case 'message-end':
+                        options?.onMessageEnd?.();
+                        break;
+                    case 'stream-end':
+                        options?.onStreamEnd?.();
+                        break;
+                }
+            }
+        }
+    }
     async healthCheck() {
         try {
             await this.request('/health');

package/dist/src/types/index.d.ts

@@ -7,15 +7,17 @@ export declare enum Provider {
     Cohere = "cohere",
     Anthropic = "anthropic"
 }
+export declare enum MessageRole {
+    System = "system",
+    User = "user",
+    Assistant = "assistant"
+}
 export interface Message {
-    role: 'system' | 'user' | 'assistant';
+    role: MessageRole;
     content: string;
 }
 export interface Model {
-    id: string;
-    object: string;
-    owned_by: string;
-    created: number;
+    name: string;
 }
 export interface ProviderModels {
     provider: Provider;
@@ -34,3 +36,12 @@ export interface GenerateContentResponse {
         content: string;
     };
 }
+export interface GenerateContentOptions {
+    onMessageStart?: (role: string) => void;
+    onStreamStart?: () => void;
+    onContentStart?: () => void;
+    onContentDelta?: (content: string) => void;
+    onContentEnd?: () => void;
+    onMessageEnd?: () => void;
+    onStreamEnd?: () => void;
+}

package/dist/src/types/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.Provider = void 0;
+exports.MessageRole = exports.Provider = void 0;
 var Provider;
 (function (Provider) {
     Provider["Ollama"] = "ollama";
@@ -11,3 +11,9 @@ var Provider;
     Provider["Cohere"] = "cohere";
     Provider["Anthropic"] = "anthropic";
 })(Provider || (exports.Provider = Provider = {}));
+var MessageRole;
+(function (MessageRole) {
+    MessageRole["System"] = "system";
+    MessageRole["User"] = "user";
+    MessageRole["Assistant"] = "assistant";
+})(MessageRole || (exports.MessageRole = MessageRole = {}));

package/dist/tests/client.test.js

@@ -16,10 +16,7 @@ describe('InferenceGatewayClient', () => {
                     provider: types_1.Provider.Ollama,
                     models: [
                         {
-                            id: 'llama2',
-                            object: 'model',
-                            owned_by: 'ollama',
-                            created: 1234567890,
+                            name: 'llama2',
                         },
                     ],
                 },
@@ -35,20 +32,50 @@ describe('InferenceGatewayClient', () => {
             }));
         });
     });
+    describe('listModelsByProvider', () => {
+        it('should fetch models for a specific provider', async () => {
+            const mockResponse = {
+                provider: types_1.Provider.OpenAI,
+                models: [
+                    {
+                        name: 'gpt-4',
+                    },
+                ],
+            };
+            global.fetch.mockResolvedValueOnce({
+                ok: true,
+                json: () => Promise.resolve(mockResponse),
+            });
+            const result = await client.listModelsByProvider(types_1.Provider.OpenAI);
+            expect(result).toEqual(mockResponse);
+            expect(global.fetch).toHaveBeenCalledWith(`${mockBaseUrl}/llms/${types_1.Provider.OpenAI}`, expect.objectContaining({
+                headers: expect.any(Headers),
+            }));
+        });
+        it('should throw error when provider request fails', async () => {
+            const errorMessage = 'Provider not found';
+            global.fetch.mockResolvedValueOnce({
+                ok: false,
+                status: 404,
+                json: () => Promise.resolve({ error: errorMessage }),
+            });
+            await expect(client.listModelsByProvider(types_1.Provider.OpenAI)).rejects.toThrow(errorMessage);
+        });
+    });
     describe('generateContent', () => {
         it('should generate content with the specified provider', async () => {
             const mockRequest = {
                 provider: types_1.Provider.Ollama,
                 model: 'llama2',
                 messages: [
-                    { role: 'system', content: 'You are a helpful assistant' },
-                    { role: 'user', content: 'Hello' },
+                    { role: types_1.MessageRole.System, content: 'You are a helpful assistant' },
+                    { role: types_1.MessageRole.User, content: 'Hello' },
                 ],
             };
             const mockResponse = {
                 provider: types_1.Provider.Ollama,
                 response: {
-                    role: 'assistant',
+                    role: types_1.MessageRole.Assistant,
                     model: 'llama2',
                     content: 'Hi there!',
                 },
@@ -95,4 +122,109 @@ describe('InferenceGatewayClient', () => {
             await expect(client.listModels()).rejects.toThrow(errorMessage);
         });
     });
+    describe('generateContentStream', () => {
+        it('should handle SSE events correctly', async () => {
+            const mockRequest = {
+                provider: types_1.Provider.Ollama,
+                model: 'llama2',
+                messages: [
+                    { role: types_1.MessageRole.System, content: 'You are a helpful assistant' },
+                    { role: types_1.MessageRole.User, content: 'Hello' },
+                ],
+            };
+            const mockStream = new TransformStream();
+            const writer = mockStream.writable.getWriter();
+            const encoder = new TextEncoder();
+            global.fetch.mockResolvedValueOnce({
+                ok: true,
+                body: mockStream.readable,
+            });
+            const callbacks = {
+                onMessageStart: jest.fn(),
+                onStreamStart: jest.fn(),
+                onContentStart: jest.fn(),
+                onContentDelta: jest.fn(),
+                onContentEnd: jest.fn(),
+                onMessageEnd: jest.fn(),
+                onStreamEnd: jest.fn(),
+            };
+            const streamPromise = client.generateContentStream(mockRequest, callbacks);
+            await writer.write(encoder.encode('event: message-start\ndata: {"role": "assistant"}\n\n' +
+                'event: stream-start\ndata: {}\n\n' +
+                'event: content-start\ndata: {}\n\n' +
+                'event: content-delta\ndata: {"content": "Hello"}\n\n' +
+                'event: content-delta\ndata: {"content": " there!"}\n\n' +
+                'event: content-end\ndata: {}\n\n' +
+                'event: message-end\ndata: {}\n\n' +
+                'event: stream-end\ndata: {}\n\n'));
+            await writer.close();
+            await streamPromise;
+            expect(callbacks.onMessageStart).toHaveBeenCalledWith('assistant');
+            expect(callbacks.onStreamStart).toHaveBeenCalledTimes(1);
+            expect(callbacks.onContentStart).toHaveBeenCalledTimes(1);
+            expect(callbacks.onContentDelta).toHaveBeenCalledWith('Hello');
+            expect(callbacks.onContentDelta).toHaveBeenCalledWith(' there!');
+            expect(callbacks.onContentEnd).toHaveBeenCalledTimes(1);
+            expect(callbacks.onMessageEnd).toHaveBeenCalledTimes(1);
+            expect(callbacks.onStreamEnd).toHaveBeenCalledTimes(1);
+            expect(global.fetch).toHaveBeenCalledWith(`${mockBaseUrl}/llms/${mockRequest.provider}/generate`, expect.objectContaining({
+                method: 'POST',
+                body: JSON.stringify({
+                    model: mockRequest.model,
+                    messages: mockRequest.messages,
+                    stream: true,
+                    ssevents: true,
+                }),
+            }));
+        });
+        it('should handle errors in the stream response', async () => {
+            const mockRequest = {
+                provider: types_1.Provider.Ollama,
+                model: 'llama2',
+                messages: [{ role: types_1.MessageRole.User, content: 'Hello' }],
+            };
+            global.fetch.mockResolvedValueOnce({
+                ok: false,
+                status: 400,
+                json: () => Promise.resolve({ error: 'Bad Request' }),
+            });
+            await expect(client.generateContentStream(mockRequest, {})).rejects.toThrow('Bad Request');
+        });
+        it('should handle non-readable response body', async () => {
+            const mockRequest = {
+                provider: types_1.Provider.Ollama,
+                model: 'llama2',
+                messages: [{ role: types_1.MessageRole.User, content: 'Hello' }],
+            };
+            global.fetch.mockResolvedValueOnce({
+                ok: true,
+                body: null,
+            });
+            await expect(client.generateContentStream(mockRequest, {})).rejects.toThrow('Response body is not readable');
+        });
+        it('should handle empty events in the stream', async () => {
+            const mockRequest = {
+                provider: types_1.Provider.Ollama,
+                model: 'llama2',
+                messages: [{ role: types_1.MessageRole.User, content: 'Hello' }],
+            };
+            const mockStream = new TransformStream();
+            const writer = mockStream.writable.getWriter();
+            const encoder = new TextEncoder();
+            global.fetch.mockResolvedValueOnce({
+                ok: true,
+                body: mockStream.readable,
+            });
+            const callbacks = {
+                onContentDelta: jest.fn(),
+            };
+            const streamPromise = client.generateContentStream(mockRequest, callbacks);
+            await writer.write(encoder.encode('\n\n'));
+            await writer.write(encoder.encode('event: content-delta\ndata: {"content": "Hello"}\n\n'));
+            await writer.close();
+            await streamPromise;
+            expect(callbacks.onContentDelta).toHaveBeenCalledTimes(1);
+            expect(callbacks.onContentDelta).toHaveBeenCalledWith('Hello');
+        });
+    });
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@inference-gateway/sdk",
-  "version": "0.1.6",
+  "version": "0.3.0",
   "description": "An SDK written in Typescript for the [Inference Gateway](https://github.com/inference-gateway/inference-gateway).",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",