@juspay/neurolink 7.3.8 → 7.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -2
- package/README.md +149 -212
- package/dist/cli/commands/mcp.js +11 -0
- package/dist/cli/factories/commandFactory.js +1 -0
- package/dist/cli/utils/interactiveSetup.js +20 -0
- package/dist/core/types.d.ts +1 -0
- package/dist/core/types.js +1 -0
- package/dist/factories/providerRegistry.js +5 -0
- package/dist/lib/core/types.d.ts +1 -0
- package/dist/lib/core/types.js +1 -0
- package/dist/lib/factories/providerRegistry.js +5 -0
- package/dist/lib/neurolink.js +1 -0
- package/dist/lib/providers/litellm.d.ts +43 -0
- package/dist/lib/providers/litellm.js +188 -0
- package/dist/lib/utils/providerUtils.js +4 -0
- package/dist/neurolink.js +1 -0
- package/dist/providers/litellm.d.ts +43 -0
- package/dist/providers/litellm.js +188 -0
- package/dist/utils/providerUtils.js +4 -0
- package/package.json +6 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,9 +1,21 @@
|
|
|
1
|
-
|
|
1
|
+
# [7.5.0](https://github.com/juspay/neurolink/compare/v7.4.0...v7.5.0) (2025-08-06)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Features
|
|
5
|
+
|
|
6
|
+
* **providers:** add LiteLLM provider integration with access to 100+ AI models ([8918f8e](https://github.com/juspay/neurolink/commit/8918f8efc853a2fa42b75838259b22d8022f02b3))
|
|
2
7
|
|
|
8
|
+
# [7.4.0](https://github.com/juspay/neurolink/compare/v7.3.8...v7.4.0) (2025-08-06)
|
|
9
|
+
|
|
10
|
+
### Features
|
|
11
|
+
|
|
12
|
+
- add Bitbucket MCP server integration ([239ca6d](https://github.com/juspay/neurolink/commit/239ca6df81be5474d983df95998f90e2e6d633b9))
|
|
13
|
+
|
|
14
|
+
## [7.3.8](https://github.com/juspay/neurolink/compare/v7.3.7...v7.3.8) (2025-08-05)
|
|
3
15
|
|
|
4
16
|
### Bug Fixes
|
|
5
17
|
|
|
6
|
-
|
|
18
|
+
- **lint:** improve linting ([580130a](https://github.com/juspay/neurolink/commit/580130aa33700b67f9d99de60dbe3d0c7415adfc))
|
|
7
19
|
|
|
8
20
|
## [7.3.7](https://github.com/juspay/neurolink/compare/v7.3.6...v7.3.7) (2025-08-04)
|
|
9
21
|
|
package/README.md
CHANGED
|
@@ -7,48 +7,44 @@
|
|
|
7
7
|
[](https://www.typescriptlang.org/)
|
|
8
8
|
[](https://github.com/juspay/neurolink/actions)
|
|
9
9
|
|
|
10
|
-
> Enterprise AI Development Platform with
|
|
10
|
+
> **Enterprise AI Development Platform** with universal provider support, factory pattern architecture, and **access to 100+ AI models** through LiteLLM integration. Production-ready with TypeScript support.
|
|
11
11
|
|
|
12
|
-
**NeuroLink** is an Enterprise AI Development Platform that unifies
|
|
12
|
+
**NeuroLink** is an Enterprise AI Development Platform that unifies **10 major AI providers** with intelligent fallback and built-in tool support. Available as both a **programmatic SDK** and **professional CLI tool**. Features LiteLLM integration for **100+ models**, plus 6 core tools working across all providers. Extracted from production use at Juspay.
|
|
13
13
|
|
|
14
|
-
##
|
|
14
|
+
## 🎉 **NEW: LiteLLM Integration - Access 100+ AI Models**
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
- **🔧 Tools-First Design** - All providers include built-in tool support without additional configuration
|
|
18
|
-
- **🌐 Real-time WebSocket Infrastructure** - [Coming Soon - Broken in migration, being fixed]
|
|
19
|
-
- **📊 Advanced Telemetry** - [Coming Soon - Broken in migration, being fixed]
|
|
20
|
-
- **💬 Enhanced Chat Services** - [Coming Soon - Broken in migration, being fixed]
|
|
21
|
-
- **🏗️ Enterprise Architecture** - Production-ready with clean abstractions
|
|
22
|
-
- **🔄 Configuration Management** - Flexible provider configuration
|
|
23
|
-
- **✅ Type Safety** - Industry-standard TypeScript interfaces
|
|
24
|
-
- **⚡ Performance** - Fast response times with streaming support
|
|
25
|
-
- **🛡️ Error Recovery** - Graceful failures with provider fallback
|
|
16
|
+
**NeuroLink now supports LiteLLM**, providing unified access to **100+ AI models** from all major providers through a single interface:
|
|
26
17
|
|
|
27
|
-
|
|
18
|
+
- **🔄 Universal Access**: OpenAI, Anthropic, Google, Mistral, Meta, and more
|
|
19
|
+
- **🎯 Unified Interface**: OpenAI-compatible API for all models
|
|
20
|
+
- **💰 Cost Optimization**: Automatic routing to cost-effective models
|
|
21
|
+
- **⚡ Load Balancing**: Automatic failover and load distribution
|
|
22
|
+
- **📊 Analytics**: Built-in usage tracking and monitoring
|
|
28
23
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
- ✅ **Real Streaming Architecture**: Vercel AI SDK real streaming with comprehensive analytics support
|
|
33
|
-
- ✅ **Performance Optimization**: 68% improvement in provider status checks (16s → 5s via parallel execution)
|
|
34
|
-
- ✅ **Memory Management**: Automatic cleanup for operations >50MB with performance tracking
|
|
35
|
-
- ✅ **Edge Case Handling**: Input validation, timeout warnings, and network resilience
|
|
36
|
-
- ✅ **Scalability Improvements**: Retry logic, circuit breakers, and rate limiting
|
|
37
|
-
- ✅ **Factory Pattern Architecture**: All providers inherit from BaseProvider with built-in tool support
|
|
38
|
-
- ✅ **Direct Tools**: Six core tools available across all providers (getCurrentTime, readFile, listDirectory, calculateMath, writeFile, searchFiles)
|
|
39
|
-
|
|
40
|
-
> **Production Ready**: NeuroLink now features enterprise-grade performance optimizations, comprehensive error handling, and real streaming architecture for multi-modal future.
|
|
24
|
+
```bash
|
|
25
|
+
# Quick start with LiteLLM
|
|
26
|
+
pip install litellm && litellm --port 4000
|
|
41
27
|
|
|
42
|
-
|
|
28
|
+
# Use any of 100+ models through one interface
|
|
29
|
+
npx @juspay/neurolink generate "Hello" --provider litellm --model "openai/gpt-4o"
|
|
30
|
+
npx @juspay/neurolink generate "Hello" --provider litellm --model "anthropic/claude-3-5-sonnet"
|
|
31
|
+
npx @juspay/neurolink generate "Hello" --provider litellm --model "google/gemini-2.0-flash"
|
|
32
|
+
```
|
|
43
33
|
|
|
44
|
-
**
|
|
34
|
+
**[📖 Complete LiteLLM Integration Guide](./docs/LITELLM-INTEGRATION.md)** - Setup, configuration, and 100+ model access
|
|
45
35
|
|
|
46
|
-
|
|
47
|
-
- ✅ **Enhanced Generation**: `generate()` as primary generation function
|
|
48
|
-
- ✅ **Factory Enhanced**: Better provider management across all methods
|
|
49
|
-
- ✅ **Zero Breaking Changes**: All existing code continues working (backward compatibility)
|
|
36
|
+
## 🚀 Enterprise Platform Features
|
|
50
37
|
|
|
51
|
-
|
|
38
|
+
- **🏭 Factory Pattern Architecture** - Unified provider management through BaseProvider inheritance
|
|
39
|
+
- **🔧 Tools-First Design** - All providers include built-in tool support without additional configuration
|
|
40
|
+
- **🔗 LiteLLM Integration** - **100+ models** from all major providers through unified interface
|
|
41
|
+
- **🏗️ Enterprise Architecture** - Production-ready with clean abstractions
|
|
42
|
+
- **🔄 Configuration Management** - Flexible provider configuration with automatic backups
|
|
43
|
+
- **✅ Type Safety** - Industry-standard TypeScript interfaces
|
|
44
|
+
- **⚡ Performance** - Fast response times with streaming support and 68% improved status checks
|
|
45
|
+
- **🛡️ Error Recovery** - Graceful failures with provider fallback and retry logic
|
|
46
|
+
- **📊 Analytics & Evaluation** - Built-in usage tracking and AI-powered quality assessment
|
|
47
|
+
- **🔧 MCP Integration** - Model Context Protocol with 6 built-in tools and 58+ discoverable servers
|
|
52
48
|
|
|
53
49
|
---
|
|
54
50
|
|
|
@@ -57,26 +53,24 @@
|
|
|
57
53
|
### Install & Run (2 minutes)
|
|
58
54
|
|
|
59
55
|
```bash
|
|
60
|
-
#
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
npx @juspay/neurolink generate "Hello, AI"
|
|
65
|
-
npx @juspay/neurolink gen "Hello, AI" # Shortest form
|
|
66
|
-
|
|
67
|
-
# ✨ Primary Method (generate) - Recommended
|
|
68
|
-
npx @juspay/neurolink generate "Explain AI" --provider google-ai
|
|
69
|
-
npx @juspay/neurolink gen "Write code" --provider openai # Shortest form
|
|
56
|
+
# Option 1: LiteLLM - Access 100+ models through one interface
|
|
57
|
+
pip install litellm && litellm --port 4000
|
|
58
|
+
export LITELLM_BASE_URL="http://localhost:4000"
|
|
59
|
+
export LITELLM_API_KEY="sk-anything"
|
|
70
60
|
|
|
71
|
-
#
|
|
72
|
-
npx @juspay/neurolink generate "
|
|
73
|
-
npx @juspay/neurolink generate "
|
|
74
|
-
npx @juspay/neurolink generate "Help me" --context '{"userId":"123"}' --debug
|
|
61
|
+
# Use any of 100+ models
|
|
62
|
+
npx @juspay/neurolink generate "Hello, AI" --provider litellm --model "openai/gpt-4o"
|
|
63
|
+
npx @juspay/neurolink generate "Hello, AI" --provider litellm --model "anthropic/claude-3-5-sonnet"
|
|
75
64
|
|
|
76
|
-
#
|
|
77
|
-
|
|
65
|
+
# Option 2: Direct Provider - Quick setup with Google AI Studio (free tier)
|
|
66
|
+
export GOOGLE_AI_API_KEY="AIza-your-google-ai-api-key"
|
|
67
|
+
npx @juspay/neurolink generate "Hello, AI" --provider google-ai
|
|
78
68
|
|
|
79
|
-
|
|
69
|
+
# CLI Commands - No installation required
|
|
70
|
+
npx @juspay/neurolink generate "Explain AI" # Auto-selects best provider
|
|
71
|
+
npx @juspay/neurolink gen "Write code" # Shortest form
|
|
72
|
+
npx @juspay/neurolink stream "Tell a story" # Real-time streaming
|
|
73
|
+
npx @juspay/neurolink status # Check all providers
|
|
80
74
|
```
|
|
81
75
|
|
|
82
76
|
```bash
|
|
@@ -87,20 +81,38 @@ npm install @juspay/neurolink
|
|
|
87
81
|
### Basic Usage
|
|
88
82
|
|
|
89
83
|
```typescript
|
|
90
|
-
import { NeuroLink } from "@juspay/neurolink";
|
|
91
|
-
|
|
92
|
-
//
|
|
93
|
-
const
|
|
94
|
-
|
|
84
|
+
import { NeuroLink, AIProviderFactory } from "@juspay/neurolink";
|
|
85
|
+
|
|
86
|
+
// LiteLLM - Access 100+ models through unified interface
|
|
87
|
+
const litellmProvider = await AIProviderFactory.createProvider(
|
|
88
|
+
"litellm",
|
|
89
|
+
"openai/gpt-4o",
|
|
90
|
+
);
|
|
91
|
+
const result = await litellmProvider.generate({
|
|
95
92
|
input: { text: "Write a haiku about programming" },
|
|
96
|
-
provider: "google-ai",
|
|
97
|
-
timeout: "30s", // Optional: Set custom timeout (default: 30s)
|
|
98
93
|
});
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
const
|
|
102
|
-
|
|
103
|
-
|
|
94
|
+
|
|
95
|
+
// Compare multiple models simultaneously
|
|
96
|
+
const models = [
|
|
97
|
+
"openai/gpt-4o",
|
|
98
|
+
"anthropic/claude-3-5-sonnet",
|
|
99
|
+
"google/gemini-2.0-flash",
|
|
100
|
+
];
|
|
101
|
+
const comparisons = await Promise.all(
|
|
102
|
+
models.map(async (model) => {
|
|
103
|
+
const provider = await AIProviderFactory.createProvider("litellm", model);
|
|
104
|
+
const result = await provider.generate({
|
|
105
|
+
input: { text: "Explain quantum computing" },
|
|
106
|
+
});
|
|
107
|
+
return { model, response: result.content, provider: result.provider };
|
|
108
|
+
}),
|
|
109
|
+
);
|
|
110
|
+
|
|
111
|
+
// Auto-select best available provider
|
|
112
|
+
const neurolink = new NeuroLink();
|
|
113
|
+
const autoResult = await neurolink.generate({
|
|
114
|
+
input: { text: "Write a business email" },
|
|
115
|
+
provider: "google-ai", // or let it auto-select
|
|
104
116
|
timeout: "30s",
|
|
105
117
|
});
|
|
106
118
|
|
|
@@ -132,119 +144,58 @@ const poem = await provider.generate({ input: { text: "Write a poem" } });
|
|
|
132
144
|
const joke = await provider.gen({ input: { text: "Tell me a joke" } });
|
|
133
145
|
```
|
|
134
146
|
|
|
135
|
-
###
|
|
147
|
+
### Enhanced Features
|
|
136
148
|
|
|
137
|
-
####
|
|
149
|
+
#### CLI with Analytics & Evaluation
|
|
138
150
|
|
|
139
151
|
```bash
|
|
140
|
-
# Basic AI generation
|
|
152
|
+
# Basic AI generation with auto-provider selection
|
|
141
153
|
npx @juspay/neurolink generate "Write a business email"
|
|
142
154
|
|
|
143
|
-
#
|
|
144
|
-
npx @juspay/neurolink generate "Write
|
|
155
|
+
# LiteLLM with specific model
|
|
156
|
+
npx @juspay/neurolink generate "Write code" --provider litellm --model "anthropic/claude-3-5-sonnet"
|
|
145
157
|
|
|
146
|
-
#
|
|
147
|
-
|
|
148
|
-
# ⭐ Response Evaluation: AI-powered quality scores
|
|
158
|
+
# With analytics and evaluation
|
|
159
|
+
npx @juspay/neurolink generate "Write a proposal" --enable-analytics --enable-evaluation --debug
|
|
149
160
|
|
|
150
|
-
#
|
|
151
|
-
npx @juspay/neurolink
|
|
161
|
+
# Streaming with tools (default behavior)
|
|
162
|
+
npx @juspay/neurolink stream "What time is it and write a file with the current date"
|
|
152
163
|
```
|
|
153
164
|
|
|
154
|
-
####
|
|
165
|
+
#### SDK with LiteLLM and Enhancement Features
|
|
155
166
|
|
|
156
167
|
```typescript
|
|
157
|
-
import { NeuroLink } from "@juspay/neurolink";
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
168
|
+
import { NeuroLink, AIProviderFactory } from "@juspay/neurolink";
|
|
169
|
+
|
|
170
|
+
// LiteLLM multi-model comparison
|
|
171
|
+
const models = [
|
|
172
|
+
"openai/gpt-4o",
|
|
173
|
+
"anthropic/claude-3-5-sonnet",
|
|
174
|
+
"google/gemini-2.0-flash",
|
|
175
|
+
];
|
|
176
|
+
const comparisons = await Promise.all(
|
|
177
|
+
models.map(async (model) => {
|
|
178
|
+
const provider = await AIProviderFactory.createProvider("litellm", model);
|
|
179
|
+
return await provider.generate({
|
|
180
|
+
input: { text: "Explain the benefits of renewable energy" },
|
|
181
|
+
enableAnalytics: true,
|
|
182
|
+
enableEvaluation: true,
|
|
183
|
+
});
|
|
184
|
+
}),
|
|
185
|
+
);
|
|
162
186
|
|
|
163
|
-
//
|
|
164
|
-
const
|
|
187
|
+
// Enhanced generation with analytics
|
|
188
|
+
const neurolink = new NeuroLink();
|
|
189
|
+
const result = await neurolink.generate({
|
|
165
190
|
input: { text: "Write a business proposal" },
|
|
166
191
|
enableAnalytics: true, // Get usage & cost data
|
|
167
192
|
enableEvaluation: true, // Get AI quality scores
|
|
168
|
-
context: { project: "Q1-sales" },
|
|
169
|
-
});
|
|
170
|
-
|
|
171
|
-
// Access enhancement data
|
|
172
|
-
console.log("📊 Usage:", enhancedResult.analytics);
|
|
173
|
-
console.log("⭐ Quality:", enhancedResult.evaluation);
|
|
174
|
-
console.log("Response:", enhancedResult.content);
|
|
175
|
-
|
|
176
|
-
// Enhanced evaluation included when enableEvaluation is true
|
|
177
|
-
// Returns basic quality scores for the generated content
|
|
178
|
-
```
|
|
179
|
-
|
|
180
|
-
### 🌐 Enterprise Real-time Features (NEW! 🚀)
|
|
181
|
-
|
|
182
|
-
#### Real-time WebSocket Chat
|
|
183
|
-
|
|
184
|
-
```typescript
|
|
185
|
-
import {
|
|
186
|
-
createEnhancedChatService,
|
|
187
|
-
NeuroLinkWebSocketServer,
|
|
188
|
-
} from "@juspay/neurolink";
|
|
189
|
-
|
|
190
|
-
// Enhanced chat with WebSocket support
|
|
191
|
-
const chatService = createEnhancedChatService({
|
|
192
|
-
provider: await createBestAIProvider(),
|
|
193
|
-
enableWebSocket: true,
|
|
194
|
-
enableSSE: true,
|
|
195
|
-
streamingConfig: {
|
|
196
|
-
bufferSize: 8192,
|
|
197
|
-
compressionEnabled: true,
|
|
198
|
-
},
|
|
199
|
-
});
|
|
200
|
-
|
|
201
|
-
// WebSocket server for real-time applications
|
|
202
|
-
const wsServer = new NeuroLinkWebSocketServer({
|
|
203
|
-
port: 8080,
|
|
204
|
-
maxConnections: 1000,
|
|
205
|
-
enableCompression: true,
|
|
193
|
+
context: { project: "Q1-sales" },
|
|
206
194
|
});
|
|
207
195
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
prompt: message.data.prompt,
|
|
212
|
-
onChunk: (chunk) => {
|
|
213
|
-
wsServer.sendMessage(connectionId, {
|
|
214
|
-
type: "ai-chunk",
|
|
215
|
-
data: { chunk },
|
|
216
|
-
});
|
|
217
|
-
},
|
|
218
|
-
});
|
|
219
|
-
});
|
|
220
|
-
```
|
|
221
|
-
|
|
222
|
-
#### Enterprise Telemetry Integration
|
|
223
|
-
|
|
224
|
-
```typescript
|
|
225
|
-
import { initializeTelemetry, getTelemetryStatus } from "@juspay/neurolink";
|
|
226
|
-
|
|
227
|
-
// Optional enterprise monitoring (zero overhead when disabled)
|
|
228
|
-
const telemetry = initializeTelemetry({
|
|
229
|
-
serviceName: "my-ai-app",
|
|
230
|
-
endpoint: "http://localhost:4318",
|
|
231
|
-
enableTracing: true,
|
|
232
|
-
enableMetrics: true,
|
|
233
|
-
enableLogs: true,
|
|
234
|
-
});
|
|
235
|
-
|
|
236
|
-
// Check telemetry status
|
|
237
|
-
const status = await getTelemetryStatus();
|
|
238
|
-
console.log("Telemetry enabled:", status.enabled);
|
|
239
|
-
console.log("Service:", status.service);
|
|
240
|
-
console.log("Version:", status.version);
|
|
241
|
-
|
|
242
|
-
// All AI operations are now automatically monitored
|
|
243
|
-
const provider = await createBestAIProvider();
|
|
244
|
-
const result = await provider.generate({
|
|
245
|
-
prompt: "Generate business report",
|
|
246
|
-
});
|
|
247
|
-
// Telemetry automatically tracks: response time, token usage, cost, errors
|
|
196
|
+
console.log("📊 Usage:", result.analytics);
|
|
197
|
+
console.log("⭐ Quality:", result.evaluation);
|
|
198
|
+
console.log("Response:", result.content);
|
|
248
199
|
```
|
|
249
200
|
|
|
250
201
|
### Environment Setup
|
|
@@ -263,22 +214,19 @@ npx @juspay/neurolink status
|
|
|
263
214
|
|
|
264
215
|
## ✨ Key Features
|
|
265
216
|
|
|
217
|
+
- 🔗 **LiteLLM Integration** - **Access 100+ AI models** from all major providers through unified interface
|
|
266
218
|
- 🏭 **Factory Pattern Architecture** - Unified provider management with BaseProvider inheritance
|
|
267
|
-
- 🔧 **Tools-First Design** - All providers automatically include direct
|
|
268
|
-
- 🔄 **
|
|
269
|
-
-
|
|
270
|
-
-
|
|
271
|
-
-
|
|
272
|
-
-
|
|
273
|
-
-
|
|
274
|
-
-
|
|
275
|
-
-
|
|
276
|
-
-
|
|
277
|
-
-
|
|
278
|
-
- 🤖 **AI Analysis Tools** - Built-in optimization and workflow assistance
|
|
279
|
-
- 🏠 **Local AI Support** - Run completely offline with Ollama
|
|
280
|
-
- 🌍 **Open Source Models** - Access 100,000+ models via Hugging Face
|
|
281
|
-
- 🇪🇺 **GDPR Compliance** - European data processing with Mistral AI
|
|
219
|
+
- 🔧 **Tools-First Design** - All providers automatically include 6 direct tools (getCurrentTime, readFile, listDirectory, calculateMath, writeFile, searchFiles)
|
|
220
|
+
- 🔄 **10 AI Providers** - OpenAI, Bedrock, Vertex AI, Google AI Studio, Anthropic, Azure, **LiteLLM**, Hugging Face, Ollama, Mistral AI
|
|
221
|
+
- 💰 **Cost Optimization** - Automatic selection of cheapest models and LiteLLM routing
|
|
222
|
+
- ⚡ **Automatic Fallback** - Never fail when providers are down, intelligent provider switching
|
|
223
|
+
- 🖥️ **CLI + SDK** - Use from command line or integrate programmatically with TypeScript support
|
|
224
|
+
- 🛡️ **Production Ready** - Enterprise-grade error handling, performance optimization, extracted from production
|
|
225
|
+
- ✅ **MCP Integration** - Model Context Protocol with working built-in tools and 58+ discoverable external servers
|
|
226
|
+
- 🔍 **Smart Model Resolution** - Fuzzy matching, aliases, and capability-based search across all providers
|
|
227
|
+
- 🏠 **Local AI Support** - Run completely offline with Ollama or through LiteLLM proxy
|
|
228
|
+
- 🌍 **Universal Model Access** - Direct providers + 100,000+ models via Hugging Face + 100+ models via LiteLLM
|
|
229
|
+
- 📊 **Analytics & Evaluation** - Built-in usage tracking and AI-powered quality assessment
|
|
282
230
|
|
|
283
231
|
## 🛠️ MCP Integration Status ✅ **BUILT-IN TOOLS WORKING**
|
|
284
232
|
|
|
@@ -303,6 +251,11 @@ npx @juspay/neurolink generate "Write a poem" --disable-tools
|
|
|
303
251
|
|
|
304
252
|
# Discover available MCP servers
|
|
305
253
|
npx @juspay/neurolink mcp discover --format table
|
|
254
|
+
|
|
255
|
+
# Install popular MCP servers (NEW: Bitbucket support added!)
|
|
256
|
+
npx @juspay/neurolink mcp install filesystem
|
|
257
|
+
npx @juspay/neurolink mcp install github
|
|
258
|
+
npx @juspay/neurolink mcp install bitbucket # 🆕 NEW
|
|
306
259
|
```
|
|
307
260
|
|
|
308
261
|
### 🔧 SDK Custom Tool Registration (NEW!)
|
|
@@ -348,45 +301,28 @@ neurolink.registerTools({
|
|
|
348
301
|
});
|
|
349
302
|
```
|
|
350
303
|
|
|
351
|
-
##
|
|
352
|
-
|
|
353
|
-
NeuroLink now features a revolutionary dynamic model configuration system that eliminates hardcoded model lists and enables automatic cost optimization.
|
|
304
|
+
## 💰 Smart Model Selection
|
|
354
305
|
|
|
355
|
-
|
|
306
|
+
NeuroLink features intelligent model selection and cost optimization:
|
|
356
307
|
|
|
357
|
-
|
|
358
|
-
- **💰 Cost-Optimized**: Automatic selection of cheapest models for tasks
|
|
359
|
-
- **🔍 Smart Search**: Find models by capabilities (functionCalling, vision, etc.)
|
|
360
|
-
- **🏷️ Alias Support**: Use friendly names like "claude-latest" or "best-coding"
|
|
361
|
-
- **📊 Real-Time Pricing**: Always current model costs and performance data
|
|
308
|
+
### Cost Optimization Features
|
|
362
309
|
|
|
363
|
-
|
|
310
|
+
- **💰 Automatic Cost Optimization**: Selects cheapest models for simple tasks
|
|
311
|
+
- **🔄 LiteLLM Model Routing**: Access 100+ models with automatic load balancing
|
|
312
|
+
- **🔍 Capability-Based Selection**: Find models with specific features (vision, function calling)
|
|
313
|
+
- **⚡ Intelligent Fallback**: Seamless switching when providers fail
|
|
364
314
|
|
|
365
315
|
```bash
|
|
366
316
|
# Cost optimization - automatically use cheapest model
|
|
367
317
|
npx @juspay/neurolink generate "Hello" --optimize-cost
|
|
368
318
|
|
|
369
|
-
#
|
|
370
|
-
npx @juspay/neurolink generate "
|
|
319
|
+
# LiteLLM specific model selection
|
|
320
|
+
npx @juspay/neurolink generate "Complex analysis" --provider litellm --model "anthropic/claude-3-5-sonnet"
|
|
371
321
|
|
|
372
|
-
#
|
|
373
|
-
npx @juspay/neurolink
|
|
374
|
-
|
|
375
|
-
# Test dynamic model server
|
|
376
|
-
npm run model-server # Starts config server on localhost:3001
|
|
377
|
-
npm run test:dynamicModels # Comprehensive test suite
|
|
322
|
+
# Auto-select best available provider
|
|
323
|
+
npx @juspay/neurolink generate "Write code" # Automatically chooses optimal provider
|
|
378
324
|
```
|
|
379
325
|
|
|
380
|
-
### 📊 Current Model Inventory (Auto-Updated)
|
|
381
|
-
|
|
382
|
-
- **10 active models** across 4 providers
|
|
383
|
-
- **Cheapest**: Gemini 2.0 Flash ($0.000075/1K tokens)
|
|
384
|
-
- **Most capable**: Claude 3 Opus (functionCalling + vision + analysis)
|
|
385
|
-
- **Best for coding**: Claude 3 Opus, Gemini 2.0 Flash
|
|
386
|
-
- **1 deprecated model** automatically excluded
|
|
387
|
-
|
|
388
|
-
**[📖 Complete Dynamic Models Guide](./docs/DYNAMIC-MODELS.md)** - Setup, configuration, and advanced usage
|
|
389
|
-
|
|
390
326
|
## 💻 Essential Examples
|
|
391
327
|
|
|
392
328
|
### CLI Commands
|
|
@@ -528,17 +464,18 @@ cd neurolink-demo && node server.js
|
|
|
528
464
|
|
|
529
465
|
## 🏗️ Supported Providers & Models
|
|
530
466
|
|
|
531
|
-
| Provider | Models
|
|
532
|
-
| -------------------- |
|
|
533
|
-
| **
|
|
534
|
-
| **Google AI Studio** | Gemini 2.5 Flash/Pro
|
|
535
|
-
| **
|
|
536
|
-
| **
|
|
537
|
-
| **
|
|
538
|
-
| **
|
|
539
|
-
| **
|
|
540
|
-
| **Ollama** 🆕 | Llama 3.2, Gemma, Mistral
|
|
541
|
-
| **
|
|
467
|
+
| Provider | Models | Auth Method | Free Tier | Tool Support | Key Benefit |
|
|
468
|
+
| -------------------- | --------------------------------- | ------------------ | --------- | ------------ | -------------------- |
|
|
469
|
+
| **🔗 LiteLLM** 🆕 | **100+ Models** (All Providers) | Proxy Server | Varies | ✅ Full | **Universal Access** |
|
|
470
|
+
| **Google AI Studio** | Gemini 2.5 Flash/Pro | API Key | ✅ | ✅ Full | Free Tier Available |
|
|
471
|
+
| **OpenAI** | GPT-4o, GPT-4o-mini | API Key | ❌ | ✅ Full | Industry Standard |
|
|
472
|
+
| **Anthropic** | Claude 3.5 Sonnet | API Key | ❌ | ✅ Full | Advanced Reasoning |
|
|
473
|
+
| **Amazon Bedrock** | Claude 3.5/3.7 Sonnet | AWS Credentials | ❌ | ✅ Full\* | Enterprise Scale |
|
|
474
|
+
| **Google Vertex AI** | Gemini 2.5 Flash | Service Account | ❌ | ✅ Full | Enterprise Google |
|
|
475
|
+
| **Azure OpenAI** | GPT-4, GPT-3.5 | API Key + Endpoint | ❌ | ✅ Full | Microsoft Ecosystem |
|
|
476
|
+
| **Ollama** 🆕 | Llama 3.2, Gemma, Mistral (Local) | None (Local) | ✅ | ⚠️ Partial | Complete Privacy |
|
|
477
|
+
| **Hugging Face** 🆕 | 100,000+ open source models | API Key | ✅ | ⚠️ Partial | Open Source |
|
|
478
|
+
| **Mistral AI** 🆕 | Tiny, Small, Medium, Large | API Key | ✅ | ✅ Full | European/GDPR |
|
|
542
479
|
|
|
543
480
|
**Tool Support Legend:**
|
|
544
481
|
|
package/dist/cli/commands/mcp.js
CHANGED
|
@@ -68,6 +68,17 @@ const POPULAR_MCP_SERVERS = {
|
|
|
68
68
|
transport: "stdio",
|
|
69
69
|
description: "Persistent memory and knowledge storage",
|
|
70
70
|
},
|
|
71
|
+
bitbucket: {
|
|
72
|
+
command: "npx",
|
|
73
|
+
args: ["-y", "@nexus2520/bitbucket-mcp-server"],
|
|
74
|
+
env: {
|
|
75
|
+
BITBUCKET_USERNAME: "${BITBUCKET_USERNAME}",
|
|
76
|
+
BITBUCKET_TOKEN: "${BITBUCKET_TOKEN}",
|
|
77
|
+
BITBUCKET_BASE_URL: "${BITBUCKET_BASE_URL}",
|
|
78
|
+
},
|
|
79
|
+
transport: "stdio",
|
|
80
|
+
description: "Bitbucket repository management and development workflows",
|
|
81
|
+
},
|
|
71
82
|
};
|
|
72
83
|
/**
|
|
73
84
|
* Convert SDK MCPStatus to CLI format with server list
|
|
@@ -133,6 +133,26 @@ export const PROVIDER_CONFIGS = [
|
|
|
133
133
|
{ key: "MISTRAL_API_KEY", prompt: "Mistral AI API Key", secure: true },
|
|
134
134
|
],
|
|
135
135
|
},
|
|
136
|
+
{
|
|
137
|
+
id: AIProviderName.LITELLM,
|
|
138
|
+
name: "LiteLLM",
|
|
139
|
+
description: "Access 100+ models via LiteLLM proxy server",
|
|
140
|
+
envVars: [
|
|
141
|
+
{
|
|
142
|
+
key: "LITELLM_BASE_URL",
|
|
143
|
+
prompt: "LiteLLM Proxy Server URL",
|
|
144
|
+
default: "http://localhost:4000",
|
|
145
|
+
secure: false,
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
key: "LITELLM_API_KEY",
|
|
149
|
+
prompt: "LiteLLM API Key (or any value)",
|
|
150
|
+
default: "sk-anything",
|
|
151
|
+
secure: false,
|
|
152
|
+
optional: true,
|
|
153
|
+
},
|
|
154
|
+
],
|
|
155
|
+
},
|
|
136
156
|
];
|
|
137
157
|
/**
|
|
138
158
|
* Run the interactive setup wizard
|
package/dist/core/types.d.ts
CHANGED
package/dist/core/types.js
CHANGED
|
@@ -12,6 +12,7 @@ export var AIProviderName;
|
|
|
12
12
|
AIProviderName["HUGGINGFACE"] = "huggingface";
|
|
13
13
|
AIProviderName["OLLAMA"] = "ollama";
|
|
14
14
|
AIProviderName["MISTRAL"] = "mistral";
|
|
15
|
+
AIProviderName["LITELLM"] = "litellm";
|
|
15
16
|
AIProviderName["AUTO"] = "auto";
|
|
16
17
|
})(AIProviderName || (AIProviderName = {}));
|
|
17
18
|
/**
|
|
@@ -71,6 +71,11 @@ export class ProviderRegistry {
|
|
|
71
71
|
const { OllamaProvider } = await import("../providers/ollama.js");
|
|
72
72
|
return new OllamaProvider(modelName);
|
|
73
73
|
}, process.env.OLLAMA_MODEL || "llama3.1:8b", ["ollama", "local"]);
|
|
74
|
+
// Register LiteLLM provider
|
|
75
|
+
ProviderFactory.registerProvider(AIProviderName.LITELLM, async (modelName, providerName, sdk) => {
|
|
76
|
+
const { LiteLLMProvider } = await import("../providers/litellm.js");
|
|
77
|
+
return new LiteLLMProvider(modelName, sdk);
|
|
78
|
+
}, process.env.LITELLM_MODEL || "openai/gpt-4o-mini", ["litellm"]);
|
|
74
79
|
logger.debug("All providers registered successfully");
|
|
75
80
|
this.registered = true;
|
|
76
81
|
}
|
package/dist/lib/core/types.d.ts
CHANGED
package/dist/lib/core/types.js
CHANGED
|
@@ -12,6 +12,7 @@ export var AIProviderName;
|
|
|
12
12
|
AIProviderName["HUGGINGFACE"] = "huggingface";
|
|
13
13
|
AIProviderName["OLLAMA"] = "ollama";
|
|
14
14
|
AIProviderName["MISTRAL"] = "mistral";
|
|
15
|
+
AIProviderName["LITELLM"] = "litellm";
|
|
15
16
|
AIProviderName["AUTO"] = "auto";
|
|
16
17
|
})(AIProviderName || (AIProviderName = {}));
|
|
17
18
|
/**
|
|
@@ -71,6 +71,11 @@ export class ProviderRegistry {
|
|
|
71
71
|
const { OllamaProvider } = await import("../providers/ollama.js");
|
|
72
72
|
return new OllamaProvider(modelName);
|
|
73
73
|
}, process.env.OLLAMA_MODEL || "llama3.1:8b", ["ollama", "local"]);
|
|
74
|
+
// Register LiteLLM provider
|
|
75
|
+
ProviderFactory.registerProvider(AIProviderName.LITELLM, async (modelName, providerName, sdk) => {
|
|
76
|
+
const { LiteLLMProvider } = await import("../providers/litellm.js");
|
|
77
|
+
return new LiteLLMProvider(modelName, sdk);
|
|
78
|
+
}, process.env.LITELLM_MODEL || "openai/gpt-4o-mini", ["litellm"]);
|
|
74
79
|
logger.debug("All providers registered successfully");
|
|
75
80
|
this.registered = true;
|
|
76
81
|
}
|
package/dist/lib/neurolink.js
CHANGED
|
@@ -701,6 +701,7 @@ export class NeuroLink {
|
|
|
701
701
|
"huggingface",
|
|
702
702
|
"ollama",
|
|
703
703
|
"mistral",
|
|
704
|
+
"litellm",
|
|
704
705
|
];
|
|
705
706
|
// 🚀 PERFORMANCE FIX: Test providers with controlled concurrency
|
|
706
707
|
// This reduces total time from 16s (sequential) to ~3s (parallel) while preventing resource exhaustion
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { ZodType, ZodTypeDef } from "zod";
|
|
2
|
+
import { type Schema, type LanguageModelV1 } from "ai";
|
|
3
|
+
import type { AIProviderName } from "../core/types.js";
|
|
4
|
+
import type { StreamOptions, StreamResult } from "../types/streamTypes.js";
|
|
5
|
+
import { BaseProvider } from "../core/baseProvider.js";
|
|
6
|
+
/**
|
|
7
|
+
* LiteLLM Provider - BaseProvider Implementation
|
|
8
|
+
* Provides access to 100+ models via LiteLLM proxy server
|
|
9
|
+
*/
|
|
10
|
+
export declare class LiteLLMProvider extends BaseProvider {
|
|
11
|
+
private model;
|
|
12
|
+
constructor(modelName?: string, sdk?: unknown);
|
|
13
|
+
protected getProviderName(): AIProviderName;
|
|
14
|
+
protected getDefaultModel(): string;
|
|
15
|
+
/**
|
|
16
|
+
* Returns the Vercel AI SDK model instance for LiteLLM
|
|
17
|
+
*/
|
|
18
|
+
protected getAISDKModel(): LanguageModelV1;
|
|
19
|
+
protected handleProviderError(error: unknown): Error;
|
|
20
|
+
/**
|
|
21
|
+
* LiteLLM supports tools for compatible models
|
|
22
|
+
*/
|
|
23
|
+
supportsTools(): boolean;
|
|
24
|
+
/**
|
|
25
|
+
* Provider-specific streaming implementation
|
|
26
|
+
* Note: This is only used when tools are disabled
|
|
27
|
+
*/
|
|
28
|
+
protected executeStream(options: StreamOptions, analysisSchema?: ZodType<unknown, ZodTypeDef, unknown> | Schema<unknown>): Promise<StreamResult>;
|
|
29
|
+
/**
|
|
30
|
+
* Get available models from LiteLLM proxy server
|
|
31
|
+
*
|
|
32
|
+
* TODO: Implement dynamic fetching from LiteLLM's /v1/models endpoint.
|
|
33
|
+
* Currently returns a hardcoded list of commonly available models.
|
|
34
|
+
*
|
|
35
|
+
* Implementation would involve:
|
|
36
|
+
* 1. Fetch from `${baseURL}/v1/models`
|
|
37
|
+
* 2. Parse response to extract model IDs
|
|
38
|
+
* 3. Handle network errors gracefully
|
|
39
|
+
* 4. Cache results to avoid repeated API calls
|
|
40
|
+
*/
|
|
41
|
+
getAvailableModels(): Promise<string[]>;
|
|
42
|
+
private validateStreamOptions;
|
|
43
|
+
}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { createOpenAI } from "@ai-sdk/openai";
|
|
2
|
+
import { streamText } from "ai";
|
|
3
|
+
import { BaseProvider } from "../core/baseProvider.js";
|
|
4
|
+
import { logger } from "../utils/logger.js";
|
|
5
|
+
import { createTimeoutController, TimeoutError, } from "../utils/timeout.js";
|
|
6
|
+
import { DEFAULT_MAX_TOKENS } from "../core/constants.js";
|
|
7
|
+
import { getProviderModel } from "../utils/providerConfig.js";
|
|
8
|
+
import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
|
|
9
|
+
// Configuration helpers
|
|
10
|
+
const getLiteLLMConfig = () => {
|
|
11
|
+
return {
|
|
12
|
+
baseURL: process.env.LITELLM_BASE_URL || "http://localhost:4000",
|
|
13
|
+
apiKey: process.env.LITELLM_API_KEY || "sk-anything",
|
|
14
|
+
};
|
|
15
|
+
};
|
|
16
|
+
/**
|
|
17
|
+
* Returns the default model name for LiteLLM.
|
|
18
|
+
*
|
|
19
|
+
* LiteLLM uses a 'provider/model' format for model names.
|
|
20
|
+
* For example:
|
|
21
|
+
* - 'openai/gpt-4o-mini'
|
|
22
|
+
* - 'openai/gpt-3.5-turbo'
|
|
23
|
+
* - 'anthropic/claude-3-sonnet-20240229'
|
|
24
|
+
* - 'google/gemini-pro'
|
|
25
|
+
*
|
|
26
|
+
* You can override the default by setting the LITELLM_MODEL environment variable.
|
|
27
|
+
*/
|
|
28
|
+
const getDefaultLiteLLMModel = () => {
|
|
29
|
+
return getProviderModel("LITELLM_MODEL", "openai/gpt-4o-mini");
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* LiteLLM Provider - BaseProvider Implementation
|
|
33
|
+
* Provides access to 100+ models via LiteLLM proxy server
|
|
34
|
+
*/
|
|
35
|
+
export class LiteLLMProvider extends BaseProvider {
|
|
36
|
+
model;
|
|
37
|
+
constructor(modelName, sdk) {
|
|
38
|
+
super(modelName, "litellm", sdk);
|
|
39
|
+
// Initialize LiteLLM using OpenAI SDK with explicit configuration
|
|
40
|
+
const config = getLiteLLMConfig();
|
|
41
|
+
// Create OpenAI SDK instance configured for LiteLLM proxy
|
|
42
|
+
// LiteLLM acts as a proxy server that implements the OpenAI-compatible API.
|
|
43
|
+
// To communicate with LiteLLM instead of the default OpenAI endpoint, we use createOpenAI
|
|
44
|
+
// with a custom baseURL and apiKey. This ensures all requests are routed through the LiteLLM
|
|
45
|
+
// proxy, allowing access to multiple models and custom authentication.
|
|
46
|
+
const customOpenAI = createOpenAI({
|
|
47
|
+
baseURL: config.baseURL,
|
|
48
|
+
apiKey: config.apiKey,
|
|
49
|
+
});
|
|
50
|
+
this.model = customOpenAI(this.modelName || getDefaultLiteLLMModel());
|
|
51
|
+
logger.debug("LiteLLM Provider initialized", {
|
|
52
|
+
modelName: this.modelName,
|
|
53
|
+
provider: this.providerName,
|
|
54
|
+
baseURL: config.baseURL,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
getProviderName() {
|
|
58
|
+
return "litellm";
|
|
59
|
+
}
|
|
60
|
+
getDefaultModel() {
|
|
61
|
+
return getDefaultLiteLLMModel();
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Returns the Vercel AI SDK model instance for LiteLLM
|
|
65
|
+
*/
|
|
66
|
+
getAISDKModel() {
|
|
67
|
+
return this.model;
|
|
68
|
+
}
|
|
69
|
+
handleProviderError(error) {
|
|
70
|
+
if (error instanceof TimeoutError) {
|
|
71
|
+
return new Error(`LiteLLM request timed out: ${error.message}`);
|
|
72
|
+
}
|
|
73
|
+
// Check for timeout by error name and message as fallback
|
|
74
|
+
const errorRecord = error;
|
|
75
|
+
if (errorRecord?.name === "TimeoutError" ||
|
|
76
|
+
(typeof errorRecord?.message === "string" &&
|
|
77
|
+
errorRecord.message.includes("Timeout"))) {
|
|
78
|
+
return new Error(`LiteLLM request timed out: ${errorRecord?.message || "Unknown timeout"}`);
|
|
79
|
+
}
|
|
80
|
+
if (typeof errorRecord?.message === "string") {
|
|
81
|
+
if (errorRecord.message.includes("ECONNREFUSED") ||
|
|
82
|
+
errorRecord.message.includes("Failed to fetch")) {
|
|
83
|
+
return new Error("LiteLLM proxy server not available. Please start the LiteLLM proxy server at " +
|
|
84
|
+
`${process.env.LITELLM_BASE_URL || "http://localhost:4000"}`);
|
|
85
|
+
}
|
|
86
|
+
if (errorRecord.message.includes("API_KEY_INVALID") ||
|
|
87
|
+
errorRecord.message.includes("Invalid API key")) {
|
|
88
|
+
return new Error("Invalid LiteLLM configuration. Please check your LITELLM_API_KEY environment variable.");
|
|
89
|
+
}
|
|
90
|
+
if (errorRecord.message.includes("rate limit")) {
|
|
91
|
+
return new Error("LiteLLM rate limit exceeded. Please try again later.");
|
|
92
|
+
}
|
|
93
|
+
if (errorRecord.message.includes("model") &&
|
|
94
|
+
errorRecord.message.includes("not found")) {
|
|
95
|
+
return new Error(`Model '${this.modelName}' not available in LiteLLM proxy. ` +
|
|
96
|
+
"Please check your LiteLLM configuration and ensure the model is configured.");
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return new Error(`LiteLLM error: ${errorRecord?.message || "Unknown error"}`);
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* LiteLLM supports tools for compatible models
|
|
103
|
+
*/
|
|
104
|
+
supportsTools() {
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Provider-specific streaming implementation
|
|
109
|
+
* Note: This is only used when tools are disabled
|
|
110
|
+
*/
|
|
111
|
+
async executeStream(options, analysisSchema) {
|
|
112
|
+
this.validateStreamOptions(options);
|
|
113
|
+
const startTime = Date.now();
|
|
114
|
+
const timeout = this.getTimeout(options);
|
|
115
|
+
const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
|
|
116
|
+
try {
|
|
117
|
+
const result = await streamText({
|
|
118
|
+
model: this.model,
|
|
119
|
+
prompt: options.input.text,
|
|
120
|
+
system: options.systemPrompt,
|
|
121
|
+
temperature: options.temperature,
|
|
122
|
+
maxTokens: options.maxTokens || DEFAULT_MAX_TOKENS,
|
|
123
|
+
tools: options.tools,
|
|
124
|
+
toolChoice: "auto",
|
|
125
|
+
abortSignal: timeoutController?.controller.signal,
|
|
126
|
+
});
|
|
127
|
+
timeoutController?.cleanup();
|
|
128
|
+
// Transform stream to match StreamResult interface
|
|
129
|
+
const transformedStream = async function* () {
|
|
130
|
+
for await (const chunk of result.textStream) {
|
|
131
|
+
yield { content: chunk };
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
// Create analytics promise that resolves after stream completion
|
|
135
|
+
const analyticsPromise = streamAnalyticsCollector.createAnalytics(this.providerName, this.modelName, result, Date.now() - startTime, {
|
|
136
|
+
requestId: `litellm-stream-${Date.now()}`,
|
|
137
|
+
streamingMode: true,
|
|
138
|
+
});
|
|
139
|
+
return {
|
|
140
|
+
stream: transformedStream(),
|
|
141
|
+
provider: this.providerName,
|
|
142
|
+
model: this.modelName,
|
|
143
|
+
analytics: analyticsPromise,
|
|
144
|
+
metadata: {
|
|
145
|
+
startTime,
|
|
146
|
+
streamId: `litellm-${Date.now()}`,
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
timeoutController?.cleanup();
|
|
152
|
+
throw this.handleProviderError(error);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Get available models from LiteLLM proxy server
|
|
157
|
+
*
|
|
158
|
+
* TODO: Implement dynamic fetching from LiteLLM's /v1/models endpoint.
|
|
159
|
+
* Currently returns a hardcoded list of commonly available models.
|
|
160
|
+
*
|
|
161
|
+
* Implementation would involve:
|
|
162
|
+
* 1. Fetch from `${baseURL}/v1/models`
|
|
163
|
+
* 2. Parse response to extract model IDs
|
|
164
|
+
* 3. Handle network errors gracefully
|
|
165
|
+
* 4. Cache results to avoid repeated API calls
|
|
166
|
+
*/
|
|
167
|
+
async getAvailableModels() {
|
|
168
|
+
// Hardcoded list of commonly available models
|
|
169
|
+
// TODO: Replace with dynamic fetch from LiteLLM proxy /v1/models endpoint
|
|
170
|
+
return [
|
|
171
|
+
"openai/gpt-4o",
|
|
172
|
+
"openai/gpt-4o-mini",
|
|
173
|
+
"anthropic/claude-3-5-sonnet",
|
|
174
|
+
"anthropic/claude-3-haiku",
|
|
175
|
+
"google/gemini-2.0-flash",
|
|
176
|
+
"mistral/mistral-large",
|
|
177
|
+
"mistral/mistral-medium",
|
|
178
|
+
];
|
|
179
|
+
}
|
|
180
|
+
// ===================
|
|
181
|
+
// PRIVATE VALIDATION METHODS
|
|
182
|
+
// ===================
|
|
183
|
+
validateStreamOptions(options) {
|
|
184
|
+
if (!options.input?.text || options.input.text.trim().length === 0) {
|
|
185
|
+
throw new Error("Input text is required and cannot be empty");
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
@@ -143,6 +143,10 @@ export function hasProviderEnvVars(provider) {
|
|
|
143
143
|
case "mistral-ai":
|
|
144
144
|
case "mistralai":
|
|
145
145
|
return !!process.env.MISTRAL_API_KEY;
|
|
146
|
+
case "litellm":
|
|
147
|
+
// LiteLLM requires a proxy server, which can be checked for availability
|
|
148
|
+
// Default base URL is assumed, or can be configured via environment
|
|
149
|
+
return true; // LiteLLM proxy availability will be checked during usage
|
|
146
150
|
default:
|
|
147
151
|
return false;
|
|
148
152
|
}
|
package/dist/neurolink.js
CHANGED
|
@@ -701,6 +701,7 @@ export class NeuroLink {
|
|
|
701
701
|
"huggingface",
|
|
702
702
|
"ollama",
|
|
703
703
|
"mistral",
|
|
704
|
+
"litellm",
|
|
704
705
|
];
|
|
705
706
|
// 🚀 PERFORMANCE FIX: Test providers with controlled concurrency
|
|
706
707
|
// This reduces total time from 16s (sequential) to ~3s (parallel) while preventing resource exhaustion
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import type { ZodType, ZodTypeDef } from "zod";
|
|
2
|
+
import { type Schema, type LanguageModelV1 } from "ai";
|
|
3
|
+
import type { AIProviderName } from "../core/types.js";
|
|
4
|
+
import type { StreamOptions, StreamResult } from "../types/streamTypes.js";
|
|
5
|
+
import { BaseProvider } from "../core/baseProvider.js";
|
|
6
|
+
/**
|
|
7
|
+
* LiteLLM Provider - BaseProvider Implementation
|
|
8
|
+
* Provides access to 100+ models via LiteLLM proxy server
|
|
9
|
+
*/
|
|
10
|
+
export declare class LiteLLMProvider extends BaseProvider {
|
|
11
|
+
private model;
|
|
12
|
+
constructor(modelName?: string, sdk?: unknown);
|
|
13
|
+
protected getProviderName(): AIProviderName;
|
|
14
|
+
protected getDefaultModel(): string;
|
|
15
|
+
/**
|
|
16
|
+
* Returns the Vercel AI SDK model instance for LiteLLM
|
|
17
|
+
*/
|
|
18
|
+
protected getAISDKModel(): LanguageModelV1;
|
|
19
|
+
protected handleProviderError(error: unknown): Error;
|
|
20
|
+
/**
|
|
21
|
+
* LiteLLM supports tools for compatible models
|
|
22
|
+
*/
|
|
23
|
+
supportsTools(): boolean;
|
|
24
|
+
/**
|
|
25
|
+
* Provider-specific streaming implementation
|
|
26
|
+
* Note: This is only used when tools are disabled
|
|
27
|
+
*/
|
|
28
|
+
protected executeStream(options: StreamOptions, analysisSchema?: ZodType<unknown, ZodTypeDef, unknown> | Schema<unknown>): Promise<StreamResult>;
|
|
29
|
+
/**
|
|
30
|
+
* Get available models from LiteLLM proxy server
|
|
31
|
+
*
|
|
32
|
+
* TODO: Implement dynamic fetching from LiteLLM's /v1/models endpoint.
|
|
33
|
+
* Currently returns a hardcoded list of commonly available models.
|
|
34
|
+
*
|
|
35
|
+
* Implementation would involve:
|
|
36
|
+
* 1. Fetch from `${baseURL}/v1/models`
|
|
37
|
+
* 2. Parse response to extract model IDs
|
|
38
|
+
* 3. Handle network errors gracefully
|
|
39
|
+
* 4. Cache results to avoid repeated API calls
|
|
40
|
+
*/
|
|
41
|
+
getAvailableModels(): Promise<string[]>;
|
|
42
|
+
private validateStreamOptions;
|
|
43
|
+
}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { openai, createOpenAI } from "@ai-sdk/openai";
|
|
2
|
+
import { streamText, Output } from "ai";
|
|
3
|
+
import { BaseProvider } from "../core/baseProvider.js";
|
|
4
|
+
import { logger } from "../utils/logger.js";
|
|
5
|
+
import { createTimeoutController, TimeoutError, getDefaultTimeout, } from "../utils/timeout.js";
|
|
6
|
+
import { DEFAULT_MAX_TOKENS } from "../core/constants.js";
|
|
7
|
+
import { validateApiKey, getProviderModel } from "../utils/providerConfig.js";
|
|
8
|
+
import { streamAnalyticsCollector } from "../core/streamAnalytics.js";
|
|
9
|
+
// Configuration helpers
|
|
10
|
+
const getLiteLLMConfig = () => {
|
|
11
|
+
return {
|
|
12
|
+
baseURL: process.env.LITELLM_BASE_URL || "http://localhost:4000",
|
|
13
|
+
apiKey: process.env.LITELLM_API_KEY || "sk-anything",
|
|
14
|
+
};
|
|
15
|
+
};
|
|
16
|
+
/**
|
|
17
|
+
* Returns the default model name for LiteLLM.
|
|
18
|
+
*
|
|
19
|
+
* LiteLLM uses a 'provider/model' format for model names.
|
|
20
|
+
* For example:
|
|
21
|
+
* - 'openai/gpt-4o-mini'
|
|
22
|
+
* - 'openai/gpt-3.5-turbo'
|
|
23
|
+
* - 'anthropic/claude-3-sonnet-20240229'
|
|
24
|
+
* - 'google/gemini-pro'
|
|
25
|
+
*
|
|
26
|
+
* You can override the default by setting the LITELLM_MODEL environment variable.
|
|
27
|
+
*/
|
|
28
|
+
const getDefaultLiteLLMModel = () => {
|
|
29
|
+
return getProviderModel("LITELLM_MODEL", "openai/gpt-4o-mini");
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* LiteLLM Provider - BaseProvider Implementation
|
|
33
|
+
* Provides access to 100+ models via LiteLLM proxy server
|
|
34
|
+
*/
|
|
35
|
+
export class LiteLLMProvider extends BaseProvider {
|
|
36
|
+
model;
|
|
37
|
+
constructor(modelName, sdk) {
|
|
38
|
+
super(modelName, "litellm", sdk);
|
|
39
|
+
// Initialize LiteLLM using OpenAI SDK with explicit configuration
|
|
40
|
+
const config = getLiteLLMConfig();
|
|
41
|
+
// Create OpenAI SDK instance configured for LiteLLM proxy
|
|
42
|
+
// LiteLLM acts as a proxy server that implements the OpenAI-compatible API.
|
|
43
|
+
// To communicate with LiteLLM instead of the default OpenAI endpoint, we use createOpenAI
|
|
44
|
+
// with a custom baseURL and apiKey. This ensures all requests are routed through the LiteLLM
|
|
45
|
+
// proxy, allowing access to multiple models and custom authentication.
|
|
46
|
+
const customOpenAI = createOpenAI({
|
|
47
|
+
baseURL: config.baseURL,
|
|
48
|
+
apiKey: config.apiKey,
|
|
49
|
+
});
|
|
50
|
+
this.model = customOpenAI(this.modelName || getDefaultLiteLLMModel());
|
|
51
|
+
logger.debug("LiteLLM Provider initialized", {
|
|
52
|
+
modelName: this.modelName,
|
|
53
|
+
provider: this.providerName,
|
|
54
|
+
baseURL: config.baseURL,
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
getProviderName() {
|
|
58
|
+
return "litellm";
|
|
59
|
+
}
|
|
60
|
+
getDefaultModel() {
|
|
61
|
+
return getDefaultLiteLLMModel();
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Returns the Vercel AI SDK model instance for LiteLLM
|
|
65
|
+
*/
|
|
66
|
+
getAISDKModel() {
|
|
67
|
+
return this.model;
|
|
68
|
+
}
|
|
69
|
+
handleProviderError(error) {
|
|
70
|
+
if (error instanceof TimeoutError) {
|
|
71
|
+
return new Error(`LiteLLM request timed out: ${error.message}`);
|
|
72
|
+
}
|
|
73
|
+
// Check for timeout by error name and message as fallback
|
|
74
|
+
const errorRecord = error;
|
|
75
|
+
if (errorRecord?.name === "TimeoutError" ||
|
|
76
|
+
(typeof errorRecord?.message === "string" &&
|
|
77
|
+
errorRecord.message.includes("Timeout"))) {
|
|
78
|
+
return new Error(`LiteLLM request timed out: ${errorRecord?.message || "Unknown timeout"}`);
|
|
79
|
+
}
|
|
80
|
+
if (typeof errorRecord?.message === "string") {
|
|
81
|
+
if (errorRecord.message.includes("ECONNREFUSED") ||
|
|
82
|
+
errorRecord.message.includes("Failed to fetch")) {
|
|
83
|
+
return new Error("LiteLLM proxy server not available. Please start the LiteLLM proxy server at " +
|
|
84
|
+
`${process.env.LITELLM_BASE_URL || "http://localhost:4000"}`);
|
|
85
|
+
}
|
|
86
|
+
if (errorRecord.message.includes("API_KEY_INVALID") ||
|
|
87
|
+
errorRecord.message.includes("Invalid API key")) {
|
|
88
|
+
return new Error("Invalid LiteLLM configuration. Please check your LITELLM_API_KEY environment variable.");
|
|
89
|
+
}
|
|
90
|
+
if (errorRecord.message.includes("rate limit")) {
|
|
91
|
+
return new Error("LiteLLM rate limit exceeded. Please try again later.");
|
|
92
|
+
}
|
|
93
|
+
if (errorRecord.message.includes("model") &&
|
|
94
|
+
errorRecord.message.includes("not found")) {
|
|
95
|
+
return new Error(`Model '${this.modelName}' not available in LiteLLM proxy. ` +
|
|
96
|
+
"Please check your LiteLLM configuration and ensure the model is configured.");
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return new Error(`LiteLLM error: ${errorRecord?.message || "Unknown error"}`);
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* LiteLLM supports tools for compatible models
|
|
103
|
+
*/
|
|
104
|
+
supportsTools() {
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Provider-specific streaming implementation
|
|
109
|
+
* Note: This is only used when tools are disabled
|
|
110
|
+
*/
|
|
111
|
+
async executeStream(options, analysisSchema) {
|
|
112
|
+
this.validateStreamOptions(options);
|
|
113
|
+
const startTime = Date.now();
|
|
114
|
+
const timeout = this.getTimeout(options);
|
|
115
|
+
const timeoutController = createTimeoutController(timeout, this.providerName, "stream");
|
|
116
|
+
try {
|
|
117
|
+
const result = await streamText({
|
|
118
|
+
model: this.model,
|
|
119
|
+
prompt: options.input.text,
|
|
120
|
+
system: options.systemPrompt,
|
|
121
|
+
temperature: options.temperature,
|
|
122
|
+
maxTokens: options.maxTokens || DEFAULT_MAX_TOKENS,
|
|
123
|
+
tools: options.tools,
|
|
124
|
+
toolChoice: "auto",
|
|
125
|
+
abortSignal: timeoutController?.controller.signal,
|
|
126
|
+
});
|
|
127
|
+
timeoutController?.cleanup();
|
|
128
|
+
// Transform stream to match StreamResult interface
|
|
129
|
+
const transformedStream = async function* () {
|
|
130
|
+
for await (const chunk of result.textStream) {
|
|
131
|
+
yield { content: chunk };
|
|
132
|
+
}
|
|
133
|
+
};
|
|
134
|
+
// Create analytics promise that resolves after stream completion
|
|
135
|
+
const analyticsPromise = streamAnalyticsCollector.createAnalytics(this.providerName, this.modelName, result, Date.now() - startTime, {
|
|
136
|
+
requestId: `litellm-stream-${Date.now()}`,
|
|
137
|
+
streamingMode: true,
|
|
138
|
+
});
|
|
139
|
+
return {
|
|
140
|
+
stream: transformedStream(),
|
|
141
|
+
provider: this.providerName,
|
|
142
|
+
model: this.modelName,
|
|
143
|
+
analytics: analyticsPromise,
|
|
144
|
+
metadata: {
|
|
145
|
+
startTime,
|
|
146
|
+
streamId: `litellm-${Date.now()}`,
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
catch (error) {
|
|
151
|
+
timeoutController?.cleanup();
|
|
152
|
+
throw this.handleProviderError(error);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Get available models from LiteLLM proxy server
|
|
157
|
+
*
|
|
158
|
+
* TODO: Implement dynamic fetching from LiteLLM's /v1/models endpoint.
|
|
159
|
+
* Currently returns a hardcoded list of commonly available models.
|
|
160
|
+
*
|
|
161
|
+
* Implementation would involve:
|
|
162
|
+
* 1. Fetch from `${baseURL}/v1/models`
|
|
163
|
+
* 2. Parse response to extract model IDs
|
|
164
|
+
* 3. Handle network errors gracefully
|
|
165
|
+
* 4. Cache results to avoid repeated API calls
|
|
166
|
+
*/
|
|
167
|
+
async getAvailableModels() {
|
|
168
|
+
// Hardcoded list of commonly available models
|
|
169
|
+
// TODO: Replace with dynamic fetch from LiteLLM proxy /v1/models endpoint
|
|
170
|
+
return [
|
|
171
|
+
"openai/gpt-4o",
|
|
172
|
+
"openai/gpt-4o-mini",
|
|
173
|
+
"anthropic/claude-3-5-sonnet",
|
|
174
|
+
"anthropic/claude-3-haiku",
|
|
175
|
+
"google/gemini-2.0-flash",
|
|
176
|
+
"mistral/mistral-large",
|
|
177
|
+
"mistral/mistral-medium",
|
|
178
|
+
];
|
|
179
|
+
}
|
|
180
|
+
// ===================
|
|
181
|
+
// PRIVATE VALIDATION METHODS
|
|
182
|
+
// ===================
|
|
183
|
+
validateStreamOptions(options) {
|
|
184
|
+
if (!options.input?.text || options.input.text.trim().length === 0) {
|
|
185
|
+
throw new Error("Input text is required and cannot be empty");
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
@@ -143,6 +143,10 @@ export function hasProviderEnvVars(provider) {
|
|
|
143
143
|
case "mistral-ai":
|
|
144
144
|
case "mistralai":
|
|
145
145
|
return !!process.env.MISTRAL_API_KEY;
|
|
146
|
+
case "litellm":
|
|
147
|
+
// LiteLLM requires a proxy server, which can be checked for availability
|
|
148
|
+
// Default base URL is assumed, or can be configured via environment
|
|
149
|
+
return true; // LiteLLM proxy availability will be checked during usage
|
|
146
150
|
default:
|
|
147
151
|
return false;
|
|
148
152
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@juspay/neurolink",
|
|
3
|
-
"version": "7.
|
|
3
|
+
"version": "7.5.0",
|
|
4
4
|
"description": "Universal AI Development Platform with working MCP integration, multi-provider support, and professional CLI. Built-in tools operational, 58+ external MCP servers discoverable. Connect to filesystem, GitHub, database operations, and more. Build, test, and deploy AI applications with 9 major providers: OpenAI, Anthropic, Google AI, AWS Bedrock, Azure, Hugging Face, Ollama, and Mistral AI.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Juspay Technologies",
|
|
@@ -252,5 +252,9 @@
|
|
|
252
252
|
"darwin",
|
|
253
253
|
"linux",
|
|
254
254
|
"win32"
|
|
255
|
-
]
|
|
255
|
+
],
|
|
256
|
+
"prettier": {
|
|
257
|
+
"tabWidth": 2,
|
|
258
|
+
"useTabs": false
|
|
259
|
+
}
|
|
256
260
|
}
|