llmjs2 1.1.1 → 1.3.0

package/router.js

@@ -0,0 +1,248 @@
+const OpenAIProvider = require('./providers/openai');
+const OllamaProvider = require('./providers/ollama');
+const OpenRouterProvider = require('./providers/openrouter');
+
+class Router {
+  constructor(modelList, strategy = 'default') {
+    this.modelList = this.normalizeModelList(modelList);
+    this.strategy = strategy;
+    this.providers = {
+      openai: new OpenAIProvider(),
+      ollama: new OllamaProvider(),
+      openrouter: new OpenRouterProvider()
+    };
+    this.guardrails = [];
+    this.sequentialIndex = 0;
+
+    // Group models by model_name for load balancing
+    this.modelsByName = {};
+    this.modelList.forEach(model => {
+      if (!this.modelsByName[model.model_name]) {
+        this.modelsByName[model.model_name] = [];
+      }
+      this.modelsByName[model.model_name].push(model);
+    });
+  }
+
+  normalizeModelList(modelList) {
+    return modelList.map(model => ({
+      model_name: model.model_name,
+      llm_params: {
+        model: model.llm_params.model,
+        api_key: this.resolveApiKey(model.llm_params.api_key),
+        api_base: model.llm_params.api_base
+      }
+    }));
+  }
+
+  resolveApiKey(apiKey) {
+    if (typeof apiKey === 'string' && apiKey.startsWith('os.environ/')) {
+      const envVar = apiKey.replace('os.environ/', '');
+      return process.env[envVar] || apiKey;
+    }
+    return apiKey;
+  }
+
+  setGuardrails(guardrails) {
+    this.guardrails = guardrails || [];
+  }
+
+  selectModel(modelName) {
+    if (!modelName) {
+      // Auto-routing based on strategy
+      return this.autoSelectModel();
+    }
+
+    // Direct model selection with load balancing
+    const availableModels = this.modelsByName[modelName];
+    if (!availableModels || availableModels.length === 0) {
+      throw new Error(`Model '${modelName}' not found in router configuration`);
+    }
+
+    if (availableModels.length === 1) {
+      return availableModels[0];
+    }
+
+    // Load balancing for multiple models with same name
+    const randomIndex = Math.floor(Math.random() * availableModels.length);
+    return availableModels[randomIndex];
+  }
+
+  autoSelectModel() {
+    const allModels = this.modelList;
+
+    switch (this.strategy) {
+      case 'random':
+        const randomIndex = Math.floor(Math.random() * allModels.length);
+        return allModels[randomIndex];
+
+      case 'sequential':
+        const model = allModels[this.sequentialIndex];
+        this.sequentialIndex = (this.sequentialIndex + 1) % allModels.length;
+        return model;
+
+      case 'default':
+      default:
+        // Use first available model
+        return allModels[0];
+    }
+  }
+
+  async applyPreGuardrails(processId, input) {
+    let currentInput = input;
+
+    for (const guardrail of this.guardrails) {
+      if (guardrail.mode === 'pre_call') {
+        try {
+          const result = await this.executeGuardrail(guardrail, processId, currentInput);
+          if (result === null || result === undefined) {
+            throw new Error(`Guardrail '${guardrail.name}' returned null/undefined`);
+          }
+          currentInput = result;
+        } catch (error) {
+          throw new Error(`Pre-call guardrail '${guardrail.name}' failed: ${error.message}`);
+        }
+      }
+    }
+
+    return currentInput;
+  }
+
+  async applyPostGuardrails(processId, result) {
+    let currentResult = result;
+
+    for (const guardrail of this.guardrails) {
+      if (guardrail.mode === 'post_call') {
+        try {
+          const processed = await this.executeGuardrail(guardrail, processId, currentResult);
+          if (processed === null || processed === undefined) {
+            throw new Error(`Guardrail '${guardrail.name}' returned null/undefined`);
+          }
+          currentResult = processed;
+        } catch (error) {
+          throw new Error(`Post-call guardrail '${guardrail.name}' failed: ${error.message}`);
+        }
+      }
+    }
+
+    return currentResult;
+  }
+
+  async executeGuardrail(guardrail, processId, data) {
+    if (typeof guardrail.code === 'string') {
+      // Execute string code as function
+      const func = new Function('processId', 'data', `return (${guardrail.code})(processId, data)`);
+      return await func(processId, data);
+    } else if (typeof guardrail.code === 'function') {
+      return await guardrail.code(processId, data);
+    } else {
+      throw new Error(`Invalid guardrail code for '${guardrail.name}'`);
+    }
+  }
+
+  async completion(options) {
+    const processId = this.generateProcessId();
+    let selectedModel;
+
+    try {
+      // Select model
+      selectedModel = this.selectModel(options.model);
+      console.log(`[${processId}] Selected model: ${selectedModel.llm_params.model}`);
+
+      // Prepare input for guardrails
+      const input = {
+        model: selectedModel.llm_params.model,
+        messages: options.messages || [],
+        temperature: options.temperature,
+        maxTokens: options.max_tokens || options.maxTokens,
+        topP: options.top_p || options.topP,
+        frequencyPenalty: options.frequency_penalty || options.frequencyPenalty,
+        presencePenalty: options.presence_penalty || options.presencePenalty,
+        stop: options.stop,
+        tools: options.tools,
+        toolChoice: options.tool_choice || options.toolChoice
+      };
+
+      // Apply pre-call guardrails
+      const processedInput = await this.applyPreGuardrails(processId, input);
+
+      // Create completion using selected model
+      const result = await this.callProvider(selectedModel, processedInput);
+
+      // Apply post-call guardrails
+      const finalResult = await this.applyPostGuardrails(processId, result);
+
+      // Return result with selected model information
+      return {
+        result: finalResult,
+        selectedModel: selectedModel.llm_params.model, // Full model name with provider prefix
+        selectedModelName: selectedModel.model_name // User-friendly model name
+      };
+
+    } catch (error) {
+      console.error(`[${processId}] Router error:`, error.message);
+      throw error;
+    }
+  }
+
+  async callProvider(modelConfig, input) {
+    const firstSlashIndex = modelConfig.llm_params.model.indexOf('/');
+    const providerName = firstSlashIndex !== -1 ? modelConfig.llm_params.model.substring(0, firstSlashIndex) : '';
+    const actualModel = firstSlashIndex !== -1 ? modelConfig.llm_params.model.substring(firstSlashIndex + 1) : modelConfig.llm_params.model;
+    const provider = this.providers[providerName];
+
+    if (!provider) {
+      throw new Error(`Unknown provider: ${providerName}`);
+    }
+
+    // Set the API key for this request (only if provided, otherwise use provider's default)
+    if (modelConfig.llm_params.api_key !== undefined) {
+      provider.apiKey = modelConfig.llm_params.api_key;
+    }
+
+    // Call the provider directly with just the actual model name (without provider prefix)
+    return await provider.createCompletion(input.messages, {
+      model: actualModel,
+      temperature: input.temperature,
+      maxTokens: input.maxTokens,
+      topP: input.topP,
+      frequencyPenalty: input.frequencyPenalty,
+      presencePenalty: input.presencePenalty,
+      stop: input.stop,
+      tools: input.tools,
+      toolChoice: input.toolChoice
+    });
+  }
+
+  generateProcessId() {
+    return `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+  }
+
+  // Method to add guardrails dynamically
+  addGuardrail(guardrail) {
+    this.guardrails.push(guardrail);
+  }
+
+  // Method to get available models
+  getAvailableModels() {
+    return Object.keys(this.modelsByName);
+  }
+
+  // Method to get model count for load balancing info
+  getModelStats() {
+    const stats = {};
+    Object.keys(this.modelsByName).forEach(modelName => {
+      stats[modelName] = this.modelsByName[modelName].length;
+    });
+    return stats;
+  }
+}
+
+function router(modelList, strategy = 'default') {
+  return new Router(modelList, strategy);
+}
+
+module.exports = {
+  Router,
+  router
+};

package/server.js

@@ -0,0 +1,186 @@
+const http = require('http');
+const url = require('url');
+
+class Server {
+  constructor(options = {}) {
+    this.port = options.port || process.env.PORT || 3000;
+    this.host = options.host || process.env.HOST || 'localhost';
+    this.router = options.router;
+    this.middlewares = [];
+  }
+
+  use(middleware) {
+    if (middleware && typeof middleware.completion === 'function') {
+      // It's a router
+      this.router = middleware;
+    } else if (typeof middleware === 'function') {
+      // It's middleware
+      this.middlewares.push(middleware);
+    }
+  }
+
+  async handleRequest(req, res) {
+    try {
+      // Set CORS headers
+      res.setHeader('Access-Control-Allow-Origin', '*');
+      res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+      res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
+
+      if (req.method === 'OPTIONS') {
+        res.writeHead(200);
+        res.end();
+        return;
+      }
+
+      const parsedUrl = url.parse(req.url, true);
+      const path = parsedUrl.pathname;
+
+      // Only handle chat completions endpoint
+      if (path !== '/v1/chat/completions' || req.method !== 'POST') {
+        this.sendError(res, 404, 'Not Found', 'Endpoint not found');
+        return;
+      }
+
+      // Parse request body
+      const body = await this.parseBody(req);
+
+      // Validate request
+      const validation = this.validateChatRequest(body);
+      if (!validation.valid) {
+        this.sendError(res, 400, 'Bad Request', validation.error);
+        return;
+      }
+
+      console.log(`[${new Date().toISOString()}] ${req.method} ${req.url}`);
+      console.log('Headers:', JSON.stringify(req.headers));
+      console.log('Body parsing completed successfully');
+
+      // Process the completion request
+      const result = await this.processCompletion(body);
+
+      // Send successful response
+      this.sendSuccess(res, result);
+
+    } catch (error) {
+      console.error('Server error:', error);
+      this.sendError(res, 500, 'Internal Server Error', error.message);
+    }
+  }
+
+  parseBody(req) {
+    return new Promise((resolve, reject) => {
+      let body = '';
+      req.on('data', chunk => {
+        body += chunk.toString();
+      });
+      req.on('end', () => {
+        try {
+          resolve(JSON.parse(body));
+        } catch (error) {
+          reject(new Error('Invalid JSON in request body'));
+        }
+      });
+      req.on('error', reject);
+    });
+  }
+
+  validateChatRequest(body) {
+    if (!body) {
+      return { valid: false, error: 'Request body is required' };
+    }
+
+    if (!body.messages || !Array.isArray(body.messages)) {
+      return { valid: false, error: 'messages array is required' };
+    }
+
+    if (body.messages.length === 0) {
+      return { valid: false, error: 'messages array cannot be empty' };
+    }
+
+    for (const message of body.messages) {
+      if (!message.role || !['system', 'user', 'assistant'].includes(message.role)) {
+        return { valid: false, error: 'Each message must have a valid role (system, user, or assistant)' };
+      }
+      if (typeof message.content !== 'string') {
+        return { valid: false, error: 'Each message must have string content' };
+      }
+    }
+
+    return { valid: true };
+  }
+
+  async processCompletion(body) {
+    if (!this.router) {
+      throw new Error('No router configured. Use app.use(router) to add a router.');
+    }
+
+    console.log(`Starting completion with model: ${body.model || 'auto-selected'}`);
+
+    const routerResponse = await this.router.completion(body);
+    const { result, selectedModel, selectedModelName } = routerResponse;
+
+    // Return format with metadata plus message array
+    const assistantMessage = {
+      role: 'assistant',
+      content: typeof result === 'string' ? result : result.content || ''
+    };
+
+    // Include original messages plus assistant response
+    const messages = [...(body.messages || []), assistantMessage];
+
+    return {
+      id: `chatcmpl-${Date.now()}`,
+      object: 'chat.completion',
+      created: Math.floor(Date.now() / 1000),
+      model: body.model || selectedModel, // Use selected model if auto-selected
+      messages: messages
+    };
+  }
+
+  sendSuccess(res, data) {
+    const response = JSON.stringify(data);
+    res.writeHead(200, {
+      'Content-Type': 'application/json',
+      'Content-Length': Buffer.byteLength(response)
+    });
+    res.end(response);
+  }
+
+  sendError(res, statusCode, type, message) {
+    const error = {
+      error: {
+        message: message,
+        type: type.toLowerCase().replace(/\s+/g, '_')
+      }
+    };
+    const response = JSON.stringify(error);
+    res.writeHead(statusCode, {
+      'Content-Type': 'application/json',
+      'Content-Length': Buffer.byteLength(response)
+    });
+    res.end(response);
+  }
+
+  listen(port, host, callback) {
+    const actualPort = port || this.port;
+    const actualHost = host || this.host;
+
+    const server = http.createServer((req, res) => this.handleRequest(req, res));
+
+    server.listen(actualPort, actualHost, () => {
+      console.log(`🚀 llmjs2 server running on http://${actualHost}:${actualPort}`);
+      if (callback) callback();
+    });
+
+    return server;
+  }
+}
+
+function createApp(options = {}) {
+  return new Server(options);
+}
+
+module.exports = {
+  Server,
+  app: createApp()
+};

package/test.js

@@ -0,0 +1,246 @@
+// Import all modules
+const { completion: llmCompletion, LLMJS2: LLMJS2Class, router, app } = require('./index');
+
+// Basic validation tests
+async function runTests() {
+  console.log('Running LLMJS2 comprehensive tests...\n');
+
+  let testCount = 0;
+  let passedCount = 0;
+
+  // Test 1: Input validation
+  testCount++;
+  try {
+    await llmCompletion('');
+    console.log('❌ Test 1 failed: Should reject empty prompt');
+  } catch (error) {
+    console.log('✅ Test 1 passed: Empty prompt rejected');
+    passedCount++;
+  }
+
+  // Test 2: Invalid input type
+  testCount++;
+  try {
+    await llmCompletion(123);
+    console.log('❌ Test 2 failed: Should reject non-string/non-object input');
+  } catch (error) {
+    console.log('✅ Test 2 passed: Invalid input type rejected');
+    passedCount++;
+  }
+
+  // Test 3: Invalid messages array
+  testCount++;
+  try {
+    await llmCompletion({ messages: [] });
+    console.log('❌ Test 3 failed: Should reject empty messages array');
+  } catch (error) {
+    console.log('✅ Test 3 passed: Empty messages array rejected');
+    passedCount++;
+  }
+
+  // Test 4: Invalid message format
+  testCount++;
+  try {
+    await llmCompletion({ messages: [{ content: 'test' }] });
+    console.log('❌ Test 4 failed: Should reject message without role');
+  } catch (error) {
+    console.log('✅ Test 4 passed: Message without role rejected');
+    passedCount++;
+  }
+
+  // Test 5: Invalid provider
+  testCount++;
+  try {
+    const llm = new LLMJS2();
+    llm.getProvider('unknown/test');
+    console.log('❌ Test 5 failed: Should reject unknown provider');
+  } catch (error) {
+    console.log('✅ Test 5 passed: Unknown provider rejected -', error.message);
+    passedCount++;
+  }
+
+  // Test 6: LLMJS2 class instantiation
+  testCount++;
+  try {
+    const llm = new LLMJS2Class();
+    console.log('✅ Test 6 passed: LLMJS2 class instantiated successfully');
+    passedCount++;
+  } catch (error) {
+    console.log('❌ Test 6 failed:', error.message);
+  }
+
+  // Test 7: Provider availability check (without API keys)
+  testCount++;
+  try {
+    // Temporarily clear API keys for this test
+    const originalKeys = {
+      openai: process.env.OPENAI_API_KEY,
+      ollama: process.env.OLLAMA_API_KEY,
+      openrouter: process.env.OPEN_ROUTER_API_KEY
+    };
+
+    delete process.env.OPENAI_API_KEY;
+    delete process.env.OLLAMA_API_KEY;
+    delete process.env.OPEN_ROUTER_API_KEY;
+
+    const llm = new LLMJS2Class();
+    const available = llm.getAvailableProviders();
+
+    // Restore original keys
+    process.env.OPENAI_API_KEY = originalKeys.openai;
+    process.env.OLLAMA_API_KEY = originalKeys.ollama;
+    process.env.OPEN_ROUTER_API_KEY = originalKeys.openrouter;
+
+    if (available.length === 0) {
+      console.log('✅ Test 7 passed: No providers available without API keys');
+      passedCount++;
+    } else {
+      console.log('❌ Test 7 failed: Should have no providers without API keys');
+    }
+  } catch (error) {
+    console.log('❌ Test 7 failed:', error.message);
+  }
+
+  // Test 8: Model parsing
+  testCount++;
+  try {
+    const llm = new LLMJS2Class();
+    const { provider, model } = llm.parseModel('openai/gpt-3.5-turbo');
+    if (provider === 'openai' && model === 'gpt-3.5-turbo') {
+      console.log('✅ Test 8 passed: Model parsing works correctly');
+      passedCount++;
+    } else {
+      console.log('❌ Test 8 failed: Model parsing incorrect');
+    }
+  } catch (error) {
+    console.log('❌ Test 8 failed:', error.message);
+  }
+
+  // Test 9: Router creation
+  testCount++;
+  try {
+    const modelList = [
+      {
+        model_name: 'test',
+        llm_params: {
+          model: 'ollama/test',
+          api_key: 'test-key'
+        }
+      }
+    ];
+    const route = router(modelList, 'random');
+    console.log('✅ Test 9 passed: Router created successfully');
+    passedCount++;
+  } catch (error) {
+    console.log('❌ Test 9 failed:', error.message);
+  }
+
+  // Test 10: Router model selection
+  testCount++;
+  try {
+    const modelList = [
+      {
+        model_name: 'test',
+        llm_params: {
+          model: 'ollama/test',
+          api_key: 'test-key'
+        }
+      }
+    ];
+    const route = router(modelList);
+    const selectedModel = route.selectModel('test');
+    if (selectedModel.llm_params.model === 'ollama/test') {
+      console.log('✅ Test 10 passed: Router model selection works');
+      passedCount++;
+    } else {
+      console.log('❌ Test 10 failed: Incorrect model selected');
+    }
+  } catch (error) {
+    console.log('❌ Test 10 failed:', error.message);
+  }
+
+  // Test 11: Router auto-selection
+  testCount++;
+  try {
+    const modelList = [
+      {
+        model_name: 'test1',
+        llm_params: {
+          model: 'ollama/test1',
+          api_key: 'test-key'
+        }
+      },
+      {
+        model_name: 'test2',
+        llm_params: {
+          model: 'ollama/test2',
+          api_key: 'test-key'
+        }
+      }
+    ];
+    const route = router(modelList, 'random');
+    const selectedModel = route.autoSelectModel();
+    if (selectedModel && selectedModel.llm_params.model.startsWith('ollama/')) {
+      console.log('✅ Test 11 passed: Router auto-selection works');
+      passedCount++;
+    } else {
+      console.log('❌ Test 11 failed: Auto-selection failed');
+    }
+  } catch (error) {
+    console.log('❌ Test 11 failed:', error.message);
+  }
+
+  // Test 12: Guardrails execution
+  testCount++;
+  try {
+    const modelList = [
+      {
+        model_name: 'test',
+        llm_params: {
+          model: 'ollama/test',
+          api_key: 'test-key'
+        }
+      }
+    ];
+    const route = router(modelList);
+
+    route.setGuardrails([
+      {
+        name: 'test_guardrail',
+        mode: 'pre_call',
+        code: (processId, input) => {
+          return { ...input, messages: [...input.messages, { role: 'system', content: 'Test' }] };
+        }
+      }
+    ]);
+
+    console.log('✅ Test 12 passed: Guardrails set successfully');
+    passedCount++;
+  } catch (error) {
+    console.log('❌ Test 12 failed:', error.message);
+  }
+
+  // Test 13: Server app creation
+  testCount++;
+  try {
+    const testApp = app; // Already created
+    if (testApp && typeof testApp.use === 'function') {
+      console.log('✅ Test 13 passed: Server app created successfully');
+      passedCount++;
+    } else {
+      console.log('❌ Test 13 failed: Server app creation failed');
+    }
+  } catch (error) {
+    console.log('❌ Test 13 failed:', error.message);
+  }
+
+  console.log(`\nTest Results: ${passedCount}/${testCount} tests passed`);
+  if (passedCount === testCount) {
+    console.log('🎉 All tests passed!');
+  } else {
+    console.log('⚠️  Some tests failed. Please review the implementation.');
+  }
+}
+
+// Run tests
+runTests().catch(console.error);