llmjs2 1.1.0 → 1.3.0

package/router.js

@@ -0,0 +1,248 @@
+const OpenAIProvider = require('./providers/openai');
+const OllamaProvider = require('./providers/ollama');
+const OpenRouterProvider = require('./providers/openrouter');
+
+class Router {
+  constructor(modelList, strategy = 'default') {
+    this.modelList = this.normalizeModelList(modelList);
+    this.strategy = strategy;
+    this.providers = {
+      openai: new OpenAIProvider(),
+      ollama: new OllamaProvider(),
+      openrouter: new OpenRouterProvider()
+    };
+    this.guardrails = [];
+    this.sequentialIndex = 0;
+
+    // Group models by model_name for load balancing
+    this.modelsByName = {};
+    this.modelList.forEach(model => {
+      if (!this.modelsByName[model.model_name]) {
+        this.modelsByName[model.model_name] = [];
+      }
+      this.modelsByName[model.model_name].push(model);
+    });
+  }
+
+  normalizeModelList(modelList) {
+    return modelList.map(model => ({
+      model_name: model.model_name,
+      llm_params: {
+        model: model.llm_params.model,
+        api_key: this.resolveApiKey(model.llm_params.api_key),
+        api_base: model.llm_params.api_base
+      }
+    }));
+  }
+
+  resolveApiKey(apiKey) {
+    if (typeof apiKey === 'string' && apiKey.startsWith('os.environ/')) {
+      const envVar = apiKey.replace('os.environ/', '');
+      return process.env[envVar] || apiKey;
+    }
+    return apiKey;
+  }
+
+  setGuardrails(guardrails) {
+    this.guardrails = guardrails || [];
+  }
+
+  selectModel(modelName) {
+    if (!modelName) {
+      // Auto-routing based on strategy
+      return this.autoSelectModel();
+    }
+
+    // Direct model selection with load balancing
+    const availableModels = this.modelsByName[modelName];
+    if (!availableModels || availableModels.length === 0) {
+      throw new Error(`Model '${modelName}' not found in router configuration`);
+    }
+
+    if (availableModels.length === 1) {
+      return availableModels[0];
+    }
+
+    // Load balancing for multiple models with same name
+    const randomIndex = Math.floor(Math.random() * availableModels.length);
+    return availableModels[randomIndex];
+  }
+
+  autoSelectModel() {
+    const allModels = this.modelList;
+
+    switch (this.strategy) {
+      case 'random':
+        const randomIndex = Math.floor(Math.random() * allModels.length);
+        return allModels[randomIndex];
+
+      case 'sequential':
+        const model = allModels[this.sequentialIndex];
+        this.sequentialIndex = (this.sequentialIndex + 1) % allModels.length;
+        return model;
+
+      case 'default':
+      default:
+        // Use first available model
+        return allModels[0];
+    }
+  }
+
+  async applyPreGuardrails(processId, input) {
+    let currentInput = input;
+
+    for (const guardrail of this.guardrails) {
+      if (guardrail.mode === 'pre_call') {
+        try {
+          const result = await this.executeGuardrail(guardrail, processId, currentInput);
+          if (result === null || result === undefined) {
+            throw new Error(`Guardrail '${guardrail.name}' returned null/undefined`);
+          }
+          currentInput = result;
+        } catch (error) {
+          throw new Error(`Pre-call guardrail '${guardrail.name}' failed: ${error.message}`);
+        }
+      }
+    }
+
+    return currentInput;
+  }
+
+  async applyPostGuardrails(processId, result) {
+    let currentResult = result;
+
+    for (const guardrail of this.guardrails) {
+      if (guardrail.mode === 'post_call') {
+        try {
+          const processed = await this.executeGuardrail(guardrail, processId, currentResult);
+          if (processed === null || processed === undefined) {
+            throw new Error(`Guardrail '${guardrail.name}' returned null/undefined`);
+          }
+          currentResult = processed;
+        } catch (error) {
+          throw new Error(`Post-call guardrail '${guardrail.name}' failed: ${error.message}`);
+        }
+      }
+    }
+
+    return currentResult;
+  }
+
+  async executeGuardrail(guardrail, processId, data) {
+    if (typeof guardrail.code === 'string') {
+      // Execute string code as function
+      const func = new Function('processId', 'data', `return (${guardrail.code})(processId, data)`);
+      return await func(processId, data);
+    } else if (typeof guardrail.code === 'function') {
+      return await guardrail.code(processId, data);
+    } else {
+      throw new Error(`Invalid guardrail code for '${guardrail.name}'`);
+    }
+  }
+
+  async completion(options) {
+    const processId = this.generateProcessId();
+    let selectedModel;
+
+    try {
+      // Select model
+      selectedModel = this.selectModel(options.model);
+      console.log(`[${processId}] Selected model: ${selectedModel.llm_params.model}`);
+
+      // Prepare input for guardrails
+      const input = {
+        model: selectedModel.llm_params.model,
+        messages: options.messages || [],
+        temperature: options.temperature,
+        maxTokens: options.max_tokens || options.maxTokens,
+        topP: options.top_p || options.topP,
+        frequencyPenalty: options.frequency_penalty || options.frequencyPenalty,
+        presencePenalty: options.presence_penalty || options.presencePenalty,
+        stop: options.stop,
+        tools: options.tools,
+        toolChoice: options.tool_choice || options.toolChoice
+      };
+
+      // Apply pre-call guardrails
+      const processedInput = await this.applyPreGuardrails(processId, input);
+
+      // Create completion using selected model
+      const result = await this.callProvider(selectedModel, processedInput);
+
+      // Apply post-call guardrails
+      const finalResult = await this.applyPostGuardrails(processId, result);
+
+      // Return result with selected model information
+      return {
+        result: finalResult,
+        selectedModel: selectedModel.llm_params.model, // Full model name with provider prefix
+        selectedModelName: selectedModel.model_name // User-friendly model name
+      };
+
+    } catch (error) {
+      console.error(`[${processId}] Router error:`, error.message);
+      throw error;
+    }
+  }
+
+  async callProvider(modelConfig, input) {
+    const firstSlashIndex = modelConfig.llm_params.model.indexOf('/');
+    const providerName = firstSlashIndex !== -1 ? modelConfig.llm_params.model.substring(0, firstSlashIndex) : '';
+    const actualModel = firstSlashIndex !== -1 ? modelConfig.llm_params.model.substring(firstSlashIndex + 1) : modelConfig.llm_params.model;
+    const provider = this.providers[providerName];
+
+    if (!provider) {
+      throw new Error(`Unknown provider: ${providerName}`);
+    }
+
+    // Set the API key for this request (only if provided, otherwise use provider's default)
+    if (modelConfig.llm_params.api_key !== undefined) {
+      provider.apiKey = modelConfig.llm_params.api_key;
+    }
+
+    // Call the provider directly with just the actual model name (without provider prefix)
+    return await provider.createCompletion(input.messages, {
+      model: actualModel,
+      temperature: input.temperature,
+      maxTokens: input.maxTokens,
+      topP: input.topP,
+      frequencyPenalty: input.frequencyPenalty,
+      presencePenalty: input.presencePenalty,
+      stop: input.stop,
+      tools: input.tools,
+      toolChoice: input.toolChoice
+    });
+  }
+
+  generateProcessId() {
+    return `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+  }
+
+  // Method to add guardrails dynamically
+  addGuardrail(guardrail) {
+    this.guardrails.push(guardrail);
+  }
+
+  // Method to get available models
+  getAvailableModels() {
+    return Object.keys(this.modelsByName);
+  }
+
+  // Method to get model count for load balancing info
+  getModelStats() {
+    const stats = {};
+    Object.keys(this.modelsByName).forEach(modelName => {
+      stats[modelName] = this.modelsByName[modelName].length;
+    });
+    return stats;
+  }
+}
+
+function router(modelList, strategy = 'default') {
+  return new Router(modelList, strategy);
+}
+
+module.exports = {
+  Router,
+  router
+};

package/server.js

@@ -0,0 +1,186 @@
+const http = require('http');
+const url = require('url');
+
+class Server {
+  constructor(options = {}) {
+    this.port = options.port || process.env.PORT || 3000;
+    this.host = options.host || process.env.HOST || 'localhost';
+    this.router = options.router;
+    this.middlewares = [];
+  }
+
+  use(middleware) {
+    if (middleware && typeof middleware.completion === 'function') {
+      // It's a router
+      this.router = middleware;
+    } else if (typeof middleware === 'function') {
+      // It's middleware
+      this.middlewares.push(middleware);
+    }
+  }
+
+  async handleRequest(req, res) {
+    try {
+      // Set CORS headers
+      res.setHeader('Access-Control-Allow-Origin', '*');
+      res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+      res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Authorization');
+
+      if (req.method === 'OPTIONS') {
+        res.writeHead(200);
+        res.end();
+        return;
+      }
+
+      const parsedUrl = url.parse(req.url, true);
+      const path = parsedUrl.pathname;
+
+      // Only handle chat completions endpoint
+      if (path !== '/v1/chat/completions' || req.method !== 'POST') {
+        this.sendError(res, 404, 'Not Found', 'Endpoint not found');
+        return;
+      }
+
+      // Parse request body
+      const body = await this.parseBody(req);
+
+      // Validate request
+      const validation = this.validateChatRequest(body);
+      if (!validation.valid) {
+        this.sendError(res, 400, 'Bad Request', validation.error);
+        return;
+      }
+
+      console.log(`[${new Date().toISOString()}] ${req.method} ${req.url}`);
+      console.log('Headers:', JSON.stringify(req.headers));
+      console.log('Body parsing completed successfully');
+
+      // Process the completion request
+      const result = await this.processCompletion(body);
+
+      // Send successful response
+      this.sendSuccess(res, result);
+
+    } catch (error) {
+      console.error('Server error:', error);
+      this.sendError(res, 500, 'Internal Server Error', error.message);
+    }
+  }
+
+  parseBody(req) {
+    return new Promise((resolve, reject) => {
+      let body = '';
+      req.on('data', chunk => {
+        body += chunk.toString();
+      });
+      req.on('end', () => {
+        try {
+          resolve(JSON.parse(body));
+        } catch (error) {
+          reject(new Error('Invalid JSON in request body'));
+        }
+      });
+      req.on('error', reject);
+    });
+  }
+
+  validateChatRequest(body) {
+    if (!body) {
+      return { valid: false, error: 'Request body is required' };
+    }
+
+    if (!body.messages || !Array.isArray(body.messages)) {
+      return { valid: false, error: 'messages array is required' };
+    }
+
+    if (body.messages.length === 0) {
+      return { valid: false, error: 'messages array cannot be empty' };
+    }
+
+    for (const message of body.messages) {
+      if (!message.role || !['system', 'user', 'assistant'].includes(message.role)) {
+        return { valid: false, error: 'Each message must have a valid role (system, user, or assistant)' };
+      }
+      if (typeof message.content !== 'string') {
+        return { valid: false, error: 'Each message must have string content' };
+      }
+    }
+
+    return { valid: true };
+  }
+
+  async processCompletion(body) {
+    if (!this.router) {
+      throw new Error('No router configured. Use app.use(router) to add a router.');
+    }
+
+    console.log(`Starting completion with model: ${body.model || 'auto-selected'}`);
+
+    const routerResponse = await this.router.completion(body);
+    const { result, selectedModel, selectedModelName } = routerResponse;
+
+    // Return format with metadata plus message array
+    const assistantMessage = {
+      role: 'assistant',
+      content: typeof result === 'string' ? result : result.content || ''
+    };
+
+    // Include original messages plus assistant response
+    const messages = [...(body.messages || []), assistantMessage];
+
+    return {
+      id: `chatcmpl-${Date.now()}`,
+      object: 'chat.completion',
+      created: Math.floor(Date.now() / 1000),
+      model: body.model || selectedModel, // Use selected model if auto-selected
+      messages: messages
+    };
+  }
+
+  sendSuccess(res, data) {
+    const response = JSON.stringify(data);
+    res.writeHead(200, {
+      'Content-Type': 'application/json',
+      'Content-Length': Buffer.byteLength(response)
+    });
+    res.end(response);
+  }
+
+  sendError(res, statusCode, type, message) {
+    const error = {
+      error: {
+        message: message,
+        type: type.toLowerCase().replace(/\s+/g, '_')
+      }
+    };
+    const response = JSON.stringify(error);
+    res.writeHead(statusCode, {
+      'Content-Type': 'application/json',
+      'Content-Length': Buffer.byteLength(response)
+    });
+    res.end(response);
+  }
+
+  listen(port, host, callback) {
+    const actualPort = port || this.port;
+    const actualHost = host || this.host;
+
+    const server = http.createServer((req, res) => this.handleRequest(req, res));
+
+    server.listen(actualPort, actualHost, () => {
+      console.log(`🚀 llmjs2 server running on http://${actualHost}:${actualPort}`);
+      if (callback) callback();
+    });
+
+    return server;
+  }
+}
+
+function createApp(options = {}) {
+  return new Server(options);
+}
+
+module.exports = {
+  Server,
+  app: createApp()
+};