llmjs2 1.3.9 → 1.7.1

package/core/index.js

@@ -0,0 +1,115 @@
+const OpenAIProvider = require('./providers/openai');
+const OllamaProvider = require('./providers/ollama');
+const OpenRouterProvider = require('./providers/openrouter');
+const { completion } = require('./completion');
+const { router } = require('./router');
+const { app } = require('./server');
+
+class LLMJS2 {
+  constructor(config = {}) {
+    this.defaultProvider = config.defaultProvider;
+    this.providers = {
+      openai: new OpenAIProvider(config.openai || {}),
+      ollama: new OllamaProvider(config.ollama || {}),
+      openrouter: new OpenRouterProvider(config.openrouter || {})
+    };
+  }
+
+  parseModel(modelString) {
+    if (!modelString || typeof modelString !== 'string') {
+      return { provider: null, model: null };
+    }
+
+    const firstSlashIndex = modelString.indexOf('/');
+    if (firstSlashIndex !== -1) {
+      return {
+        provider: modelString.substring(0, firstSlashIndex),
+        model: modelString.substring(firstSlashIndex + 1)
+      };
+    }
+
+    return { provider: null, model: modelString };
+  }
+
+  getAvailableProviders() {
+    const available = [];
+
+    const openaiKey = process.env.OPENAI_API_KEY || this.providers.openai.apiKey;
+    const ollamaKey = process.env.OLLAMA_API_KEY || this.providers.ollama.apiKey;
+    const openrouterKey = process.env.OPEN_ROUTER_API_KEY || this.providers.openrouter.apiKey;
+
+    if (openaiKey && typeof openaiKey === 'string' && openaiKey.trim() && !openaiKey.startsWith(':')) {
+      available.push('openai');
+    }
+    if (ollamaKey && typeof ollamaKey === 'string' && ollamaKey.trim() && !ollamaKey.startsWith(':')) {
+      available.push('ollama');
+    }
+    if (openrouterKey && typeof openrouterKey === 'string' && openrouterKey.trim() && !openrouterKey.startsWith(':')) {
+      available.push('openrouter');
+    }
+
+    return available;
+  }
+
+  getAutoProvider() {
+    const availableProviders = this.getAvailableProviders();
+    if (availableProviders.length === 0) {
+      throw new Error('No API keys found. Set OLLAMA_API_KEY, OPEN_ROUTER_API_KEY, or OPENAI_API_KEY environment variables.');
+    }
+
+    const randomIndex = Math.floor(Math.random() * availableProviders.length);
+    const providerName = availableProviders[randomIndex];
+    const provider = this.providers[providerName];
+
+    return {
+      provider,
+      model: provider.defaultModel
+    };
+  }
+
+  getProvider(modelString) {
+    const { provider: specifiedProvider, model } = this.parseModel(modelString);
+
+    if (specifiedProvider) {
+      const provider = this.providers[specifiedProvider];
+      if (!provider) {
+        throw new Error(`Unknown provider: ${specifiedProvider}`);
+      }
+      return { provider, model };
+    }
+
+    const availableProviders = this.getAvailableProviders();
+    if (availableProviders.length === 0) {
+      throw new Error('No API keys configured. Set OPENAI_API_KEY, OLLAMA_API_KEY, or OPEN_ROUTER_API_KEY environment variables.');
+    }
+
+    const providerName = this.defaultProvider && availableProviders.includes(this.defaultProvider)
+      ? this.defaultProvider
+      : availableProviders[0];
+    const provider = this.providers[providerName];
+
+    return {
+      provider,
+      model: model || provider.defaultModel
+    };
+  }
+
+  completion(input) {
+    return completion(input, {
+      defaultProvider: this.defaultProvider,
+      openai: { apiKey: this.providers.openai.apiKey, baseURL: this.providers.openai.baseURL, timeout: this.providers.openai.timeout },
+      ollama: { apiKey: this.providers.ollama.apiKey, baseURL: this.providers.ollama.baseURL, timeout: this.providers.ollama.timeout },
+      openrouter: { apiKey: this.providers.openrouter.apiKey, baseURL: this.providers.openrouter.baseURL, timeout: this.providers.openrouter.timeout }
+    });
+  }
+}
+
+module.exports = {
+  completion,
+  LLMJS2,
+  router,
+  app,
+  OpenAIProvider,
+  OllamaProvider,
+  OpenRouterProvider
+};

package/{providers → core/providers}/ollama.js

@@ -8,14 +8,18 @@ class OllamaProvider {
     this.timeout = config.timeout || 120000; // 2 minutes (LLMs can be slow)
   }
 
-  async makeRequest(data) {
-    if (!this.apiKey) {
+  async makeRequest(data, requestOptions = {}) {
+    const apiKey = requestOptions.apiKey || this.apiKey;
+    const baseURL = requestOptions.baseURL || this.baseURL;
+    const timeout = requestOptions.timeout || this.timeout;
+
+    if (!apiKey) {
       throw new Error('Ollama API key is required. Set OLLAMA_API_KEY environment variable or pass apiKey in config.');
     }
 
     const postData = JSON.stringify(data);
 
-    const parsedUrl = new URL(this.baseURL);
+    const parsedUrl = new URL(baseURL);
 
     const options = {
       hostname: parsedUrl.hostname,
@@ -24,7 +28,7 @@ class OllamaProvider {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
-        'Authorization': `Bearer ${this.apiKey}`,
+        'Authorization': `Bearer ${apiKey}`,
         'Content-Length': Buffer.byteLength(postData)
       }
     };
@@ -56,7 +60,7 @@ class OllamaProvider {
         reject(new Error(`Ollama request failed: ${error.message}`));
       });
 
-      req.setTimeout(this.timeout, () => {
+      req.setTimeout(timeout, () => {
         req.destroy();
         reject(new Error('Ollama request timed out'));
       });
@@ -100,7 +104,11 @@ class OllamaProvider {
       });
     }
 
-    const response = await this.makeRequest(data);
+    const response = await this.makeRequest(data, {
+      apiKey: options.apiKey,
+      baseURL: options.baseURL,
+      timeout: options.timeout
+    });
 
     return {
       content: response.message?.content || '',

package/{providers → core/providers}/openai.js

@@ -8,12 +8,16 @@ class OpenAIProvider {
     this.timeout = config.timeout || 60000; // 60 seconds
   }
 
-  async makeRequest(endpoint, data) {
-    if (!this.apiKey) {
+  async makeRequest(endpoint, data, requestOptions = {}) {
+    const apiKey = requestOptions.apiKey || this.apiKey;
+    const baseURL = requestOptions.baseURL || this.baseURL;
+    const timeout = requestOptions.timeout || this.timeout;
+
+    if (!apiKey) {
       throw new Error('OpenAI API key is required. Set OPENAI_API_KEY environment variable or pass apiKey in config.');
     }
 
-    const requestUrl = `${this.baseURL}${endpoint}`;
+    const requestUrl = `${baseURL}${endpoint}`;
     const postData = JSON.stringify(data);
 
     const parsedUrl = new URL(requestUrl);
@@ -25,7 +29,7 @@ class OpenAIProvider {
       method: 'POST',
       headers: {
         'Content-Type': 'application/json',
-        'Authorization': `Bearer ${this.apiKey}`,
+        'Authorization': `Bearer ${apiKey}`,
         'Content-Length': Buffer.byteLength(postData)
       }
     };
@@ -57,7 +61,7 @@ class OpenAIProvider {
         reject(new Error(`OpenAI request failed: ${error.message}`));
       });
 
-      req.setTimeout(this.timeout, () => {
+      req.setTimeout(timeout, () => {
         req.destroy();
         reject(new Error('OpenAI request timed out'));
       });
@@ -88,7 +92,11 @@ class OpenAIProvider {
       }
     });
 
-    const response = await this.makeRequest('/chat/completions', data);
+    const response = await this.makeRequest('/chat/completions', data, {
+      apiKey: options.apiKey,
+      baseURL: options.baseURL,
+      timeout: options.timeout
+    });
 
     return {
       content: response.choices[0]?.message?.content || '',

package/core/providers/openrouter.js

@@ -0,0 +1,206 @@
+const https = require('https');
+
+function asNonEmptyString(value) {
+  if (typeof value !== 'string') {
+    return '';
+  }
+  return value.trim();
+}
+
+function firstNonEmptyString(...values) {
+  for (const value of values) {
+    const normalized = asNonEmptyString(value);
+    if (normalized) {
+      return normalized;
+    }
+  }
+  return '';
+}
+
+function isValidHttpUrl(value) {
+  const normalized = asNonEmptyString(value);
+  if (!normalized) {
+    return false;
+  }
+
+  try {
+    const parsed = new URL(normalized);
+    return parsed.protocol === 'https:' || parsed.protocol === 'http:';
+  } catch (_error) {
+    return false;
+  }
+}
+
+function normalizeOpenRouterModel(model) {
+  let normalized = asNonEmptyString(model);
+
+  if (!normalized) {
+    return '';
+  }
+
+  // Only inspect the first provider separator and preserve the rest of the path.
+  const firstSlashIndex = normalized.indexOf('/');
+  if (firstSlashIndex === -1) {
+    return `openrouter/${normalized}`;
+  }
+
+  const provider = normalized.substring(0, firstSlashIndex).trim().toLowerCase();
+  if (provider === 'openrouter') {
+    const remainder = normalized.substring(firstSlashIndex + 1).trim();
+
+    if (!remainder) {
+      return '';
+    }
+
+    // Keep "openrouter/free" and collapse "openrouter/openrouter/free" to "openrouter/free".
+    if (remainder.includes('/')) {
+      normalized = remainder;
+    } else {
+      normalized = `openrouter/${remainder}`;
+    }
+  }
+
+  return normalized;
+}
+
+class OpenRouterProvider {
+  constructor(config = {}) {
+    const configuredBaseURL = firstNonEmptyString(config.baseURL, process.env.OPEN_ROUTER_BASE_URL);
+    this.baseURL = isValidHttpUrl(configuredBaseURL)
+      ? configuredBaseURL
+      : 'https://openrouter.ai/api/v1/chat/completions';
+
+    this.apiKey = firstNonEmptyString(config.apiKey, process.env.OPEN_ROUTER_API_KEY);
+    const configuredDefaultModel = firstNonEmptyString(config.defaultModel, process.env.OPEN_ROUTER_DEFAULT_MODEL) || 'openrouter/openrouter/free';
+    this.defaultModel = configuredDefaultModel;
+    this.timeout = config.timeout || 60000; // 60 seconds
+    this.config = config; // Store entire config for additional properties like referer, title
+  }
+
+  async makeRequest(data, requestOptions = {}) {
+    const apiKey = firstNonEmptyString(requestOptions.apiKey, this.apiKey);
+    const baseURL = firstNonEmptyString(requestOptions.baseURL, this.baseURL);
+    const timeout = requestOptions.timeout || this.timeout;
+    const referer = firstNonEmptyString(requestOptions.referer, this.config.referer, process.env.OPEN_ROUTER_REFERER);
+    const title = firstNonEmptyString(requestOptions.title, this.config.title, process.env.OPEN_ROUTER_TITLE);
+
+    if (!apiKey) {
+      throw new Error('OpenRouter API key is required. Set OPEN_ROUTER_API_KEY environment variable or pass apiKey in config.');
+    }
+
+    if (!isValidHttpUrl(baseURL)) {
+      throw new Error('OpenRouter base URL is invalid. Set OPEN_ROUTER_BASE_URL to a valid http(s) URL.');
+    }
+
+    const postData = JSON.stringify(data);
+
+    const parsedUrl = new URL(baseURL);
+
+    const headers = {
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${apiKey}`,
+      'Content-Length': Buffer.byteLength(postData)
+    };
+
+    if (isValidHttpUrl(referer)) {
+      headers['HTTP-Referer'] = referer;
+    }
+
+    if (title) {
+      headers['X-Title'] = title;
+    }
+
+    const options = {
+      hostname: parsedUrl.hostname,
+      port: parsedUrl.port || 443,
+      path: parsedUrl.pathname + parsedUrl.search,
+      method: 'POST',
+      headers
+    };
+    
+
+    return new Promise((resolve, reject) => {
+      const req = https.request(options, (res) => {
+        let body = '';
+
+        res.on('data', (chunk) => {
+          body += chunk;
+        });
+
+        res.on('end', () => {
+          try {
+            if (res.statusCode >= 200 && res.statusCode < 300) {
+              const response = JSON.parse(body);
+              resolve(response);
+            } else {
+              const error = JSON.parse(body);
+              reject(new Error(`OpenRouter API error (${res.statusCode}): ${error.error?.message || 'Unknown error'}`));
+            }
+          } catch (parseError) {
+            reject(new Error(`Failed to parse OpenRouter response: ${parseError.message}`));
+          }
+        });
+      });
+
+      req.on('error', (error) => {
+        reject(new Error(`OpenRouter request failed: ${error.message}`));
+      });
+
+      req.setTimeout(timeout, () => {
+        req.destroy();
+        reject(new Error('OpenRouter request timed out'));
+      });
+
+      req.write(postData);
+      req.end();
+    });
+  }
+
+  async createCompletion(messages, options = {}) {
+    const normalizedModel = normalizeOpenRouterModel(options.model || this.defaultModel);
+
+    if (!normalizedModel) {
+      throw new Error('OpenRouter model is invalid. Provide a non-empty model, for example "openrouter/free" or "meta-llama/llama-3.1-8b-instruct:free".');
+    }
+
+    const data = {
+      model: normalizedModel,
+      messages: messages,
+      temperature: options.temperature || 0.7,
+      max_tokens: options.maxTokens,
+      top_p: options.topP,
+      frequency_penalty: options.frequencyPenalty,
+      presence_penalty: options.presencePenalty,
+      stop: options.stop,
+      tools: options.tools,
+      tool_choice: options.toolChoice,
+      transforms: options.transforms,
+      models: options.models,
+      route: options.route
+    };
+
+    // Remove undefined values
+    Object.keys(data).forEach(key => {
+      if (data[key] === undefined) {
+        delete data[key];
+      }
+    });
+
+    const response = await this.makeRequest(data, {
+      apiKey: options.apiKey,
+      baseURL: options.baseURL,
+      timeout: options.timeout
+    });
+
+    return {
+      content: response.choices[0]?.message?.content || '',
+      role: response.choices[0]?.message?.role || 'assistant',
+      tool_calls: response.choices[0]?.message?.tool_calls,
+      usage: response.usage,
+      model: response.model,
+      finishReason: response.choices[0]?.finish_reason
+    };
+  }
+}
+
+module.exports = OpenRouterProvider;

package/core/router.js

@@ -0,0 +1,252 @@
+const logger = require('./logger');
+const { completion } = require('./completion');
+
+function normalizeStrategy(strategy) {
+  const supportedStrategies = new Set(['default', 'random', 'sequential']);
+  return supportedStrategies.has(strategy) ? strategy : 'default';
+}
+
+function resolveApiKey(apiKey) {
+  if (typeof apiKey === 'string' && apiKey.startsWith('os.environ/')) {
+    const envVar = apiKey.replace('os.environ/', '');
+    return process.env[envVar] || apiKey;
+  }
+  return apiKey;
+}
+
+function normalizeModelList(modelList) {
+  if (!Array.isArray(modelList) || modelList.length === 0) {
+    throw new Error('Router requires a non-empty model list');
+  }
+
+  return modelList.map((model, index) => {
+    if (!model || typeof model !== 'object') {
+      throw new Error(`Model at index ${index} must be an object`);
+    }
+    if (!model.model_name) {
+      throw new Error(`Model at index ${index} is missing model_name`);
+    }
+    if (!model.llm_params || typeof model.llm_params !== 'object') {
+      throw new Error(`Model '${model.model_name}' is missing llm_params`);
+    }
+    if (!model.llm_params.model) {
+      throw new Error(`Model '${model.model_name}' is missing llm_params.model`);
+    }
+
+    return {
+      model_name: model.model_name,
+      llm_params: {
+        model: model.llm_params.model,
+        api_key: resolveApiKey(model.llm_params.api_key),
+        api_base: model.llm_params.api_base
+      }
+    };
+  });
+}
+
+function buildModelsByName(modelList) {
+  return modelList.reduce((accumulator, model) => {
+    if (!accumulator[model.model_name]) {
+      accumulator[model.model_name] = [];
+    }
+    accumulator[model.model_name].push(model);
+    return accumulator;
+  }, {});
+}
+
+function router(modelList, strategy = 'default', options = {}) {
+  const normalizedModelList = normalizeModelList(modelList);
+  const normalizedStrategy = normalizeStrategy(strategy);
+  const allowUnsafeGuardrails = Boolean(options.allowUnsafeGuardrails) || process.env.LLMJS2_ALLOW_UNSAFE_GUARDRAILS === 'true';
+  const guardrails = [];
+  const modelsByName = buildModelsByName(normalizedModelList);
+  let sequentialIndex = 0;
+
+  function getRandomItem(items) {
+    const randomIndex = Math.floor(Math.random() * items.length);
+    return items[randomIndex];
+  }
+
+  function getNextSequentialModel() {
+    const selectedModel = normalizedModelList[sequentialIndex];
+    sequentialIndex = (sequentialIndex + 1) % normalizedModelList.length;
+    return selectedModel;
+  }
+
+  function ensureModelsAvailable() {
+    if (normalizedModelList.length === 0) {
+      throw new Error('Router has no configured models');
+    }
+  }
+
+  function selectModel(modelName) {
+    ensureModelsAvailable();
+
+    if (!modelName) {
+      return autoSelectModel();
+    }
+
+    const availableModels = modelsByName[modelName];
+    if (!availableModels || availableModels.length === 0) {
+      throw new Error(`Model '${modelName}' not found in router configuration`);
+    }
+
+    if (availableModels.length === 1) {
+      return availableModels[0];
+    }
+
+    return getRandomItem(availableModels);
+  }
+
+  function autoSelectModel() {
+    ensureModelsAvailable();
+
+    switch (normalizedStrategy) {
+      case 'random':
+        return getRandomItem(normalizedModelList);
+      case 'sequential':
+      case 'default':
+      default:
+        return getNextSequentialModel();
+    }
+  }
+
+  function normalizeCompletionInput(input = {}) {
+    return {
+      modelName: input.model,
+      messages: input.messages || [],
+      stop: input.stop,
+      tools: input.tools,
+      toolChoice: input.tool_choice ?? input.toolChoice,
+      timeout: input.timeout,
+      apiKey: input.apiKey,
+      host: input.host || input.baseURL || input.baseUrl,
+      final: input.final ?? true
+    };
+  }
+
+  async function executeGuardrail(guardrail, processId, data) {
+    if (typeof guardrail.code === 'string') {
+      if (!allowUnsafeGuardrails) {
+        throw new Error(
+          `Guardrail '${guardrail.name}' uses string code, but unsafe execution is disabled. Set router option allowUnsafeGuardrails=true or LLMJS2_ALLOW_UNSAFE_GUARDRAILS=true to enable.`
+        );
+      }
+
+      const func = new Function('processId', 'data', `return (${guardrail.code})(processId, data)`);
+      return await func(processId, data);
+    }
+
+    if (typeof guardrail.code === 'function') {
+      return await guardrail.code(processId, data);
+    }
+
+    throw new Error(`Invalid guardrail code for '${guardrail.name}'`);
+  }
+
+  async function applyGuardrails(mode, processId, data) {
+    const modeLabel = mode === 'pre_call' ? 'Pre-call' : 'Post-call';
+    let currentValue = data;
+
+    for (const guardrail of guardrails) {
+      if (guardrail.mode !== mode) {
+        continue;
+      }
+
+      try {
+        const processed = await executeGuardrail(guardrail, processId, currentValue);
+        if (processed === null || processed === undefined) {
+          throw new Error(`Guardrail '${guardrail.name}' returned null/undefined`);
+        }
+        currentValue = processed;
+      } catch (error) {
+        throw new Error(`${modeLabel} guardrail '${guardrail.name}' failed: ${error.message}`);
+      }
+    }
+
+    return currentValue;
+  }
+
+  async function completionWithModel(modelConfig, input) {
+    return completion({
+      ...input,
+      model: modelConfig.llm_params.model,
+      apiKey: input.apiKey || modelConfig.llm_params.api_key,
+      host: input.host || modelConfig.llm_params.api_base,
+      final: false
+    });
+  }
+
+  function generateProcessId() {
+    return `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+  }
+
+  return {
+    setGuardrails(nextGuardrails) {
+      guardrails.length = 0;
+      if (Array.isArray(nextGuardrails)) {
+        guardrails.push(...nextGuardrails);
+      }
+    },
+
+    addGuardrail(guardrail) {
+      guardrails.push(guardrail);
+    },
+
+    getAvailableModels() {
+      return Object.keys(modelsByName);
+    },
+
+    getModelStats() {
+      const stats = {};
+      Object.keys(modelsByName).forEach((modelName) => {
+        stats[modelName] = modelsByName[modelName].length;
+      });
+      return stats;
+    },
+
+    selectModel,
+
+    autoSelectModel,
+
+    executeGuardrail,
+
+    async completion(input = {}) {
+      const processId = generateProcessId();
+
+      try {
+        const normalizedInput = normalizeCompletionInput(input);
+        const selectedModel = selectModel(normalizedInput.modelName);
+
+        logger.debug('Selected model', {
+          processId,
+          model: selectedModel.llm_params.model,
+          modelName: selectedModel.model_name
+        });
+
+        const { modelName, ...completionInput } = normalizedInput;
+        const preparedInput = {
+          model: selectedModel.llm_params.model,
+          ...completionInput
+        };
+
+        const processedInput = await applyGuardrails('pre_call', processId, preparedInput);
+        const result = await completionWithModel(selectedModel, processedInput);
+        const finalResult = await applyGuardrails('post_call', processId, result);
+
+        return {
+          result: finalResult,
+          selectedModel: selectedModel.llm_params.model,
+          selectedModelName: selectedModel.model_name
+        };
+      } catch (error) {
+        logger.error('Router error', { processId, error: error.message });
+        throw error;
+      }
+    }
+  };
+}
+
+module.exports = {
+  router
+};