neoagent 2.4.1-beta.9 → 2.4.1

package/server/services/ai/providers/nvidia.js

@@ -0,0 +1,154 @@
+const OpenAI = require('openai');
+const { OpenAICompatibleProvider } = require('./openaiCompatible');
+
+const NVIDIA_BASE_URL = 'https://integrate.api.nvidia.com/v1';
+
+// Context windows per model (tokens)
+const CONTEXT_WINDOWS = {
+  'nvidia/nemotron-3-super-120b-a12b': 262144,
+  'moonshotai/kimi-k2.5':             262144,
+  'minimaxai/minimax-m2.5':           196608,
+  'z-ai/glm5':                        202752,
+  'meta/llama-4-maverick-17b-128e-instruct': 1048576,
+  'meta/llama-4-scout-17b-16e-instruct':     1048576,
+  'deepseek-ai/deepseek-r1-0528':     163840,
+  'qwen/qwq-32b':                     131072,
+};
+
+// Reasoning models: no temperature, no top_p
+const REASONING_MODELS = new Set([
+  'deepseek-ai/deepseek-r1-0528',
+  'qwen/qwq-32b',
+]);
+
+class NvidiaProvider extends OpenAICompatibleProvider {
+  constructor(config = {}) {
+    super(config);
+    this.name = 'nvidia';
+    this.models = Object.keys(CONTEXT_WINDOWS);
+    this.client = new OpenAI({
+      apiKey: config.apiKey || process.env.NVIDIA_API_KEY,
+      baseURL: config.baseUrl || NVIDIA_BASE_URL,
+    });
+  }
+
+  async listModels() {
+    try {
+      const res = await this.client.models.list();
+      const DROP = /embed|bge|e5-|rerank|guard|safety|moderat|diffus|flux|stable|imagen|vision-enc|whisper|tts|speech|paraphrase|classif/i;
+      return res.data
+        .filter((m) => !DROP.test(m.id))
+        .map((m) => ({ id: m.id, name: m.id }));
+    } catch (err) {
+      throw new Error(`NVIDIA NIM request failed: ${err?.message || String(err)}`);
+    }
+  }
+
+  getContextWindow(model) {
+    return CONTEXT_WINDOWS[model] ?? 131072;
+  }
+
+  _isReasoningModel(model) {
+    return REASONING_MODELS.has(model);
+  }
+
+  _buildParams(model, messages, tools, options) {
+    const params = {
+      model,
+      messages,
+      max_tokens: options.maxTokens || 8192,
+    };
+
+    if (!this._isReasoningModel(model)) {
+      params.temperature = options.temperature ?? 0.6;
+    }
+
+    if (tools && tools.length > 0) {
+      params.tools = this.formatTools(tools);
+      params.tool_choice = 'auto';
+    }
+
+    return params;
+  }
+
+  async chat(messages, tools = [], options = {}) {
+    const model = options.model || this.getDefaultModel();
+    const params = this._buildParams(model, messages, tools, options);
+    let response;
+    try {
+      response = await this.client.chat.completions.create(params);
+    } catch (err) {
+      throw new Error(`NVIDIA NIM request failed: ${err?.message || String(err)}`);
+    }
+    return this.normalizeResponse(response);
+  }
+
+  async *stream(messages, tools = [], options = {}) {
+    const model = options.model || this.getDefaultModel();
+    const params = {
+      ...this._buildParams(model, messages, tools, options),
+      stream: true,
+      stream_options: { include_usage: true },
+    };
+
+    let stream;
+    try {
+      stream = await this.client.chat.completions.create(params);
+    } catch (err) {
+      throw new Error(`NVIDIA NIM request failed: ${err?.message || String(err)}`);
+    }
+
+    let toolCalls = [];
+    let content = '';
+    let finalUsage = null;
+
+    for await (const chunk of stream) {
+      if (chunk.usage && (!chunk.choices || chunk.choices.length === 0)) {
+        finalUsage = this.normalizeUsage(chunk.usage);
+        continue;
+      }
+
+      const delta = chunk.choices?.[0]?.delta;
+      if (!delta) continue;
+
+      if (delta.content) {
+        content += delta.content;
+        yield { type: 'content', content: delta.content };
+      }
+
+      if (delta.tool_calls) {
+        for (const tc of delta.tool_calls) {
+          if (!toolCalls[tc.index]) {
+            toolCalls[tc.index] = {
+              id: tc.id || '',
+              type: 'function',
+              function: { name: tc.function?.name || '', arguments: '' },
+            };
+          }
+          if (tc.id) toolCalls[tc.index].id = tc.id;
+          if (tc.function?.name) toolCalls[tc.index].function.name = tc.function.name;
+          if (tc.function?.arguments) toolCalls[tc.index].function.arguments += tc.function.arguments;
+        }
+      }
+
+      const finishReason = chunk.choices[0]?.finish_reason;
+      if (finishReason === 'tool_calls' || (finishReason === 'stop' && toolCalls.length > 0)) {
+        yield { type: 'tool_calls', toolCalls, content, usage: this.normalizeUsage(chunk.usage) || finalUsage };
+        return;
+      }
+      if (finishReason === 'stop') {
+        yield { type: 'done', content, usage: this.normalizeUsage(chunk.usage) || finalUsage };
+        return;
+      }
+    }
+
+    if (toolCalls.length > 0) {
+      yield { type: 'tool_calls', toolCalls, content, usage: finalUsage };
+    } else {
+      yield { type: 'done', content, usage: finalUsage };
+    }
+  }
+
+}
+
+module.exports = { NvidiaProvider };

package/server/services/ai/providers/ollama.js

@@ -81,33 +81,78 @@ class OllamaProvider extends BaseProvider {
     }));
   }
 
-  async chat(messages, tools = [], options = {}) {
-    const model = options.model || this.config.model || 'llama3.1';
-    await this.ensureModel(model);
+  buildChatBody(messages, tools, options, stream) {
     const body = {
-      model,
-      messages: messages.map(m => ({
-        role: m.role,
-        content: m.content || '',
-        ...(m.tool_calls ? { tool_calls: m.tool_calls } : {}),
-        ...(m.tool_call_id ? { tool_call_id: m.tool_call_id } : {})
-      })),
-      stream: false,
+      model: options.model || this.config.model || 'llama3.1',
+      messages: messages.map(m => {
+        const msg = {
+          role: m.role,
+          content: m.content || '',
+          ...(m.tool_call_id ? { tool_call_id: m.tool_call_id } : {})
+        };
+        if (m.tool_calls) {
+          msg.tool_calls = m.tool_calls.map(tc => ({
+            ...tc,
+            function: {
+              ...tc.function,
+              arguments: typeof tc.function.arguments === 'string'
+                ? (function() { try { return JSON.parse(tc.function.arguments || '{}'); } catch(e) { return {}; } })()
+                : tc.function.arguments
+            }
+          }));
+        }
+        return msg;
+      }),
+      stream,
       options: {
         temperature: options.temperature ?? 0.7,
         num_predict: options.maxTokens || 16384
       }
     };
-
     if (tools.length > 0) {
       body.tools = this.formatToolsForOllama(tools);
     }
+    return body;
+  }
 
+  // Ollama returns HTTP 200 with an error body for some failures and a non-2xx
+  // status for others; surface both as real errors instead of letting callers
+  // see a silently empty response. Tags models that reject tools so the caller
+  // can transparently retry without them.
+  async postChat(body) {
     const res = await fetch(`${this.baseUrl}/api/chat`, {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
       body: JSON.stringify(body)
     });
+    if (!res.ok) {
+      const detail = await res.text().catch(() => '');
+      let message = detail;
+      try { message = JSON.parse(detail)?.error || detail; } catch {}
+      const err = new Error(`Ollama /api/chat failed (HTTP ${res.status}): ${message || res.statusText}`);
+      if (/does not support tools|tools.*not supported/i.test(message)) {
+        err.code = 'OLLAMA_TOOLS_UNSUPPORTED';
+      }
+      throw err;
+    }
+    return res;
+  }
+
+  async chat(messages, tools = [], options = {}) {
+    const model = options.model || this.config.model || 'llama3.1';
+    await this.ensureModel(model);
+
+    let res;
+    try {
+      res = await this.postChat(this.buildChatBody(messages, tools, { ...options, model }, false));
+    } catch (err) {
+      if (err.code === 'OLLAMA_TOOLS_UNSUPPORTED' && tools.length > 0) {
+        console.warn(`[Ollama] Model '${model}' does not support tools; retrying without them.`);
+        res = await this.postChat(this.buildChatBody(messages, [], { ...options, model }, false));
+      } else {
+        throw err;
+      }
+    }
 
     const data = await res.json();
     const msg = data.message || {};
@@ -135,35 +180,24 @@ class OllamaProvider extends BaseProvider {
   async *stream(messages, tools = [], options = {}) {
     const model = options.model || this.config.model || 'llama3.1';
     await this.ensureModel(model);
-    const body = {
-      model,
-      messages: messages.map(m => ({
-        role: m.role,
-        content: m.content || '',
-        ...(m.tool_calls ? { tool_calls: m.tool_calls } : {}),
-        ...(m.tool_call_id ? { tool_call_id: m.tool_call_id } : {})
-      })),
-      stream: true,
-      options: {
-        temperature: options.temperature ?? 0.7,
-        num_predict: options.maxTokens || 16384
-      }
-    };
 
-    if (tools.length > 0) {
-      body.tools = this.formatToolsForOllama(tools);
+    let res;
+    try {
+      res = await this.postChat(this.buildChatBody(messages, tools, { ...options, model }, true));
+    } catch (err) {
+      if (err.code === 'OLLAMA_TOOLS_UNSUPPORTED' && tools.length > 0) {
+        console.warn(`[Ollama] Model '${model}' does not support tools; retrying stream without them.`);
+        res = await this.postChat(this.buildChatBody(messages, [], { ...options, model }, true));
+      } else {
+        throw err;
+      }
     }
 
-    const res = await fetch(`${this.baseUrl}/api/chat`, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/json' },
-      body: JSON.stringify(body)
-    });
-
     const reader = res.body.getReader();
     const decoder = new TextDecoder();
     let content = '';
     let buffer = '';
+    let accumulatedToolCalls = [];
 
     while (true) {
       const { done, value } = await reader.read();
@@ -181,20 +215,23 @@ class OllamaProvider extends BaseProvider {
             content += data.message.content;
             yield { type: 'content', content: data.message.content };
           }
-          if (data.done) {
-            const toolCalls = (data.message?.tool_calls || []).map((tc, i) => ({
+          if (data.message?.tool_calls && Array.isArray(data.message.tool_calls)) {
+            const mapped = data.message.tool_calls.map((tc, i) => ({
               id: `call_ollama_${Date.now()}_${i}`,
               type: 'function',
               function: {
                 name: tc.function.name,
-                arguments: JSON.stringify(tc.function.arguments || {})
+                arguments: typeof tc.function.arguments === 'string' ? tc.function.arguments : JSON.stringify(tc.function.arguments || {})
               }
             }));
+            accumulatedToolCalls = accumulatedToolCalls.concat(mapped);
+          }
+          if (data.done) {
             yield {
               type: 'done',
               content,
-              toolCalls,
-              finishReason: toolCalls.length > 0 ? 'tool_calls' : 'stop',
+              toolCalls: accumulatedToolCalls,
+              finishReason: accumulatedToolCalls.length > 0 ? 'tool_calls' : 'stop',
               usage: data.prompt_eval_count ? {
                 promptTokens: data.prompt_eval_count || 0,
                 completionTokens: data.eval_count || 0,

package/server/services/ai/providers/openai.js

@@ -1,29 +1,31 @@
 const OpenAI = require('openai');
-const { BaseProvider } = require('./base');
+const { OpenAICompatibleProvider } = require('./openaiCompatible');
 
-class OpenAIProvider extends BaseProvider {
+class OpenAIProvider extends OpenAICompatibleProvider {
   constructor(config = {}) {
     super(config);
     this.name = 'openai';
     this.models = [
+      'gpt-5.5',
       'gpt-5',
       'gpt-5-mini',
       'gpt-5-nano',
       'gpt-5.2',
-      'gpt-4.1',
       'o3',
+      'o3-pro',
       'o4-mini'
     ];
     // Reasoning models: no temperature, use max_completion_tokens, support reasoning_effort
-    this.reasoningModels = new Set(['gpt-5', 'gpt-5-mini', 'gpt-5-nano', 'gpt-5.2', 'gpt-5.1', 'o1', 'o3', 'o4-mini', 'o3-mini']);
+    this.reasoningModels = new Set(['gpt-5.5', 'gpt-5', 'gpt-5-mini', 'gpt-5-nano', 'gpt-5.2', 'gpt-5.1', 'o1', 'o3', 'o3-pro', 'o4-mini', 'o3-mini']);
     this.contextWindows = {
+      'gpt-5.5': 1000000,
       'gpt-5': 400000,
-      'gpt-5-mini': 200000,
+      'gpt-5-mini': 400000,
       'gpt-5-nano': 128000,
       'gpt-5.2': 400000,
       'gpt-5.1': 400000,
-      'gpt-4.1': 1047576,
       'o3': 200000,
+      'o3-pro': 200000,
       'o4-mini': 200000,
       'o3-mini': 200000
     };
@@ -34,6 +36,18 @@ class OpenAIProvider extends BaseProvider {
     });
   }
 
+  async listModels() {
+    try {
+      const res = await this.client.models.list();
+      const DROP = /dall-e|whisper|tts|embed|moderat|realtime|audio|transcribe|search-api|-image-|babbage|davinci-002|^sora|-instruct/i;
+      return res.data
+        .filter((m) => !DROP.test(m.id))
+        .map((m) => ({ id: m.id, name: m.id }));
+    } catch (err) {
+      throw new Error(`Failed to list OpenAI models: ${err.message || String(err)}`);
+    }
+  }
+
   isReasoningModel(model) {
     // Match exact IDs and prefix variants (gpt-5-2025-08-07 etc)
     for (const id of this.reasoningModels) {
@@ -173,31 +187,6 @@ class OpenAIProvider extends BaseProvider {
     }
   }
 
-  async analyzeImage(options = {}) {
-    const model = options.model || this.getDefaultVisionModel();
-    const b64 = BaseProvider.readImageAsBase64(options.imagePath);
-    const response = await this.client.chat.completions.create({
-      model,
-      max_tokens: options.maxTokens || 4096,
-      messages: [{
-        role: 'user',
-        content: [
-          { type: 'text', text: options.question || 'Describe this image in detail.' },
-          {
-            type: 'image_url',
-            image_url: {
-              url: `data:${options.mimeType || 'image/jpeg'};base64,${b64}`
-            }
-          }
-        ]
-      }]
-    });
-
-    return {
-      content: response.choices[0]?.message?.content || '',
-      model: response.model || model,
-    };
-  }
 }
 
 module.exports = { OpenAIProvider };

package/server/services/ai/providers/openaiCodex.js

@@ -340,8 +340,12 @@ class OpenAICodexProvider extends BaseProvider {
         request.tool_choice = 'auto';
       }
       request.parallel_tool_calls = false;
-      // reasoning: both effort and summary required for Codex reasoning models
-      if (this._isReasoningModel(model)) {
+      // Reasoning parameters are only sent when there are no tools: on the Codex
+      // backend, combining reasoning mode with function-calling causes the model to
+      // produce only internal reasoning output (no function_calls, no text), which
+      // makes the engine see steps=0 and finalResponse=no on every run.
+      const hasTools = tools && tools.length > 0;
+      if (this._isReasoningModel(model) && !hasTools) {
         const effort = options.reasoningEffort || options.reasoning_effort || 'medium';
         request.reasoning = { effort, summary: 'auto' };
         request.include = ['reasoning.encrypted_content'];

package/server/services/ai/providers/openaiCompatible.js

@@ -0,0 +1,70 @@
+const { BaseProvider } = require('./base');
+
+// Shared base for providers that speak the OpenAI Chat Completions wire format
+// (OpenAI, Grok, NVIDIA NIM, GitHub Copilot, ...). It owns the response/usage
+// normalization and vision request that were previously copy-pasted into each
+// provider. Per-provider concerns — client construction, model lists, context
+// windows, reasoning detection, and the streaming loop — stay in the subclasses,
+// since those genuinely differ between vendors.
+class OpenAICompatibleProvider extends BaseProvider {
+  normalizeUsage(usage) {
+    if (!usage) return null;
+    return {
+      promptTokens: usage.prompt_tokens ?? usage.promptTokens ?? 0,
+      completionTokens: usage.completion_tokens ?? usage.completionTokens ?? 0,
+      totalTokens: usage.total_tokens ?? usage.totalTokens ?? 0,
+    };
+  }
+
+  normalizeResponse(response) {
+    const choice = response?.choices?.[0];
+    if (!choice) {
+      throw new Error(`Provider '${this.name}' returned no choices in the response`);
+    }
+    const msg = choice.message || {};
+    return {
+      content: msg.content || '',
+      toolCalls: (msg.tool_calls || [])
+        .filter((tc) => tc?.function)
+        .map((tc) => ({
+          id: tc.id,
+          type: 'function',
+          function: { name: tc.function.name, arguments: tc.function.arguments },
+        })),
+      finishReason: choice.finish_reason,
+      usage: this.normalizeUsage(response.usage),
+    };
+  }
+
+  async analyzeImage(options = {}) {
+    if (!this.supportsVision()) {
+      throw new Error(`Provider '${this.name}' does not support image analysis`);
+    }
+
+    const model = options.model || this.getDefaultVisionModel();
+    const b64 = BaseProvider.readImageAsBase64(options.imagePath);
+    const response = await this.client.chat.completions.create({
+      model,
+      max_tokens: options.maxTokens || 4096,
+      messages: [{
+        role: 'user',
+        content: [
+          { type: 'text', text: options.question || 'Describe this image in detail.' },
+          {
+            type: 'image_url',
+            image_url: {
+              url: `data:${options.mimeType || 'image/jpeg'};base64,${b64}`,
+            },
+          },
+        ],
+      }],
+    });
+
+    return {
+      content: response.choices[0]?.message?.content || '',
+      model: response.model || model,
+    };
+  }
+}
+
+module.exports = { OpenAICompatibleProvider };

package/server/services/ai/providers/openrouter.js

@@ -0,0 +1,162 @@
+const OpenAI = require('openai');
+const { OpenAICompatibleProvider } = require('./openaiCompatible');
+
+const OPENROUTER_BASE_URL = 'https://openrouter.ai/api/v1';
+
+// Context windows fetched from the API are cached here so getContextWindow
+// can serve them without a network call at inference time.
+const contextWindowCache = new Map();
+
+class OpenRouterProvider extends OpenAICompatibleProvider {
+  constructor(config = {}) {
+    super(config);
+    this.name = 'openrouter';
+    this.models = [];
+    this.baseURL = config.baseUrl || OPENROUTER_BASE_URL;
+    this.client = new OpenAI({
+      apiKey: config.apiKey || process.env.OPENROUTER_API_KEY,
+      baseURL: this.baseURL,
+      timeout: 90_000,  // 90 s — free models can queue; avoids hanging forever
+      maxRetries: 0,    // engine handles retries; avoid SDK silently re-queuing slow models
+      defaultHeaders: {
+        'HTTP-Referer': 'https://github.com/NeoLabs-Systems/NeoAgent',
+        'X-Title': 'NeoAgent',
+      },
+    });
+  }
+
+  async listModels() {
+    const res = await fetch(`${this.baseURL}/models`, {
+      headers: { 'Authorization': `Bearer ${this.client.apiKey}` },
+    });
+    if (!res.ok) throw new Error(`OpenRouter /models returned HTTP ${res.status}`);
+    const { data } = await res.json();
+    const models = data || [];
+    for (const m of models) {
+      if (m.context_length) contextWindowCache.set(m.id, m.context_length);
+    }
+    this.models = models.map((m) => m.id);
+    return models;
+  }
+
+  getContextWindow(model) {
+    return contextWindowCache.get(model) ?? 128000;
+  }
+
+  _buildParams(model, messages, tools, options) {
+    const params = {
+      model,
+      messages,
+      temperature: options.temperature ?? 0.7,
+      max_tokens: options.maxTokens || 16384,
+    };
+
+    if (tools && tools.length > 0) {
+      params.tools = this.formatTools(tools);
+      params.tool_choice = 'auto';
+    }
+
+    return params;
+  }
+
+  _extractOpenRouterError(obj) {
+    if (!obj) return null;
+    const e = obj.error;
+    if (!e) return null;
+    const msg = e.message || (typeof e === 'string' ? e : JSON.stringify(e));
+    return `OpenRouter: ${msg}${e.code ? ` (code ${e.code})` : ''}`;
+  }
+
+  async chat(messages, tools = [], options = {}) {
+    const model = options.model || this.getDefaultModel();
+    const params = this._buildParams(model, messages, tools, options);
+    let response;
+    try {
+      response = await this.client.chat.completions.create(params);
+    } catch (err) {
+      throw new Error(`OpenRouter request failed: ${err?.message || String(err)}`);
+    }
+    // OpenRouter returns HTTP 200 even for errors (rate limits, model unavailable, etc.)
+    const orErr = this._extractOpenRouterError(response);
+    if (orErr) throw new Error(orErr);
+    if (!response?.choices?.length) {
+      throw new Error(`OpenRouter: model returned no choices (may be rate-limited or unavailable)`);
+    }
+    return this.normalizeResponse(response);
+  }
+
+  async *stream(messages, tools = [], options = {}) {
+    const model = options.model || this.getDefaultModel();
+    const params = {
+      ...this._buildParams(model, messages, tools, options),
+      stream: true,
+      stream_options: { include_usage: true },
+    };
+
+    let stream;
+    try {
+      stream = await this.client.chat.completions.create(params);
+    } catch (err) {
+      throw new Error(`OpenRouter request failed: ${err?.message || String(err)}`);
+    }
+
+    let toolCalls = [];
+    let content = '';
+    let finalUsage = null;
+
+    // The OpenAI SDK converts OpenRouter's SSE error events (data.error) into
+    // APIErrors before yielding — so we wrap the loop to add context.
+    try {
+    for await (const chunk of stream) {
+      if (chunk.usage && (!chunk.choices || chunk.choices.length === 0)) {
+        finalUsage = this.normalizeUsage(chunk.usage);
+        continue;
+      }
+
+      const delta = chunk.choices?.[0]?.delta;
+      if (!delta) continue;
+
+      if (delta.content) {
+        content += delta.content;
+        yield { type: 'content', content: delta.content };
+      }
+
+      if (delta.tool_calls) {
+        for (const tc of delta.tool_calls) {
+          if (!toolCalls[tc.index]) {
+            toolCalls[tc.index] = {
+              id: tc.id || '',
+              type: 'function',
+              function: { name: tc.function?.name || '', arguments: '' },
+            };
+          }
+          if (tc.id) toolCalls[tc.index].id = tc.id;
+          if (tc.function?.name) toolCalls[tc.index].function.name = tc.function.name;
+          if (tc.function?.arguments) toolCalls[tc.index].function.arguments += tc.function.arguments;
+        }
+      }
+
+      const finishReason = chunk.choices[0]?.finish_reason;
+      if (finishReason === 'tool_calls' || (finishReason === 'stop' && toolCalls.length > 0)) {
+        yield { type: 'tool_calls', toolCalls, content, usage: this.normalizeUsage(chunk.usage) || finalUsage };
+        return;
+      }
+      if (finishReason === 'stop') {
+        yield { type: 'done', content, usage: this.normalizeUsage(chunk.usage) || finalUsage };
+        return;
+      }
+    }
+    } catch (err) {
+      // Re-throw SDK APIErrors (from OpenRouter SSE error events) with OpenRouter context.
+      throw new Error(`OpenRouter: ${err?.message || String(err)}`);
+    }
+
+    if (toolCalls.length > 0) {
+      yield { type: 'tool_calls', toolCalls, content, usage: finalUsage };
+    } else {
+      yield { type: 'done', content, usage: finalUsage };
+    }
+  }
+}
+
+module.exports = { OpenRouterProvider };