@build-astron-co/nimbus 0.2.0 → 0.4.0

package/dist/src/llm/providers/ollama.js

@@ -0,0 +1,661 @@
+/**
+ * Ollama Provider
+ * Supports local Ollama models: Llama 3.2, CodeLlama, Mistral, etc.
+ */
+import { BaseProvider, getTextContent, } from '../types';
+import { getProviderBaseUrl } from '../auth-bridge';
+export class OllamaProvider extends BaseProvider {
+    name = 'ollama';
+    baseUrl;
+    defaultModel = 'llama3.2';
+    constructor(baseUrl) {
+        super();
+        this.baseUrl =
+            baseUrl ||
+                getProviderBaseUrl('ollama') ||
+                process.env.OLLAMA_BASE_URL ||
+                'http://localhost:11434';
+    }
+    async complete(request) {
+        const messages = this.convertMessages(request.messages);
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), 120000); // 2 minute timeout
+        try {
+            const response = await fetch(`${this.baseUrl}/api/chat`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    model: request.model || this.defaultModel,
+                    messages,
+                    options: {
+                        temperature: request.temperature,
+                        num_predict: request.maxTokens,
+                        stop: request.stopSequences,
+                    },
+                    stream: false,
+                }),
+                signal: controller.signal,
+            });
+            clearTimeout(timeoutId);
+            if (!response.ok) {
+                throw new Error(`Ollama API error: ${response.status} ${await response.text()}`);
+            }
+            const data = (await response.json());
+            return {
+                content: data.message.content,
+                usage: {
+                    promptTokens: data.prompt_eval_count || 0,
+                    completionTokens: data.eval_count || 0,
+                    totalTokens: (data.prompt_eval_count || 0) + (data.eval_count || 0),
+                },
+                model: data.model,
+                finishReason: 'stop',
+            };
+        }
+        catch (error) {
+            clearTimeout(timeoutId);
+            if (error.name === 'AbortError') {
+                throw new Error('Ollama request timed out after 120 seconds');
+            }
+            throw error;
+        }
+    }
+    async *stream(request) {
+        const messages = this.convertMessages(request.messages);
+        const response = await fetch(`${this.baseUrl}/api/chat`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                model: request.model || this.defaultModel,
+                messages,
+                options: {
+                    temperature: request.temperature,
+                    num_predict: request.maxTokens,
+                },
+                stream: true,
+            }),
+        });
+        if (!response.ok) {
+            throw new Error(`Ollama API error: ${response.status} ${await response.text()}`);
+        }
+        const reader = response.body?.getReader();
+        if (!reader) {
+            throw new Error('No response body');
+        }
+        const decoder = new TextDecoder();
+        let buffer = '';
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) {
+                break;
+            }
+            buffer += decoder.decode(value, { stream: true });
+            const lines = buffer.split('\n');
+            // Keep the last incomplete line in the buffer
+            buffer = lines.pop() || '';
+            for (const line of lines) {
+                if (!line.trim()) {
+                    continue;
+                }
+                try {
+                    const data = JSON.parse(line);
+                    if (data.message?.content) {
+                        yield {
+                            content: data.message.content,
+                            done: false,
+                        };
+                    }
+                    if (data.done) {
+                        yield { done: true };
+                    }
+                }
+                catch (e) {
+                    // Ignore malformed JSON lines
+                    continue;
+                }
+            }
+        }
+    }
+    async completeWithTools(request) {
+        // Try native Ollama tool calling first (supported by llama3.x, mistral, etc.)
+        try {
+            const nativeResult = await this.completeWithNativeTools(request);
+            if (nativeResult) {
+                return nativeResult;
+            }
+        }
+        catch {
+            // Native tool calling not supported by this model — fall through
+        }
+        // Fallback: prompt engineering approach for models without native tool support
+        const toolPrompt = this.buildToolPrompt(request.tools);
+        const modifiedRequest = {
+            ...request,
+            messages: [{ role: 'system', content: toolPrompt }, ...request.messages],
+        };
+        const response = await this.complete(modifiedRequest);
+        // Attempt to extract tool calls from the text response. If extraction
+        // fails for any reason (malformed JSON, unexpected structure, etc.),
+        // return the raw text so the agent loop can still display it.
+        let toolCalls = [];
+        try {
+            toolCalls = this.extractToolCalls(response.content);
+        }
+        catch {
+            // Extraction failed -- return the original text as a plain response
+            return response;
+        }
+        return {
+            ...response,
+            content: toolCalls.length > 0 ? '' : response.content,
+            toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
+        };
+    }
+    /**
+     * Attempt native Ollama tool calling via the /api/chat tools parameter.
+     * Returns null if the model doesn't support tools (response has no tool_calls).
+     */
+    async completeWithNativeTools(request) {
+        const messages = this.convertMessages(request.messages);
+        const tools = request.tools.map(t => ({
+            type: 'function',
+            function: {
+                name: t.function.name,
+                description: t.function.description,
+                parameters: t.function.parameters,
+            },
+        }));
+        const controller = new AbortController();
+        const timeoutId = setTimeout(() => controller.abort(), 120000);
+        try {
+            const response = await fetch(`${this.baseUrl}/api/chat`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({
+                    model: request.model || this.defaultModel,
+                    messages,
+                    tools,
+                    options: {
+                        temperature: request.temperature,
+                        num_predict: request.maxTokens,
+                    },
+                    stream: false,
+                }),
+                signal: controller.signal,
+            });
+            clearTimeout(timeoutId);
+            if (!response.ok) {
+                return null;
+            }
+            const data = (await response.json());
+            const msg = data.message;
+            // If no tool_calls in response, native tools aren't working
+            if (!msg?.tool_calls || msg.tool_calls.length === 0) {
+                return null;
+            }
+            const toolCalls = msg.tool_calls.map(tc => ({
+                id: crypto.randomUUID(),
+                type: 'function',
+                function: {
+                    name: tc.function.name,
+                    arguments: JSON.stringify(tc.function.arguments),
+                },
+            }));
+            return {
+                content: msg.content || '',
+                toolCalls,
+                usage: {
+                    promptTokens: data.prompt_eval_count || 0,
+                    completionTokens: data.eval_count || 0,
+                    totalTokens: (data.prompt_eval_count || 0) + (data.eval_count || 0),
+                },
+                model: data.model,
+                finishReason: 'tool_calls',
+            };
+        }
+        catch {
+            clearTimeout(timeoutId);
+            return null;
+        }
+    }
+    /**
+     * Stream a chat completion with tool calling support.
+     * Attempts native Ollama tool streaming via the OpenAI-compatible endpoint first.
+     * Falls back to the non-streaming completeWithTools for models without native
+     * tool support, yielding the result as a single chunk.
+     */
+    async *streamWithTools(request) {
+        // Try native streaming with tools via the OpenAI-compatible endpoint
+        try {
+            const nativeStream = this.streamWithNativeTools(request);
+            let gotToolCalls = false;
+            for await (const chunk of nativeStream) {
+                if (chunk.toolCalls && chunk.toolCalls.length > 0) {
+                    gotToolCalls = true;
+                }
+                yield chunk;
+            }
+            // If we successfully streamed (even without tool calls), we're done
+            if (gotToolCalls) {
+                return;
+            }
+            // If the native stream completed without any tool calls, it may mean
+            // the model answered with text only, which is a valid response — return.
+            return;
+        }
+        catch {
+            // Native streaming not supported — fall through to fallback
+        }
+        // Fallback: use non-streaming completeWithTools and yield as a single chunk
+        const response = await this.completeWithTools(request);
+        if (response.content) {
+            yield { content: response.content, done: false };
+        }
+        yield {
+            done: true,
+            toolCalls: response.toolCalls,
+            usage: response.usage,
+        };
+    }
+    /**
+     * Stream with native tool calling via the Ollama OpenAI-compatible endpoint.
+     * Uses fetch + SSE parsing against /v1/chat/completions with stream: true.
+     */
+    async *streamWithNativeTools(request) {
+        const messages = this.convertMessages(request.messages);
+        const tools = request.tools.map(t => ({
+            type: 'function',
+            function: {
+                name: t.function.name,
+                description: t.function.description,
+                parameters: t.function.parameters,
+            },
+        }));
+        const response = await fetch(`${this.baseUrl}/v1/chat/completions`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                model: request.model || this.defaultModel,
+                messages,
+                tools,
+                tool_choice: request.toolChoice || 'auto',
+                max_tokens: request.maxTokens,
+                temperature: request.temperature,
+                stream: true,
+            }),
+        });
+        if (!response.ok) {
+            throw new Error(`Ollama OpenAI-compatible API error: ${response.status} ${await response.text()}`);
+        }
+        const reader = response.body?.getReader();
+        if (!reader) {
+            throw new Error('No response body');
+        }
+        const decoder = new TextDecoder();
+        let buffer = '';
+        const toolCallAccumulator = new Map();
+        let usage;
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done) {
+                break;
+            }
+            buffer += decoder.decode(value, { stream: true });
+            const lines = buffer.split('\n');
+            // Keep the last incomplete line in the buffer
+            buffer = lines.pop() || '';
+            for (const line of lines) {
+                const trimmed = line.trim();
+                if (!trimmed || trimmed === 'data: [DONE]') {
+                    continue;
+                }
+                if (!trimmed.startsWith('data: ')) {
+                    continue;
+                }
+                try {
+                    const data = JSON.parse(trimmed.slice(6));
+                    const delta = data.choices?.[0]?.delta;
+                    const finishReason = data.choices?.[0]?.finish_reason;
+                    if (delta?.content) {
+                        yield { content: delta.content, done: false };
+                    }
+                    if (delta?.tool_calls) {
+                        for (const tc of delta.tool_calls) {
+                            const index = tc.index ?? 0;
+                            const existing = toolCallAccumulator.get(index);
+                            if (existing) {
+                                if (tc.function?.arguments) {
+                                    existing.arguments += tc.function.arguments;
+                                }
+                            }
+                            else {
+                                toolCallAccumulator.set(index, {
+                                    id: tc.id || crypto.randomUUID(),
+                                    name: tc.function?.name || '',
+                                    arguments: tc.function?.arguments || '',
+                                });
+                            }
+                        }
+                    }
+                    if (data.usage) {
+                        usage = {
+                            promptTokens: data.usage.prompt_tokens || 0,
+                            completionTokens: data.usage.completion_tokens || 0,
+                            totalTokens: data.usage.total_tokens || 0,
+                        };
+                    }
+                    if (finishReason) {
+                        const toolCalls = toolCallAccumulator.size > 0
+                            ? Array.from(toolCallAccumulator.values()).map(tc => ({
+                                id: tc.id,
+                                type: 'function',
+                                function: { name: tc.name, arguments: tc.arguments },
+                            }))
+                            : undefined;
+                        yield { done: true, toolCalls, usage };
+                        return;
+                    }
+                }
+                catch {
+                    // Ignore malformed SSE lines
+                    continue;
+                }
+            }
+        }
+        // If we reach here without a finishReason, emit a final done chunk
+        const toolCalls = toolCallAccumulator.size > 0
+            ? Array.from(toolCallAccumulator.values()).map(tc => ({
+                id: tc.id,
+                type: 'function',
+                function: { name: tc.name, arguments: tc.arguments },
+            }))
+            : undefined;
+        yield { done: true, toolCalls, usage };
+    }
+    async countTokens(text) {
+        try {
+            // Use gpt-tokenizer (already a dependency) for better approximation
+            const { encode } = await import('gpt-tokenizer');
+            return encode(text).length;
+        }
+        catch {
+            // Fallback to approximation if tokenizer fails
+            return Math.ceil(text.length / 4);
+        }
+    }
+    getMaxTokens(model) {
+        // Check the runtime cache first (populated by getModelInfo)
+        const cached = this.modelContextCache.get(model);
+        if (cached !== undefined) {
+            return cached;
+        }
+        const limits = {
+            // Llama 3.x family
+            llama3: 8192,
+            'llama3.1': 131072,
+            'llama3.1:8b': 131072,
+            'llama3.1:70b': 131072,
+            'llama3.1:405b': 131072,
+            'llama3.2': 131072,
+            'llama3.2:1b': 131072,
+            'llama3.2:3b': 131072,
+            'llama3.2:70b': 131072,
+            'llama3.3': 131072,
+            'llama3.3:70b': 131072,
+            // Llama 2 family
+            llama2: 4096,
+            'llama2:13b': 4096,
+            'llama2:70b': 4096,
+            // CodeLlama
+            codellama: 16384,
+            'codellama:7b': 16384,
+            'codellama:13b': 16384,
+            'codellama:34b': 16384,
+            'codellama:70b': 16384,
+            // Mistral / Mixtral
+            mistral: 32768,
+            'mistral:7b': 32768,
+            'mistral-nemo': 128000,
+            'mistral-large': 128000,
+            'mistral-small': 32768,
+            mixtral: 32768,
+            'mixtral:8x7b': 32768,
+            'mixtral:8x22b': 65536,
+            // Phi family
+            phi: 2048,
+            phi3: 131072,
+            'phi3:mini': 131072,
+            'phi3:medium': 131072,
+            'phi3.5': 131072,
+            phi4: 16384,
+            // Gemma family
+            gemma: 8192,
+            'gemma:2b': 8192,
+            'gemma:7b': 8192,
+            gemma2: 8192,
+            'gemma2:2b': 8192,
+            'gemma2:9b': 8192,
+            'gemma2:27b': 8192,
+            // Qwen family
+            qwen: 32768,
+            qwen2: 131072,
+            'qwen2:7b': 131072,
+            'qwen2:72b': 131072,
+            'qwen2.5': 131072,
+            'qwen2.5:7b': 131072,
+            'qwen2.5:14b': 131072,
+            'qwen2.5:32b': 131072,
+            'qwen2.5:72b': 131072,
+            'qwen2.5-coder': 131072,
+            // DeepSeek family
+            'deepseek-coder': 16384,
+            'deepseek-coder-v2': 131072,
+            'deepseek-r1': 131072,
+            // StarCoder
+            starcoder: 8192,
+            starcoder2: 16384,
+            // Command R
+            'command-r': 131072,
+            'command-r-plus': 131072,
+            // Others
+            'nomic-embed-text': 8192,
+            'mxbai-embed-large': 512,
+            yi: 200000,
+            solar: 4096,
+            'neural-chat': 4096,
+            'wizard-vicuna': 4096,
+        };
+        // Handle model variants (e.g., "llama3.2:7b-instruct" -> "llama3.2:7b" -> "llama3.2")
+        if (limits[model]) {
+            return limits[model];
+        }
+        const baseModel = model.split(':')[0];
+        return limits[baseModel] || 8192;
+    }
+    /**
+     * Query the Ollama `/api/show` endpoint to retrieve the actual model
+     * context length from the running server. Results are cached in memory
+     * so repeated calls for the same model are free.
+     *
+     * If the call fails (Ollama not running, model not pulled, etc.) the
+     * method returns `null` and the caller should fall back to the
+     * hardcoded map in {@link getMaxTokens}.
+     */
+    async getModelInfo(model) {
+        const cached = this.modelContextCache.get(model);
+        if (cached !== undefined) {
+            return cached;
+        }
+        try {
+            const response = await fetch(`${this.baseUrl}/api/show`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ name: model }),
+                signal: AbortSignal.timeout(5000),
+            });
+            if (!response.ok) {
+                return null;
+            }
+            const data = (await response.json());
+            // Strategy 1: Check model_info for context_length or num_ctx keys
+            if (data.model_info) {
+                for (const [key, value] of Object.entries(data.model_info)) {
+                    if ((key.includes('context_length') || key.includes('num_ctx')) &&
+                        typeof value === 'number' &&
+                        value > 0) {
+                        this.modelContextCache.set(model, value);
+                        return value;
+                    }
+                }
+            }
+            // Strategy 2: Parse the parameters string for num_ctx
+            if (data.parameters) {
+                const match = data.parameters.match(/num_ctx\s+(\d+)/);
+                if (match) {
+                    const numCtx = parseInt(match[1], 10);
+                    if (numCtx > 0) {
+                        this.modelContextCache.set(model, numCtx);
+                        return numCtx;
+                    }
+                }
+            }
+            return null;
+        }
+        catch {
+            return null;
+        }
+    }
+    /** In-memory cache for model context lengths retrieved via /api/show. */
+    modelContextCache = new Map();
+    async listModels() {
+        try {
+            const response = await fetch(`${this.baseUrl}/api/tags`, {
+                signal: AbortSignal.timeout(3000),
+            });
+            if (response.ok) {
+                const data = (await response.json());
+                if (data.models && Array.isArray(data.models)) {
+                    return data.models.map(m => m.name || m.model || '').filter(Boolean);
+                }
+            }
+        }
+        catch {
+            // Ollama not available, return static fallback
+        }
+        return ['llama3.2', 'llama3.2:70b', 'codellama', 'mistral', 'mixtral', 'phi'];
+    }
+    /**
+     * Convert messages to Ollama format
+     */
+    convertMessages(messages) {
+        return messages.map(m => ({
+            role: m.role === 'assistant' ? 'assistant' : m.role === 'system' ? 'system' : 'user',
+            content: getTextContent(m.content),
+        }));
+    }
+    /**
+     * Build tool prompt for prompt engineering approach
+     */
+    buildToolPrompt(tools) {
+        const toolDescriptions = tools
+            .map(t => {
+            const params = t.function.parameters;
+            const required = params.required || [];
+            const props = params.properties || {};
+            const paramList = Object.entries(props)
+                .map(([k, v]) => {
+                const req = required.includes(k) ? ' (required)' : ' (optional)';
+                return `    "${k}": ${v.type || 'string'}${req} — ${v.description || ''}`;
+            })
+                .join('\n');
+            return `- ${t.function.name}: ${t.function.description}\n  Parameters:\n${paramList}`;
+        })
+            .join('\n\n');
+        return `You are a helpful AI assistant with access to tools. When you need to use a tool, respond with ONLY a JSON object in this exact format (no other text before or after):
+
+\`\`\`json
+{"tool": "tool_name", "arguments": {"param1": "value1"}}
+\`\`\`
+
+Available tools:
+
+${toolDescriptions}
+
+IMPORTANT: When using a tool, output ONLY the JSON object. Do not include any explanation text before or after it. If you want to respond without using a tool, respond normally with plain text (no JSON).`;
+    }
+    /**
+     * Extract tool calls from response content using robust pattern matching.
+     * Handles JSON in code blocks, bare JSON, and multiple tool calls.
+     */
+    extractToolCalls(content) {
+        const toolCalls = [];
+        // Strategy 1: Look for JSON in code blocks
+        const codeBlockRegex = /```(?:json)?\s*\n?([\s\S]*?)\n?```/g;
+        let match;
+        while ((match = codeBlockRegex.exec(content)) !== null) {
+            const parsed = this.tryParseToolCall(match[1].trim());
+            if (parsed) {
+                toolCalls.push(parsed);
+            }
+        }
+        if (toolCalls.length > 0) {
+            return toolCalls;
+        }
+        // Strategy 2: Try to parse the entire content as JSON
+        const wholeContent = content.trim();
+        if (wholeContent.startsWith('{')) {
+            const parsed = this.tryParseToolCall(wholeContent);
+            if (parsed) {
+                return [parsed];
+            }
+        }
+        // Strategy 3: Find JSON objects using balanced brace matching
+        let depth = 0;
+        let start = -1;
+        for (let i = 0; i < content.length; i++) {
+            if (content[i] === '{') {
+                if (depth === 0) {
+                    start = i;
+                }
+                depth++;
+            }
+            else if (content[i] === '}') {
+                depth--;
+                if (depth === 0 && start !== -1) {
+                    const candidate = content.slice(start, i + 1);
+                    const parsed = this.tryParseToolCall(candidate);
+                    if (parsed) {
+                        toolCalls.push(parsed);
+                    }
+                    start = -1;
+                }
+            }
+        }
+        return toolCalls;
+    }
+    /**
+     * Try to parse a string as a tool call JSON object.
+     */
+    tryParseToolCall(text) {
+        try {
+            const parsed = JSON.parse(text);
+            if (parsed.tool &&
+                typeof parsed.tool === 'string' &&
+                parsed.arguments &&
+                typeof parsed.arguments === 'object') {
+                return {
+                    id: crypto.randomUUID(),
+                    type: 'function',
+                    function: {
+                        name: parsed.tool,
+                        arguments: JSON.stringify(parsed.arguments),
+                    },
+                };
+            }
+        }
+        catch {
+            // Not valid JSON
+        }
+        return null;
+    }
+}