@hivemagnus/magnus-server 0.0.0-stage → 1.0.2

package/dist/services/llm-providers/llama-cpp.js

@@ -0,0 +1,237 @@
+import { createHttpProviderError, LlmProviderError, normalizeProviderError } from './provider-error.js';
+function joinContent(content) {
+    if (typeof content === 'string')
+        return content;
+    if (Array.isArray(content)) {
+        return content
+            .map((entry) => {
+            if (typeof entry === 'string')
+                return entry;
+            if (entry && typeof entry === 'object' && 'text' in entry) {
+                return String(entry.text ?? '');
+            }
+            return '';
+        })
+            .join('');
+    }
+    return '';
+}
+export class LlamaCppProvider {
+    baseUrl;
+    id = 'llamacpp';
+    name = 'llama.cpp';
+    supportsStreaming = true;
+    supportsToolCalling = true;
+    supportsStructuredOutputs = true;
+    constructor(baseUrl = process.env['LLAMACPP_URL'] ?? process.env['VLLM_URL'] ?? 'http://127.0.0.1:8080') {
+        this.baseUrl = baseUrl;
+    }
+    isConfigured() {
+        return Boolean(this.baseUrl);
+    }
+    get apiBase() {
+        return this.baseUrl.replace(/\/+$/, '');
+    }
+    async chat(request) {
+        if (!this.isConfigured()) {
+            throw new LlmProviderError(this.id, 'configuration', 'llama.cpp is not configured', false);
+        }
+        const payload = {
+            model: request.model,
+            messages: request.messages.map((m) => {
+                if (m.role === 'assistant' && m.tool_calls?.length) {
+                    return {
+                        role: 'assistant',
+                        content: m.content || null,
+                        tool_calls: m.tool_calls.map((tc) => ({
+                            id: tc.id,
+                            type: 'function',
+                            function: {
+                                name: tc.name,
+                                arguments: JSON.stringify(tc.arguments ?? {}),
+                            },
+                        })),
+                    };
+                }
+                if (m.role === 'tool') {
+                    return { role: 'tool', content: m.content, tool_call_id: m.tool_call_id };
+                }
+                return { role: m.role, content: m.content };
+            }),
+            temperature: request.temperature ?? 0.3,
+            max_tokens: request.maxTokens ?? 4096,
+            stream: false,
+        };
+        if (request.tools?.length) {
+            payload.tools = request.tools.map((tool) => ({
+                type: tool.type,
+                function: {
+                    name: tool.function.name,
+                    description: tool.function.description,
+                    parameters: tool.function.parameters,
+                },
+            }));
+            payload.tool_choice = 'auto';
+        }
+        if (request.jsonSchema) {
+            payload.response_format = {
+                type: 'json_schema',
+                json_schema: {
+                    name: 'structured_output',
+                    schema: request.jsonSchema,
+                    strict: true,
+                },
+            };
+        }
+        try {
+            const response = await fetch(`${this.apiBase}/v1/chat/completions`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(payload),
+                signal: AbortSignal.timeout(300_000),
+            });
+            if (!response.ok) {
+                const text = await response.text().catch(() => '');
+                throw createHttpProviderError(this.id, response.status, `llama.cpp API error: ${response.status} ${text.slice(0, 300)}`);
+            }
+            const data = (await response.json());
+            const message = data.choices?.[0]?.message;
+            const nativeToolCalls = (message?.tool_calls ?? []).flatMap((tc) => {
+                const name = tc.function?.name;
+                if (!name)
+                    return [];
+                let args = {};
+                try {
+                    args = tc.function?.arguments ? JSON.parse(tc.function.arguments) : {};
+                }
+                catch {
+                    args = {};
+                }
+                return [{ id: tc.id, name, arguments: args }];
+            });
+            return {
+                providerId: this.id,
+                providerName: this.name,
+                model: data.model ?? request.model,
+                content: joinContent(message?.content),
+                thinking: '',
+                evalCount: data.usage?.total_tokens ?? 0,
+                nativeToolCalls,
+                usedStructuredOutput: Boolean(request.jsonSchema),
+            };
+        }
+        catch (error) {
+            if (error instanceof LlmProviderError)
+                throw error;
+            if (error instanceof Error && error.name === 'TimeoutError') {
+                throw new LlmProviderError(this.id, 'connectivity', 'llama.cpp request timed out', true);
+            }
+            throw normalizeProviderError(this.id, error);
+        }
+    }
+    async *streamChat(request) {
+        if (!this.isConfigured()) {
+            throw new LlmProviderError(this.id, 'configuration', 'llama.cpp is not configured', false);
+        }
+        const payload = {
+            model: request.model,
+            messages: request.messages.map((m) => ({ role: m.role, content: m.content })),
+            temperature: request.temperature ?? 0.3,
+            max_tokens: request.maxTokens ?? 4096,
+            stream: true,
+            stream_options: { include_usage: true },
+        };
+        const response = await fetch(`${this.apiBase}/v1/chat/completions`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(payload),
+            signal: AbortSignal.timeout(300_000),
+        });
+        if (!response.ok || !response.body) {
+            const text = await response.text().catch(() => '');
+            throw createHttpProviderError(this.id, response.status, `llama.cpp stream error: ${response.status} ${text.slice(0, 200)}`);
+        }
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder();
+        let buffer = '';
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            buffer += decoder.decode(value, { stream: true });
+            const lines = buffer.split('\n');
+            buffer = lines.pop() ?? '';
+            for (const line of lines) {
+                const trimmed = line.trim();
+                if (!trimmed || trimmed === 'data: [DONE]')
+                    continue;
+                if (!trimmed.startsWith('data: '))
+                    continue;
+                try {
+                    const chunk = JSON.parse(trimmed.slice(6));
+                    const delta = chunk.choices?.[0]?.delta?.content;
+                    if (delta) {
+                        yield { type: 'content', text: delta };
+                    }
+                    if (chunk.usage?.total_tokens) {
+                        yield { type: 'usage', evalCount: chunk.usage.total_tokens };
+                    }
+                }
+                catch {
+                    // skip malformed SSE lines
+                }
+            }
+        }
+        yield { type: 'done' };
+    }
+    async healthCheck() {
+        const checkedAt = new Date().toISOString();
+        if (!this.isConfigured()) {
+            return {
+                id: this.id,
+                name: this.name,
+                configured: false,
+                healthy: false,
+                baseUrl: this.baseUrl,
+                error: 'LLAMACPP_URL is not configured',
+                checkedAt,
+                supportsStreaming: this.supportsStreaming,
+                supportsToolCalling: this.supportsToolCalling,
+                supportsStructuredOutputs: this.supportsStructuredOutputs,
+            };
+        }
+        try {
+            const response = await fetch(`${this.apiBase}/v1/models`, { signal: AbortSignal.timeout(5_000) });
+            if (!response.ok) {
+                throw createHttpProviderError(this.id, response.status, `llama.cpp health check failed: ${response.status}`);
+            }
+            return {
+                id: this.id,
+                name: this.name,
+                configured: true,
+                healthy: true,
+                baseUrl: this.baseUrl,
+                checkedAt,
+                supportsStreaming: this.supportsStreaming,
+                supportsToolCalling: this.supportsToolCalling,
+                supportsStructuredOutputs: this.supportsStructuredOutputs,
+            };
+        }
+        catch (error) {
+            const normalized = normalizeProviderError(this.id, error);
+            return {
+                id: this.id,
+                name: this.name,
+                configured: true,
+                healthy: false,
+                baseUrl: this.baseUrl,
+                error: normalized.message,
+                checkedAt,
+                supportsStreaming: this.supportsStreaming,
+                supportsToolCalling: this.supportsToolCalling,
+                supportsStructuredOutputs: this.supportsStructuredOutputs,
+            };
+        }
+    }
+}
+//# sourceMappingURL=llama-cpp.js.map

package/dist/services/llm-providers/ollama.js

@@ -0,0 +1,292 @@
+import { createHttpProviderError, LlmProviderError, normalizeProviderError } from './provider-error.js';
+const THINKING_CAPABILITY_TTL_MS = 5 * 60 * 1000;
+const thinkingCapabilityCache = new Map();
+function supportsThinkingHeuristic(model) {
+    return model.includes('qwen3') || model.includes('gpt-oss') || model.includes('deepseek-r1');
+}
+function containsThinkingSignal(value) {
+    if (typeof value === 'string') {
+        return /thinking|reasoning|<think>/i.test(value);
+    }
+    if (Array.isArray(value)) {
+        return value.some((entry) => containsThinkingSignal(entry));
+    }
+    if (value && typeof value === 'object') {
+        return Object.values(value).some((entry) => containsThinkingSignal(entry));
+    }
+    return false;
+}
+export async function getOllamaThinkingCapability(baseUrl, model) {
+    const cacheKey = `${baseUrl}::${model}`;
+    const cached = thinkingCapabilityCache.get(cacheKey);
+    if (cached && (Date.now() - cached.checkedAt) < THINKING_CAPABILITY_TTL_MS) {
+        return cached.supportsThinking;
+    }
+    let supportsThinking = supportsThinkingHeuristic(model);
+    try {
+        const response = await fetch(`${baseUrl}/api/show`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ name: model }),
+            signal: AbortSignal.timeout(10_000),
+        });
+        if (response.ok) {
+            const data = (await response.json());
+            supportsThinking = containsThinkingSignal(data.capabilities)
+                || containsThinkingSignal(data.details)
+                || containsThinkingSignal(data.model_info)
+                || containsThinkingSignal(data.template)
+                || containsThinkingSignal(data.modelfile)
+                || supportsThinkingHeuristic(model);
+        }
+    }
+    catch {
+        supportsThinking = supportsThinkingHeuristic(model);
+    }
+    thinkingCapabilityCache.set(cacheKey, { checkedAt: Date.now(), supportsThinking });
+    return supportsThinking;
+}
+function supportsStructuredOutputs(model) {
+    return !model.includes('legacy');
+}
+function getThinkingInstruction(mode) {
+    switch (mode) {
+        case 'deep':
+            return 'Reasoning: high';
+        case 'standard':
+            return 'Reasoning: medium';
+        default:
+            return null;
+    }
+}
+export class OllamaProvider {
+    baseUrl;
+    id = 'ollama';
+    name = 'Ollama';
+    supportsStreaming = true;
+    supportsToolCalling = true;
+    supportsStructuredOutputs = true;
+    constructor(baseUrl = process.env['OLLAMA_URL'] ?? 'http://127.0.0.1:11434') {
+        this.baseUrl = baseUrl;
+    }
+    isConfigured() {
+        return Boolean(this.baseUrl);
+    }
+    async chat(request) {
+        const usedModel = request.model;
+        const thinkingCapable = await getOllamaThinkingCapability(this.baseUrl, usedModel);
+        const thinkingMode = thinkingCapable ? (request.thinkingMode ?? 'standard') : 'off';
+        const reasoningInstruction = getThinkingInstruction(thinkingMode);
+        const messages = reasoningInstruction
+            ? [{ role: 'system', content: reasoningInstruction }, ...request.messages]
+            : request.messages;
+        // Map images for multi-modal models
+        const ollamaMessages = messages.map((m) => {
+            const out = { role: m.role, content: m.content };
+            if (m.images?.length) {
+                out.images = m.images.map((img) => img.data);
+            }
+            if (m.tool_calls)
+                out.tool_calls = m.tool_calls;
+            if (m.tool_call_id)
+                out.tool_call_id = m.tool_call_id;
+            return out;
+        });
+        const body = {
+            model: usedModel,
+            messages: ollamaMessages,
+            stream: false,
+            options: {
+                temperature: request.temperature ?? 0.3,
+                num_predict: request.maxTokens ?? 4096,
+            },
+        };
+        if (request.tools?.length) {
+            body.tools = request.tools;
+        }
+        if (request.jsonSchema && supportsStructuredOutputs(usedModel)) {
+            body.format = request.jsonSchema;
+        }
+        try {
+            const response = await fetch(`${this.baseUrl}/api/chat`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(body),
+                signal: AbortSignal.timeout(300_000),
+            });
+            if (!response.ok) {
+                const text = await response.text().catch(() => '');
+                throw createHttpProviderError(this.id, response.status, `Ollama API error: ${response.status} ${text.slice(0, 200)}`);
+            }
+            const data = (await response.json());
+            const nativeToolCalls = (data.message?.tool_calls ?? []).map((toolCall) => ({
+                name: toolCall.function.name,
+                arguments: toolCall.function.arguments ?? {},
+            }));
+            return {
+                providerId: this.id,
+                providerName: this.name,
+                model: data.model ?? usedModel,
+                content: data.message?.content ?? '',
+                thinking: data.message?.thinking ?? '',
+                evalCount: data.eval_count ?? 0,
+                nativeToolCalls,
+                usedStructuredOutput: Boolean(request.jsonSchema) && supportsStructuredOutputs(usedModel),
+            };
+        }
+        catch (error) {
+            if (error instanceof LlmProviderError)
+                throw error;
+            if (error instanceof Error && error.name === 'TimeoutError') {
+                throw new LlmProviderError(this.id, 'connectivity', 'Ollama request timed out', true);
+            }
+            throw normalizeProviderError(this.id, error);
+        }
+    }
+    async *streamChat(request) {
+        const usedModel = request.model;
+        const thinkingCapable = await getOllamaThinkingCapability(this.baseUrl, usedModel);
+        const thinkingMode = thinkingCapable ? (request.thinkingMode ?? 'standard') : 'off';
+        const reasoningInstruction = getThinkingInstruction(thinkingMode);
+        const messages = reasoningInstruction
+            ? [{ role: 'system', content: reasoningInstruction }, ...request.messages]
+            : request.messages;
+        // Map images for multi-modal models
+        const ollamaMessages = messages.map((m) => {
+            const out = { role: m.role, content: m.content };
+            if (m.images?.length) {
+                out.images = m.images.map((img) => img.data);
+            }
+            if (m.tool_calls)
+                out.tool_calls = m.tool_calls;
+            if (m.tool_call_id)
+                out.tool_call_id = m.tool_call_id;
+            return out;
+        });
+        const response = await fetch(`${this.baseUrl}/api/chat`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({
+                model: usedModel,
+                messages: ollamaMessages,
+                tools: request.tools?.length ? request.tools : undefined,
+                stream: true,
+                options: {
+                    temperature: request.temperature ?? 0.3,
+                    num_predict: request.maxTokens ?? 4096,
+                },
+            }),
+            signal: AbortSignal.timeout(300_000),
+        });
+        if (!response.ok || !response.body) {
+            const text = await response.text().catch(() => '');
+            throw createHttpProviderError(this.id, response.status, `Ollama stream error: ${response.status} ${text.slice(0, 200)}`);
+        }
+        const reader = response.body.getReader();
+        const decoder = new TextDecoder();
+        let buffer = '';
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                break;
+            buffer += decoder.decode(value, { stream: true });
+            const lines = buffer.split('\n');
+            buffer = lines.pop() ?? '';
+            for (const line of lines) {
+                if (!line.trim())
+                    continue;
+                const chunk = JSON.parse(line);
+                if (chunk.message?.thinking) {
+                    yield { type: 'thinking', text: chunk.message.thinking };
+                }
+                if (chunk.message?.content) {
+                    yield { type: 'content', text: chunk.message.content };
+                }
+                if (chunk.message?.tool_calls?.length) {
+                    yield {
+                        type: 'tool_calls',
+                        toolCalls: chunk.message.tool_calls.map((toolCall) => ({
+                            name: toolCall.function.name,
+                            arguments: toolCall.function.arguments ?? {},
+                        })),
+                    };
+                }
+                if (chunk.eval_count) {
+                    yield { type: 'usage', evalCount: chunk.eval_count };
+                }
+            }
+        }
+        if (buffer.trim()) {
+            const chunk = JSON.parse(buffer);
+            if (chunk.message?.thinking) {
+                yield { type: 'thinking', text: chunk.message.thinking };
+            }
+            if (chunk.message?.content) {
+                yield { type: 'content', text: chunk.message.content };
+            }
+            if (chunk.message?.tool_calls?.length) {
+                yield {
+                    type: 'tool_calls',
+                    toolCalls: chunk.message.tool_calls.map((toolCall) => ({
+                        name: toolCall.function.name,
+                        arguments: toolCall.function.arguments ?? {},
+                    })),
+                };
+            }
+            if (chunk.eval_count) {
+                yield { type: 'usage', evalCount: chunk.eval_count };
+            }
+        }
+        yield { type: 'done' };
+    }
+    async healthCheck() {
+        const checkedAt = new Date().toISOString();
+        if (!this.isConfigured()) {
+            return {
+                id: this.id,
+                name: this.name,
+                configured: false,
+                healthy: false,
+                baseUrl: this.baseUrl,
+                error: 'OLLAMA_URL is not configured',
+                checkedAt,
+                supportsStreaming: this.supportsStreaming,
+                supportsToolCalling: this.supportsToolCalling,
+                supportsStructuredOutputs: this.supportsStructuredOutputs,
+            };
+        }
+        try {
+            const response = await fetch(`${this.baseUrl}/api/tags`, { signal: AbortSignal.timeout(5_000) });
+            if (!response.ok) {
+                throw createHttpProviderError(this.id, response.status, `Ollama health check failed: ${response.status}`);
+            }
+            return {
+                id: this.id,
+                name: this.name,
+                configured: true,
+                healthy: true,
+                baseUrl: this.baseUrl,
+                checkedAt,
+                supportsStreaming: this.supportsStreaming,
+                supportsToolCalling: this.supportsToolCalling,
+                supportsStructuredOutputs: this.supportsStructuredOutputs,
+            };
+        }
+        catch (error) {
+            const normalized = normalizeProviderError(this.id, error);
+            return {
+                id: this.id,
+                name: this.name,
+                configured: true,
+                healthy: false,
+                baseUrl: this.baseUrl,
+                error: normalized.message,
+                checkedAt,
+                supportsStreaming: this.supportsStreaming,
+                supportsToolCalling: this.supportsToolCalling,
+                supportsStructuredOutputs: this.supportsStructuredOutputs,
+            };
+        }
+    }
+}
+//# sourceMappingURL=ollama.js.map