llmflow 0.3.1

package/providers/cohere.js

@@ -0,0 +1,225 @@
+const BaseProvider = require('./base');
+
+/**
+ * Cohere v2 Chat API provider.
+ * 
+ * Key differences from OpenAI:
+ * - Endpoint: POST /v2/chat
+ * - Uses Bearer token authentication
+ * - Response has nested usage structure (tokens.input_tokens, tokens.output_tokens)
+ * - Assistant content is array of {type: "text", text: "..."} objects
+ * - Different finish reasons: COMPLETE, STOP_SEQUENCE, MAX_TOKENS, TOOL_CALL
+ * - Streaming uses granular event types (message-start, content-delta, message-end)
+ */
+class CohereProvider extends BaseProvider {
+    constructor(config = {}) {
+        super();
+        this.name = 'cohere';
+        this.displayName = 'Cohere';
+        this.hostname = config.hostname || 'api.cohere.com';
+    }
+
+    getTarget(req) {
+        let path = req.path;
+        
+        // Map OpenAI-style paths to Cohere paths
+        if (path === '/v1/chat/completions' || path === '/chat/completions') {
+            path = '/v2/chat';
+        }
+
+        return {
+            hostname: this.hostname,
+            port: 443,
+            path: path,
+            protocol: 'https'
+        };
+    }
+
+    transformRequestHeaders(headers, req) {
+        return {
+            'Content-Type': 'application/json',
+            'Authorization': headers.authorization,
+            'X-Client-Name': 'llmflow-proxy'
+        };
+    }
+
+    transformRequestBody(body, req) {
+        if (!body) return body;
+
+        // Cohere v2 is very similar to OpenAI format
+        // Main differences: max_tokens -> max_tokens, stop -> stop_sequences
+        const transformed = {
+            model: body.model,
+            messages: body.messages,
+            stream: body.stream || false
+        };
+
+        // Optional parameters
+        if (body.max_tokens) transformed.max_tokens = body.max_tokens;
+        if (body.temperature !== undefined) transformed.temperature = body.temperature;
+        if (body.top_p !== undefined) transformed.p = body.top_p;  // Cohere uses 'p' not 'top_p'
+        if (body.frequency_penalty !== undefined) transformed.frequency_penalty = body.frequency_penalty;
+        if (body.presence_penalty !== undefined) transformed.presence_penalty = body.presence_penalty;
+        if (body.stop) {
+            transformed.stop_sequences = Array.isArray(body.stop) ? body.stop : [body.stop];
+        }
+
+        return transformed;
+    }
+
+    normalizeResponse(body, req) {
+        if (!body || body.error) {
+            return { data: body, usage: null, model: req.body?.model };
+        }
+
+        // Extract text content from message.content array
+        let textContent = '';
+        if (body.message?.content) {
+            if (Array.isArray(body.message.content)) {
+                textContent = body.message.content
+                    .filter(c => c.type === 'text')
+                    .map(c => c.text)
+                    .join('');
+            } else if (typeof body.message.content === 'string') {
+                textContent = body.message.content;
+            }
+        }
+
+        // Map Cohere finish reasons to OpenAI format
+        const finishReasonMap = {
+            'COMPLETE': 'stop',
+            'STOP_SEQUENCE': 'stop',
+            'MAX_TOKENS': 'length',
+            'TOOL_CALL': 'tool_calls',
+            'ERROR': 'content_filter',
+            'TIMEOUT': 'content_filter'
+        };
+
+        // Extract usage - Cohere has nested structure
+        const tokens = body.usage?.tokens || {};
+        const billedUnits = body.usage?.billed_units || {};
+        const normalizedUsage = {
+            prompt_tokens: tokens.input_tokens || billedUnits.input_tokens || 0,
+            completion_tokens: tokens.output_tokens || billedUnits.output_tokens || 0,
+            total_tokens: (tokens.input_tokens || 0) + (tokens.output_tokens || 0)
+        };
+
+        // Build OpenAI-compatible response
+        const normalized = {
+            id: body.id || `cohere-${Date.now()}`,
+            object: 'chat.completion',
+            model: req.body?.model || 'command',
+            choices: [{
+                index: 0,
+                message: {
+                    role: 'assistant',
+                    content: textContent
+                },
+                finish_reason: finishReasonMap[body.finish_reason] || 'stop'
+            }],
+            usage: normalizedUsage
+        };
+
+        return {
+            data: normalized,
+            usage: normalizedUsage,
+            model: req.body?.model || 'command'
+        };
+    }
+
+    parseStreamChunk(chunk) {
+        const lines = chunk.split('\n');
+        let content = '';
+        let usage = null;
+        let done = false;
+
+        for (const line of lines) {
+            const trimmed = line.trim();
+            if (!trimmed.startsWith('data:')) continue;
+
+            const payload = trimmed.slice(5).trim();
+            if (payload === '[DONE]') {
+                done = true;
+                continue;
+            }
+
+            try {
+                const json = JSON.parse(payload);
+
+                // Handle different Cohere streaming event types
+                switch (json.type) {
+                    case 'content-delta':
+                        // Content is in delta.message.content array
+                        if (json.delta?.message?.content) {
+                            for (const c of json.delta.message.content) {
+                                if (c.type === 'text' && c.text) {
+                                    content += c.text;
+                                }
+                            }
+                        }
+                        break;
+
+                    case 'message-end':
+                        done = true;
+                        // Extract usage from message-end event
+                        if (json.delta?.usage) {
+                            const tokens = json.delta.usage.tokens || {};
+                            const billedUnits = json.delta.usage.billed_units || {};
+                            usage = {
+                                prompt_tokens: tokens.input_tokens || billedUnits.input_tokens || 0,
+                                completion_tokens: tokens.output_tokens || billedUnits.output_tokens || 0,
+                                total_tokens: (tokens.input_tokens || 0) + (tokens.output_tokens || 0)
+                            };
+                        }
+                        break;
+
+                    case 'message-start':
+                    case 'content-start':
+                    case 'content-end':
+                        // These are structural events, no content to extract
+                        break;
+                }
+            } catch {
+                // Ignore parse errors for partial chunks
+            }
+        }
+
+        return { content, usage, done };
+    }
+
+    extractUsage(response) {
+        // Handle both normalized and raw Cohere response
+        if (response.usage) {
+            // Already normalized
+            if (response.usage.prompt_tokens !== undefined) {
+                return response.usage;
+            }
+            // Raw Cohere format
+            const tokens = response.usage.tokens || {};
+            const billedUnits = response.usage.billed_units || {};
+            return {
+                prompt_tokens: tokens.input_tokens || billedUnits.input_tokens || 0,
+                completion_tokens: tokens.output_tokens || billedUnits.output_tokens || 0,
+                total_tokens: (tokens.input_tokens || 0) + (tokens.output_tokens || 0)
+            };
+        }
+
+        return { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 };
+    }
+
+    assembleStreamingResponse(fullContent, usage, req, traceId) {
+        return {
+            id: traceId,
+            object: 'chat.completion',
+            model: req.body?.model || 'command',
+            choices: [{
+                message: { role: 'assistant', content: fullContent },
+                finish_reason: 'stop'
+            }],
+            usage: usage,
+            _streaming: true
+        };
+    }
+}
+
+module.exports = CohereProvider;

package/providers/gemini.js

@@ -0,0 +1,278 @@
+const BaseProvider = require('./base');
+
+/**
+ * Google Gemini provider.
+ * Handles the unique Gemini API format with request/response transformation.
+ * 
+ * Key differences from OpenAI:
+ * - API key in query string OR Authorization header
+ * - Different endpoint structure: /v1beta/models/{model}:generateContent
+ * - Different request format (contents, systemInstruction, generationConfig)
+ * - Different response format (candidates, usageMetadata)
+ */
+class GeminiProvider extends BaseProvider {
+    constructor(config = {}) {
+        super();
+        this.name = 'gemini';
+        this.displayName = 'Google Gemini';
+        this.hostname = config.hostname || 'generativelanguage.googleapis.com';
+        this.apiVersion = config.apiVersion || 'v1beta';
+    }
+
+    getTarget(req) {
+        // Extract model from request body for endpoint construction
+        const model = req.body?.model || 'gemini-2.0-flash';
+        const isStreaming = req.body?.stream === true;
+        
+        // Gemini uses different endpoints for streaming
+        const action = isStreaming ? 'streamGenerateContent' : 'generateContent';
+        
+        // Build the path with model
+        let path = `/${this.apiVersion}/models/${model}:${action}`;
+        
+        // Add API key as query param if provided in headers
+        const apiKey = this.extractApiKey(req.headers);
+        if (apiKey) {
+            path += `?key=${apiKey}`;
+        }
+
+        return {
+            hostname: this.hostname,
+            port: 443,
+            path: path,
+            protocol: 'https'
+        };
+    }
+
+    extractApiKey(headers) {
+        if (!headers) return null;
+        
+        // Check for API key in various header formats
+        let apiKey = headers['x-goog-api-key'];
+        
+        if (!apiKey && headers.authorization) {
+            const auth = headers.authorization;
+            if (auth.startsWith('Bearer ')) {
+                apiKey = auth.slice(7);
+            }
+        }
+        
+        return apiKey;
+    }
+
+    transformRequestHeaders(headers, req) {
+        // Gemini prefers API key in URL, but we can also use header
+        const result = {
+            'Content-Type': 'application/json'
+        };
+        
+        // If using OAuth, include Authorization header
+        if (headers.authorization && !this.extractApiKey(headers)) {
+            result['Authorization'] = headers.authorization;
+        }
+        
+        return result;
+    }
+
+    transformRequestBody(body, req) {
+        if (!body) return body;
+        
+        // If already in Gemini format, pass through
+        if (body.contents) {
+            return body;
+        }
+
+        // Transform from OpenAI format to Gemini format
+        const transformed = {};
+
+        // Transform messages to contents
+        if (body.messages) {
+            const systemMessages = body.messages.filter(m => m.role === 'system');
+            const otherMessages = body.messages.filter(m => m.role !== 'system');
+
+            // System instruction
+            if (systemMessages.length > 0) {
+                transformed.systemInstruction = {
+                    parts: [{ text: systemMessages.map(m => m.content).join('\n') }]
+                };
+            }
+
+            // Contents (user/assistant messages)
+            transformed.contents = otherMessages.map(msg => ({
+                role: msg.role === 'assistant' ? 'model' : 'user',
+                parts: [{ text: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content) }]
+            }));
+        }
+
+        // Generation config
+        const generationConfig = {};
+        if (body.max_tokens) generationConfig.maxOutputTokens = body.max_tokens;
+        if (body.temperature !== undefined) generationConfig.temperature = body.temperature;
+        if (body.top_p !== undefined) generationConfig.topP = body.top_p;
+        if (body.stop) {
+            generationConfig.stopSequences = Array.isArray(body.stop) ? body.stop : [body.stop];
+        }
+        
+        if (Object.keys(generationConfig).length > 0) {
+            transformed.generationConfig = generationConfig;
+        }
+
+        return transformed;
+    }
+
+    normalizeResponse(body, req) {
+        if (!body || body.error) {
+            return { data: body, usage: null, model: req.body?.model };
+        }
+
+        // Extract text content from candidates
+        let textContent = '';
+        let finishReason = 'stop';
+        
+        if (Array.isArray(body.candidates) && body.candidates.length > 0) {
+            const candidate = body.candidates[0];
+            if (candidate.content?.parts) {
+                textContent = candidate.content.parts
+                    .filter(p => p.text)
+                    .map(p => p.text)
+                    .join('');
+            }
+            
+            // Map finish reason
+            const reasonMap = {
+                'STOP': 'stop',
+                'MAX_TOKENS': 'length',
+                'SAFETY': 'content_filter',
+                'RECITATION': 'content_filter'
+            };
+            finishReason = reasonMap[candidate.finishReason] || candidate.finishReason?.toLowerCase() || 'stop';
+        }
+
+        // Extract usage
+        const usage = body.usageMetadata || {};
+        const normalizedUsage = {
+            prompt_tokens: usage.promptTokenCount || 0,
+            completion_tokens: usage.candidatesTokenCount || 0,
+            total_tokens: usage.totalTokenCount || 0
+        };
+
+        // Build OpenAI-compatible response
+        const normalized = {
+            id: `gemini-${Date.now()}`,
+            object: 'chat.completion',
+            model: req.body?.model || 'gemini',
+            choices: [{
+                index: 0,
+                message: {
+                    role: 'assistant',
+                    content: textContent
+                },
+                finish_reason: finishReason
+            }],
+            usage: normalizedUsage
+        };
+
+        return {
+            data: normalized,
+            usage: normalizedUsage,
+            model: req.body?.model || 'gemini'
+        };
+    }
+
+    parseStreamChunk(chunk) {
+        const lines = chunk.split('\n');
+        let content = '';
+        let usage = null;
+        let done = false;
+
+        for (const line of lines) {
+            const trimmed = line.trim();
+            if (!trimmed) continue;
+
+            // Gemini streaming returns JSON array items or objects
+            try {
+                let json;
+                
+                // Handle data: prefix if present
+                if (trimmed.startsWith('data:')) {
+                    const payload = trimmed.slice(5).trim();
+                    if (payload === '[DONE]') {
+                        done = true;
+                        continue;
+                    }
+                    json = JSON.parse(payload);
+                } else if (trimmed.startsWith('[') || trimmed.startsWith('{')) {
+                    // Direct JSON response (Gemini sometimes returns array)
+                    json = JSON.parse(trimmed);
+                    if (Array.isArray(json)) {
+                        json = json[0];
+                    }
+                } else {
+                    continue;
+                }
+
+                // Extract content from candidates
+                if (json.candidates?.[0]?.content?.parts) {
+                    for (const part of json.candidates[0].content.parts) {
+                        if (part.text) content += part.text;
+                    }
+                }
+
+                // Check for usage metadata
+                if (json.usageMetadata) {
+                    usage = {
+                        prompt_tokens: json.usageMetadata.promptTokenCount || 0,
+                        completion_tokens: json.usageMetadata.candidatesTokenCount || 0,
+                        total_tokens: json.usageMetadata.totalTokenCount || 0
+                    };
+                }
+
+                // Check finish reason
+                if (json.candidates?.[0]?.finishReason) {
+                    done = true;
+                }
+            } catch {
+                // Ignore parse errors for partial chunks
+            }
+        }
+
+        return { content, usage, done };
+    }
+
+    extractUsage(response) {
+        // Handle both normalized and raw Gemini response
+        if (response.usage) {
+            return {
+                prompt_tokens: response.usage.prompt_tokens || response.usage.promptTokenCount || 0,
+                completion_tokens: response.usage.completion_tokens || response.usage.candidatesTokenCount || 0,
+                total_tokens: response.usage.total_tokens || response.usage.totalTokenCount || 0
+            };
+        }
+        
+        if (response.usageMetadata) {
+            return {
+                prompt_tokens: response.usageMetadata.promptTokenCount || 0,
+                completion_tokens: response.usageMetadata.candidatesTokenCount || 0,
+                total_tokens: response.usageMetadata.totalTokenCount || 0
+            };
+        }
+
+        return { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 };
+    }
+
+    assembleStreamingResponse(fullContent, usage, req, traceId) {
+        return {
+            id: traceId,
+            object: 'chat.completion',
+            model: req.body?.model || 'gemini',
+            choices: [{
+                message: { role: 'assistant', content: fullContent },
+                finish_reason: 'stop'
+            }],
+            usage: usage,
+            _streaming: true
+        };
+    }
+}
+
+module.exports = GeminiProvider;

package/providers/index.js

@@ -0,0 +1,130 @@
+const BaseProvider = require('./base');
+const OpenAIProvider = require('./openai');
+const OllamaProvider = require('./ollama');
+const AnthropicProvider = require('./anthropic');
+const GeminiProvider = require('./gemini');
+const CohereProvider = require('./cohere');
+const AzureOpenAIProvider = require('./azure');
+const {
+    OpenAICompatibleProvider,
+    GroqProvider,
+    MistralProvider,
+    TogetherProvider,
+    PerplexityProvider,
+    OpenRouterProvider
+} = require('./openai-compatible');
+
+/**
+ * Provider Registry
+ * Maps path prefixes to provider instances
+ */
+class ProviderRegistry {
+    constructor() {
+        this.providers = new Map();
+        this.defaultProvider = null;
+        
+        // Register default providers
+        this.registerDefaults();
+    }
+
+    registerDefaults() {
+        // Default OpenAI provider (no prefix)
+        this.defaultProvider = new OpenAIProvider();
+        
+        // Path-based providers
+        this.register('ollama', new OllamaProvider());
+        this.register('anthropic', new AnthropicProvider());
+        this.register('gemini', new GeminiProvider());
+        this.register('cohere', new CohereProvider());
+        this.register('azure', new AzureOpenAIProvider());
+        this.register('groq', GroqProvider);
+        this.register('mistral', MistralProvider);
+        this.register('together', TogetherProvider);
+        this.register('perplexity', PerplexityProvider);
+        this.register('openrouter', OpenRouterProvider);
+    }
+
+    /**
+     * Register a provider with a path prefix
+     * @param {string} prefix - URL path prefix (e.g., 'anthropic' for /anthropic/v1/...)
+     * @param {BaseProvider} provider - Provider instance
+     */
+    register(prefix, provider) {
+        this.providers.set(prefix.toLowerCase(), provider);
+    }
+
+    /**
+     * Get a provider based on request path or header
+     * @param {Object} req - Express request object
+     * @returns {{ provider: BaseProvider, cleanPath: string }}
+     */
+    resolve(req) {
+        // Check for X-LLMFlow-Provider header override
+        const headerProvider = req.headers['x-llmflow-provider'];
+        if (headerProvider && this.providers.has(headerProvider.toLowerCase())) {
+            return {
+                provider: this.providers.get(headerProvider.toLowerCase()),
+                cleanPath: req.path
+            };
+        }
+
+        // Check path prefix: /ollama/v1/... -> ollama provider
+        const pathMatch = req.path.match(/^\/([^\/]+)(\/.*)?$/);
+        if (pathMatch) {
+            const prefix = pathMatch[1].toLowerCase();
+            if (this.providers.has(prefix)) {
+                const cleanPath = pathMatch[2] || '/';
+                return {
+                    provider: this.providers.get(prefix),
+                    cleanPath: cleanPath
+                };
+            }
+        }
+
+        // Default to OpenAI
+        return {
+            provider: this.defaultProvider,
+            cleanPath: req.path
+        };
+    }
+
+    /**
+     * List all registered providers
+     * @returns {Array} List of { name, displayName, prefix }
+     */
+    list() {
+        const result = [{
+            name: this.defaultProvider.name,
+            displayName: this.defaultProvider.displayName,
+            prefix: '/v1/*',
+            default: true
+        }];
+
+        for (const [prefix, provider] of this.providers) {
+            result.push({
+                name: provider.name,
+                displayName: provider.displayName,
+                prefix: `/${prefix}/v1/*`,
+                default: false
+            });
+        }
+
+        return result;
+    }
+}
+
+// Singleton instance
+const registry = new ProviderRegistry();
+
+module.exports = {
+    registry,
+    ProviderRegistry,
+    BaseProvider,
+    OpenAIProvider,
+    OllamaProvider,
+    AnthropicProvider,
+    GeminiProvider,
+    CohereProvider,
+    AzureOpenAIProvider,
+    OpenAICompatibleProvider
+};

package/providers/ollama.js

@@ -0,0 +1,36 @@
+const BaseProvider = require('./base');
+
+/**
+ * Ollama provider - local LLM server with OpenAI-compatible API.
+ * Uses HTTP instead of HTTPS.
+ */
+class OllamaProvider extends BaseProvider {
+    constructor(config = {}) {
+        super();
+        this.name = 'ollama';
+        this.displayName = 'Ollama';
+        this.hostname = config.hostname || process.env.OLLAMA_HOST || 'localhost';
+        this.port = config.port || parseInt(process.env.OLLAMA_PORT) || 11434;
+    }
+
+    getTarget(req) {
+        return {
+            hostname: this.hostname,
+            port: this.port,
+            path: req.path,
+            protocol: 'http'
+        };
+    }
+
+    transformRequestHeaders(headers, req) {
+        return {
+            'Content-Type': 'application/json'
+        };
+    }
+
+    getHttpModule() {
+        return require('http');
+    }
+}
+
+module.exports = OllamaProvider;