agentic-flow 1.0.0

package/dist/router/providers/onnx.js

@@ -0,0 +1,242 @@
+/**
+ * ONNX Runtime Provider for Local Model Inference
+ *
+ * Supports CPU and GPU execution providers for optimized local inference
+ * Compatible with Phi-3, Llama, and other ONNX models
+ */
+import { pipeline, env } from '@xenova/transformers';
+env.allowLocalModels = true;
+env.allowRemoteModels = true;
+export class ONNXProvider {
+    name = 'onnx';
+    type = 'custom';
+    supportsStreaming = true;
+    supportsTools = false;
+    supportsMCP = false;
+    session = null;
+    generator = null;
+    config;
+    executionProviders = [];
+    constructor(config = {}) {
+        this.config = {
+            modelId: config.modelId || 'Xenova/Phi-3-mini-4k-instruct',
+            maxTokens: config.maxTokens || 512,
+            temperature: config.temperature || 0.7,
+            ...config
+        };
+    }
+    /**
+     * Detect available execution providers
+     */
+    async detectExecutionProviders() {
+        const providers = [];
+        // Try CUDA for NVIDIA GPUs
+        try {
+            if (process.platform === 'linux') {
+                providers.push('cuda');
+                this.executionProviders.push('cuda');
+            }
+        }
+        catch (e) {
+            // CUDA not available
+        }
+        // Try DirectML for Windows GPUs
+        try {
+            if (process.platform === 'win32') {
+                providers.push('dml');
+                this.executionProviders.push('dml');
+            }
+        }
+        catch (e) {
+            // DirectML not available
+        }
+        // Always fallback to CPU
+        providers.push('cpu');
+        this.executionProviders.push('cpu');
+        console.log(`🔧 ONNX Execution Providers: ${this.executionProviders.join(', ')}`);
+        return providers;
+    }
+    /**
+     * Initialize ONNX session with model
+     */
+    async initializeSession() {
+        if (this.generator)
+            return;
+        try {
+            console.log(`📦 Loading ONNX model: ${this.config.modelId}`);
+            // Use Transformers.js for easier model loading
+            this.generator = await pipeline('text-generation', this.config.modelId, {
+                quantized: true, // Use quantized models for better CPU performance
+            });
+            console.log(`✅ ONNX model loaded successfully`);
+        }
+        catch (error) {
+            const providerError = {
+                name: 'ONNXInitError',
+                message: `Failed to initialize ONNX model: ${error}`,
+                provider: 'onnx',
+                retryable: false
+            };
+            throw providerError;
+        }
+    }
+    /**
+     * Format messages for model input
+     */
+    formatMessages(messages) {
+        // Simple chat template for Phi-3
+        let prompt = '';
+        for (const msg of messages) {
+            const content = typeof msg.content === 'string'
+                ? msg.content
+                : msg.content.map(c => c.type === 'text' ? c.text : '').join('');
+            if (msg.role === 'user') {
+                prompt += `<|user|>\n${content}<|end|>\n`;
+            }
+            else if (msg.role === 'assistant') {
+                prompt += `<|assistant|>\n${content}<|end|>\n`;
+            }
+            else if (msg.role === 'system') {
+                prompt += `<|system|>\n${content}<|end|>\n`;
+            }
+        }
+        prompt += '<|assistant|>\n';
+        return prompt;
+    }
+    /**
+     * Chat completion
+     */
+    async chat(params) {
+        await this.initializeSession();
+        const startTime = Date.now();
+        const prompt = this.formatMessages(params.messages);
+        try {
+            const result = await this.generator(prompt, {
+                max_new_tokens: params.maxTokens || this.config.maxTokens,
+                temperature: params.temperature || this.config.temperature,
+                do_sample: true,
+                top_p: 0.9,
+            });
+            const generatedText = result[0].generated_text;
+            // Extract only the new assistant response
+            const assistantResponse = generatedText
+                .split('<|assistant|>')
+                .pop()
+                ?.split('<|end|>')[0]
+                ?.trim() || '';
+            const latency = Date.now() - startTime;
+            // Estimate token counts (rough approximation)
+            const inputTokens = Math.ceil(prompt.length / 4);
+            const outputTokens = Math.ceil(assistantResponse.length / 4);
+            const content = [{
+                    type: 'text',
+                    text: assistantResponse
+                }];
+            return {
+                id: `onnx-${Date.now()}`,
+                model: this.config.modelId || 'onnx-model',
+                content,
+                stopReason: 'end_turn',
+                usage: {
+                    inputTokens,
+                    outputTokens
+                },
+                metadata: {
+                    provider: 'onnx',
+                    model: this.config.modelId,
+                    latency,
+                    cost: 0, // Local inference is free
+                    executionProviders: this.executionProviders
+                }
+            };
+        }
+        catch (error) {
+            const providerError = {
+                name: 'ONNXInferenceError',
+                message: `ONNX inference failed: ${error}`,
+                provider: 'onnx',
+                retryable: true
+            };
+            throw providerError;
+        }
+    }
+    /**
+     * Streaming generation
+     */
+    async *stream(params) {
+        await this.initializeSession();
+        const prompt = this.formatMessages(params.messages);
+        try {
+            // Note: Transformers.js doesn't natively support streaming
+            // We'll simulate it by yielding tokens as they're generated
+            const result = await this.generator(prompt, {
+                max_new_tokens: params.maxTokens || this.config.maxTokens,
+                temperature: params.temperature || this.config.temperature,
+                do_sample: true,
+                top_p: 0.9,
+            });
+            const generatedText = result[0].generated_text;
+            const assistantResponse = generatedText
+                .split('<|assistant|>')
+                .pop()
+                ?.split('<|end|>')[0]
+                ?.trim() || '';
+            // Simulate streaming by chunking the response
+            const words = assistantResponse.split(' ');
+            for (let i = 0; i < words.length; i++) {
+                const chunk = words[i] + (i < words.length - 1 ? ' ' : '');
+                yield {
+                    type: 'content_block_delta',
+                    delta: {
+                        type: 'text_delta',
+                        text: chunk
+                    }
+                };
+                // Small delay to simulate real streaming
+                await new Promise(resolve => setTimeout(resolve, 10));
+            }
+            yield {
+                type: 'message_stop'
+            };
+        }
+        catch (error) {
+            const providerError = {
+                name: 'ONNXStreamError',
+                message: `ONNX streaming failed: ${error}`,
+                provider: 'onnx',
+                retryable: true
+            };
+            throw providerError;
+        }
+    }
+    /**
+     * Validate capabilities
+     */
+    validateCapabilities(features) {
+        const supported = ['chat', 'stream'];
+        return features.every(f => supported.includes(f));
+    }
+    /**
+     * Get model info
+     */
+    getModelInfo() {
+        return {
+            modelId: this.config.modelId,
+            executionProviders: this.executionProviders,
+            supportsGPU: this.executionProviders.includes('cuda') || this.executionProviders.includes('dml'),
+            initialized: this.generator !== null
+        };
+    }
+    /**
+     * Cleanup resources
+     */
+    async dispose() {
+        if (this.generator) {
+            this.generator = null;
+        }
+        if (this.session) {
+            await this.session.release();
+            this.session = null;
+        }
+    }
+}

package/dist/router/providers/openrouter.js

@@ -0,0 +1,242 @@
+// OpenRouter provider implementation
+import axios from 'axios';
+export class OpenRouterProvider {
+    name = 'openrouter';
+    type = 'openrouter';
+    supportsStreaming = true;
+    supportsTools = true;
+    supportsMCP = false; // Requires translation
+    client;
+    config;
+    constructor(config) {
+        this.config = config;
+        if (!config.apiKey) {
+            throw new Error('OpenRouter API key is required');
+        }
+        this.client = axios.create({
+            baseURL: config.baseUrl || 'https://openrouter.ai/api/v1',
+            headers: {
+                'Authorization': `Bearer ${config.apiKey}`,
+                'HTTP-Referer': 'https://github.com/ruvnet/agentic-flow',
+                'X-Title': 'Agentic Flow Multi-Model Router',
+                'Content-Type': 'application/json'
+            },
+            timeout: config.timeout || 180000
+        });
+    }
+    validateCapabilities(features) {
+        const supported = ['chat', 'streaming', 'tools'];
+        return features.every(f => supported.includes(f));
+    }
+    async chat(params) {
+        try {
+            const requestBody = this.formatRequest(params);
+            const response = await this.client.post('/chat/completions', requestBody);
+            return this.formatResponse(response.data, params.model);
+        }
+        catch (error) {
+            throw this.handleError(error);
+        }
+    }
+    async *stream(params) {
+        try {
+            const requestBody = this.formatRequest(params, true);
+            const response = await this.client.post('/chat/completions', requestBody, {
+                responseType: 'stream'
+            });
+            for await (const chunk of response.data) {
+                const lines = chunk.toString().split('\n').filter((line) => line.trim() !== '');
+                for (const line of lines) {
+                    if (line.startsWith('data: ')) {
+                        const data = line.slice(6);
+                        if (data === '[DONE]') {
+                            yield { type: 'message_stop' };
+                            return;
+                        }
+                        try {
+                            const parsed = JSON.parse(data);
+                            const streamChunk = this.formatStreamChunk(parsed);
+                            if (streamChunk) {
+                                yield streamChunk;
+                            }
+                        }
+                        catch (e) {
+                            // Skip invalid JSON
+                        }
+                    }
+                }
+            }
+        }
+        catch (error) {
+            throw this.handleError(error);
+        }
+    }
+    formatRequest(params, stream = false) {
+        const messages = params.messages.map(msg => ({
+            role: msg.role,
+            content: typeof msg.content === 'string'
+                ? msg.content
+                : msg.content.map(block => {
+                    if (block.type === 'text')
+                        return { type: 'text', text: block.text };
+                    if (block.type === 'tool_use')
+                        return {
+                            type: 'function',
+                            function: {
+                                name: block.name,
+                                arguments: JSON.stringify(block.input)
+                            }
+                        };
+                    if (block.type === 'tool_result')
+                        return {
+                            type: 'function_result',
+                            content: block.content,
+                            is_error: block.is_error
+                        };
+                    return block;
+                })
+        }));
+        const body = {
+            model: params.model,
+            messages,
+            temperature: params.temperature ?? 0.7,
+            max_tokens: params.maxTokens,
+            stream
+        };
+        // Add tools if provided
+        if (params.tools && params.tools.length > 0) {
+            body.tools = params.tools.map(tool => ({
+                type: 'function',
+                function: {
+                    name: tool.name,
+                    description: tool.description,
+                    parameters: tool.input_schema
+                }
+            }));
+            if (params.toolChoice) {
+                if (params.toolChoice === 'auto' || params.toolChoice === 'none') {
+                    body.tool_choice = params.toolChoice;
+                }
+                else if (typeof params.toolChoice === 'object') {
+                    body.tool_choice = {
+                        type: 'function',
+                        function: { name: params.toolChoice.name }
+                    };
+                }
+            }
+        }
+        // Add OpenRouter-specific preferences
+        if (this.config.preferences) {
+            if (this.config.preferences.requireParameters) {
+                body.require_parameters = true;
+            }
+            if (this.config.preferences.dataCollection) {
+                body['X-Data-Collection'] = this.config.preferences.dataCollection;
+            }
+            if (this.config.preferences.order) {
+                body.provider = {
+                    order: this.config.preferences.order
+                };
+            }
+        }
+        return body;
+    }
+    formatResponse(data, model) {
+        const choice = data.choices[0];
+        const message = choice.message;
+        const content = [];
+        // Handle text content
+        if (message.content) {
+            content.push({
+                type: 'text',
+                text: message.content
+            });
+        }
+        // Handle tool calls
+        if (message.tool_calls) {
+            for (const toolCall of message.tool_calls) {
+                content.push({
+                    type: 'tool_use',
+                    id: toolCall.id,
+                    name: toolCall.function.name,
+                    input: JSON.parse(toolCall.function.arguments)
+                });
+            }
+        }
+        const stopReason = this.mapFinishReason(choice.finish_reason);
+        return {
+            id: data.id,
+            model: data.model || model,
+            content,
+            stopReason,
+            usage: {
+                inputTokens: data.usage?.prompt_tokens || 0,
+                outputTokens: data.usage?.completion_tokens || 0
+            },
+            metadata: {
+                provider: 'openrouter',
+                cost: this.calculateCost(data.usage),
+                latency: 0 // Will be set by router
+            }
+        };
+    }
+    formatStreamChunk(data) {
+        const choice = data.choices?.[0];
+        if (!choice)
+            return null;
+        const delta = choice.delta;
+        if (delta?.content) {
+            return {
+                type: 'content_block_delta',
+                delta: {
+                    type: 'text_delta',
+                    text: delta.content
+                }
+            };
+        }
+        if (delta?.tool_calls) {
+            const toolCall = delta.tool_calls[0];
+            return {
+                type: 'content_block_delta',
+                delta: {
+                    type: 'input_json_delta',
+                    partial_json: toolCall.function?.arguments || ''
+                }
+            };
+        }
+        if (choice.finish_reason) {
+            return {
+                type: 'message_stop',
+                usage: data.usage ? {
+                    inputTokens: data.usage.prompt_tokens || 0,
+                    outputTokens: data.usage.completion_tokens || 0
+                } : undefined
+            };
+        }
+        return null;
+    }
+    mapFinishReason(reason) {
+        switch (reason) {
+            case 'stop': return 'end_turn';
+            case 'length': return 'max_tokens';
+            case 'tool_calls': return 'tool_use';
+            default: return 'end_turn';
+        }
+    }
+    calculateCost(usage) {
+        if (!usage)
+            return 0;
+        // OpenRouter pricing varies by model
+        // Using average pricing: $0.01/1K input, $0.03/1K output
+        const inputCost = (usage.prompt_tokens || 0) * 0.00001;
+        const outputCost = (usage.completion_tokens || 0) * 0.00003;
+        return inputCost + outputCost;
+    }
+    handleError(error) {
+        const providerError = new Error(error.response?.data?.error?.message || error.message || 'OpenRouter request failed');
+        providerError.provider = 'openrouter';
+        providerError.statusCode = error.response?.status;
+        providerError.retryable = error.response?.status >= 500 || error.code === 'ECONNABORTED';
+        return providerError;
+    }
+}