adaptive-memory-multi-model-router 1.2.2 → 1.3.1

package/src/providers/cerebras.py

@@ -0,0 +1,373 @@
+"""
+Cost-Optimized Providers - Agent 3 Output
+
+Implements Cerebras and Groq providers for cost-effective inference.
+"""
+
+import os
+import time
+from typing import Dict, Any
+from datetime import datetime
+import asyncio
+
+from .base import BaseProvider, ProviderConfig, ProviderResponse, ProviderHealth
+
+
+class CerebrasProvider(BaseProvider):
+    """
+    Cerebras provider - Extremely fast LLaMA inference.
+
+    Key advantage: 10x faster than standard inference, very low cost.
+    Ideal for: Simple tasks, prototyping, high-volume requests.
+    """
+
+    # Pricing (per 1M tokens - significantly cheaper)
+    PRICING = {
+        "llama-3.3-70b": {"input": 0.10, "output": 0.10},
+        "llama-3.1-8b": {"input": 0.05, "output": 0.05},
+    }
+
+    def __init__(self, config: ProviderConfig):
+        super().__init__(config)
+        self.api_key = os.getenv(config.api_key_env)
+
+        if not self.api_key:
+            raise ValueError(f"API key not found: {config.api_key_env}")
+
+    async def execute(
+        self,
+        prompt: str,
+        max_tokens: int = 2000,
+        temperature: float = 0.7,
+        **kwargs
+    ) -> ProviderResponse:
+        """
+        Execute prompt with Cerebras (LLaMA on Cerebras hardware).
+
+        Note: Placeholder implementation.
+        In production, use actual Cerebras SDK or API.
+        """
+        start_time = time.time()
+
+        try:
+            # Simulate Cerebras fast inference (0.1s - much faster!)
+            response_content = await self._mock_api_call(prompt, max_tokens)
+
+            latency_ms = (time.time() - start_time) * 1000
+            estimated_tokens = len(prompt) // 4 + len(response_content) // 4
+
+            response = ProviderResponse(
+                success=True,
+                content=response_content,
+                tokens_used=estimated_tokens,
+                cost=self.calculate_cost(estimated_tokens),
+                latency_ms=latency_ms,
+                model=self.config.model,
+                provider=self.config.name,
+                timestamp=datetime.now().isoformat(),
+                metadata={
+                    "temperature": temperature,
+                    "max_tokens": max_tokens,
+                    "inference_speed": "ultra_fast"
+                }
+            )
+
+            return response
+
+        except Exception as e:
+            latency_ms = (time.time() - start_time) * 1000
+
+            return ProviderResponse(
+                success=False,
+                content="",
+                tokens_used=0,
+                cost=0.0,
+                latency_ms=latency_ms,
+                model=self.config.model,
+                provider=self.config.name,
+                timestamp=datetime.now().isoformat(),
+                metadata={},
+                error=str(e)
+            )
+
+    async def _mock_api_call(self, prompt: str, max_tokens: int) -> str:
+        """Mock Cerebras API call (ultra fast!)"""
+        await asyncio.sleep(0.1)  # 10x faster than others
+        return f"[Cerebras {self.config.model}] Fast response to: {prompt[:100]}..."
+
+    def calculate_cost(self, tokens: int) -> float:
+        """Calculate cost - VERY LOW compared to others"""
+        pricing = self.PRICING.get(self.config.model, {"input": 0.10, "output": 0.10})
+
+        input_cost = (tokens / 2) * pricing["input"] / 1_000_000
+        output_cost = (tokens / 2) * pricing["output"] / 1_000_000
+
+        return input_cost + output_cost
+
+    async def health_check(self) -> ProviderHealth:
+        """Check Cerebras API health"""
+        try:
+            start = time.time()
+            await asyncio.sleep(0.05)  # Very fast health check
+            latency_ms = (time.time() - start) * 1000
+
+            self.health = ProviderHealth(
+                status="healthy",
+                last_check=datetime.now().isoformat(),
+                consecutive_failures=0,
+                latency_ms=latency_ms,
+                uptime_percentage=100.0
+            )
+
+            return self.health
+
+        except Exception as e:
+            self.health = ProviderHealth(
+                status="unhealthy",
+                last_check=datetime.now().isoformat(),
+                consecutive_failures=self.health.consecutive_failures + 1,
+                last_error=str(e)
+            )
+
+            return self.health
+
+
+class GroqProvider(BaseProvider):
+    """
+    Groq provider - Fast inference on LPU (Language Processing Units).
+
+    Key advantage: Very fast, very low cost, open models.
+    Ideal for: Trivial/simple tasks, prototyping, high-volume.
+    """
+
+    # Pricing (per 1M tokens - extremely low)
+    PRICING = {
+        "llama-3.3-70b-8192": {"input": 0.59, "output": 0.79},
+        "llama-3.1-70b-8192": {"input": 0.59, "output": 0.79},
+        "mixtral-8x7b-32768": {"input": 0.27, "output": 0.27},
+    }
+
+    def __init__(self, config: ProviderConfig):
+        super().__init__(config)
+        self.api_key = os.getenv(config.api_key_env)
+
+        if not self.api_key:
+            raise ValueError(f"API key not found: {config.api_key_env}")
+
+    async def execute(
+        self,
+        prompt: str,
+        max_tokens: int = 2000,
+        temperature: float = 0.7,
+        **kwargs
+    ) -> ProviderResponse:
+        """
+        Execute prompt with Groq.
+
+        Note: Placeholder implementation.
+        In production, use Groq SDK:
+        ```python
+        from groq import Groq
+        client = Groq(api_key=self.api_key)
+        response = client.chat.completions.create(
+            model=self.config.model,
+            messages=[{"role": "user", "content": prompt}],
+            max_tokens=max_tokens,
+            temperature=temperature
+        )
+        ```
+        """
+        start_time = time.time()
+
+        try:
+            # Simulate Groq fast inference
+            response_content = await self._mock_api_call(prompt, max_tokens)
+
+            latency_ms = (time.time() - start_time) * 1000
+            estimated_tokens = len(prompt) // 4 + len(response_content) // 4
+
+            response = ProviderResponse(
+                success=True,
+                content=response_content,
+                tokens_used=estimated_tokens,
+                cost=self.calculate_cost(estimated_tokens),
+                latency_ms=latency_ms,
+                model=self.config.model,
+                provider=self.config.name,
+                timestamp=datetime.now().isoformat(),
+                metadata={
+                    "temperature": temperature,
+                    "max_tokens": max_tokens,
+                    "inference_speed": "very_fast"
+                }
+            )
+
+            return response
+
+        except Exception as e:
+            latency_ms = (time.time() - start_time) * 1000
+
+            return ProviderResponse(
+                success=False,
+                content="",
+                tokens_used=0,
+                cost=0.0,
+                latency_ms=latency_ms,
+                model=self.config.model,
+                provider=self.config.name,
+                timestamp=datetime.now().isoformat(),
+                metadata={},
+                error=str(e)
+            )
+
+    async def _mock_api_call(self, prompt: str, max_tokens: int) -> str:
+        """Mock Groq API call (fast!)"""
+        await asyncio.sleep(0.15)
+        return f"[Groq {self.config.model}] Fast response to: {prompt[:100]}..."
+
+    def calculate_cost(self, tokens: int) -> float:
+        """Calculate cost - LOW compared to premium providers"""
+        pricing = self.PRICING.get(self.config.model, {"input": 0.59, "output": 0.79})
+
+        input_cost = (tokens / 2) * pricing["input"] / 1_000_000
+        output_cost = (tokens / 2) * pricing["output"] / 1_000_000
+
+        return input_cost + output_cost
+
+    async def health_check(self) -> ProviderHealth:
+        """Check Groq API health"""
+        try:
+            start = time.time()
+            await asyncio.sleep(0.1)
+            latency_ms = (time.time() - start) * 1000
+
+            self.health = ProviderHealth(
+                status="healthy",
+                last_check=datetime.now().isoformat(),
+                consecutive_failures=0,
+                latency_ms=latency_ms,
+                uptime_percentage=100.0
+            )
+
+            return self.health
+
+        except Exception as e:
+            self.health = ProviderHealth(
+                status="unhealthy",
+                last_check=datetime.now().isoformat(),
+                consecutive_failures=self.health.consecutive_failures + 1,
+                last_error=str(e)
+            )
+
+            return self.health
+
+
+class TogetherProvider(BaseProvider):
+    """
+    Together AI provider - Open models at competitive pricing.
+
+    Key advantage: Wide variety of open-source models, good price/performance.
+    Ideal for: Specific model requirements, cost optimization.
+    """
+
+    # Pricing (per 1M tokens)
+    PRICING = {
+        "mistralai/Mixtral-8x7B-Instruct-v0.1": {"input": 0.50, "output": 0.50},
+        "meta-llama/Llama-3-70b-chat-hf": {"input": 0.70, "output": 0.70},
+    }
+
+    def __init__(self, config: ProviderConfig):
+        super().__init__(config)
+        self.api_key = os.getenv(config.api_key_env)
+
+        if not self.api_key:
+            raise ValueError(f"API key not found: {config.api_key_env}")
+
+    async def execute(
+        self,
+        prompt: str,
+        max_tokens: int = 2000,
+        temperature: float = 0.7,
+        **kwargs
+    ) -> ProviderResponse:
+        """Execute prompt with Together AI"""
+        start_time = time.time()
+
+        try:
+            response_content = await self._mock_api_call(prompt, max_tokens)
+            latency_ms = (time.time() - start_time) * 1000
+            estimated_tokens = len(prompt) // 4 + len(response_content) // 4
+
+            response = ProviderResponse(
+                success=True,
+                content=response_content,
+                tokens_used=estimated_tokens,
+                cost=self.calculate_cost(estimated_tokens),
+                latency_ms=latency_ms,
+                model=self.config.model,
+                provider=self.config.name,
+                timestamp=datetime.now().isoformat(),
+                metadata={
+                    "temperature": temperature,
+                    "max_tokens": max_tokens
+                }
+            )
+
+            return response
+
+        except Exception as e:
+            latency_ms = (time.time() - start_time) * 1000
+
+            return ProviderResponse(
+                success=False,
+                content="",
+                tokens_used=0,
+                cost=0.0,
+                latency_ms=latency_ms,
+                model=self.config.model,
+                provider=self.config.name,
+                timestamp=datetime.now().isoformat(),
+                metadata={},
+                error=str(e)
+            )
+
+    async def _mock_api_call(self, prompt: str, max_tokens: int) -> str:
+        """Mock Together API call"""
+        await asyncio.sleep(0.3)
+        return f"[Together {self.config.model}] Response to: {prompt[:100]}..."
+
+    def calculate_cost(self, tokens: int) -> float:
+        """Calculate cost"""
+        pricing = self.PRICING.get(self.config.model, {"input": 0.50, "output": 0.50})
+
+        input_cost = (tokens / 2) * pricing["input"] / 1_000_000
+        output_cost = (tokens / 2) * pricing["output"] / 1_000_000
+
+        return input_cost + output_cost
+
+    async def health_check(self) -> ProviderHealth:
+        """Check Together API health"""
+        try:
+            start = time.time()
+            await asyncio.sleep(0.2)
+            latency_ms = (time.time() - start) * 1000
+
+            self.health = ProviderHealth(
+                status="healthy",
+                last_check=datetime.now().isoformat(),
+                consecutive_failures=0,
+                latency_ms=latency_ms,
+                uptime_percentage=100.0
+            )
+
+            return self.health
+
+        except Exception as e:
+            self.health = ProviderHealth(
+                status="unhealthy",
+                last_check=datetime.now().isoformat(),
+                consecutive_failures=self.health.consecutive_failures + 1,
+                last_error=str(e)
+            )
+
+            return self.health