devcouncil 0.3.1 → 0.4.1

package/src/semantic_layer/pipeline.py

@@ -0,0 +1,111 @@
+"""Orchestrator: semantic cache → router → compressor → LLM."""
+
+from __future__ import annotations
+
+import random
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+from .cache import SemanticCache
+from .compressor import SemanticCompressor
+from .config import SemanticLayerConfig
+from .embeddings import EmbeddingService
+from .llm_backends import LLMBackend, create_backend
+from .router import SemanticRouter
+from .tuner import ThresholdAutoTuner
+
+
+@dataclass
+class PipelineResponse:
+    text: str
+    cache_hit: bool
+    model_used: str | None
+    latency_ms: dict[str, float] = field(default_factory=dict)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class SemanticPipeline:
+    def __init__(
+        self,
+        config: SemanticLayerConfig | None = None,
+        llm: LLMBackend | None = None,
+    ) -> None:
+        self.config = config or SemanticLayerConfig()
+        self.embedder = EmbeddingService.get_instance(self.config.embedding)
+        self.cache = SemanticCache(self.config.cache, self.embedder, self.config.embedding.dimension)
+        self.router = SemanticRouter(self.config.router, self.embedder)
+        self.compressor = SemanticCompressor(self.config.compressor, self.embedder)
+        self.llm: LLMBackend = llm or create_backend(self.config.llm)
+        self.tuner = ThresholdAutoTuner(self.config.cache)
+
+    def run(
+        self,
+        query: str,
+        documents: list[str] | None = None,
+        system_prompt: str | None = None,
+        requires_tools: bool = False,
+    ) -> PipelineResponse:
+        timings: dict[str, float] = {}
+        t0 = time.perf_counter()
+
+        # Single embed pass — shared across all components
+        t_embed = time.perf_counter()
+        query_vec = self.embedder.embed_one(query)
+        timings["embed_ms"] = (time.perf_counter() - t_embed) * 1000
+
+        has_rag = bool(documents)
+        route = self.router.route(query, query_vec, has_rag=has_rag, requires_tools=requires_tools)
+
+        # Exploration: force miss for calibration
+        force_miss = random.random() < self.config.cache.exploration_rate
+
+        t_cache = time.perf_counter()
+        cache_result = self.cache.lookup(query, query_vec, intent=route.intent, force_miss=force_miss)
+        timings["cache_ms"] = (time.perf_counter() - t_cache) * 1000
+
+        if cache_result.hit and cache_result.response is not None:
+            timings["total_ms"] = (time.perf_counter() - t0) * 1000
+            self.tuner.record(cache_result.similarity, was_hit=True)
+            return PipelineResponse(
+                text=cache_result.response,
+                cache_hit=True,
+                model_used=None,
+                latency_ms=timings,
+                metadata={
+                    "similarity": cache_result.similarity,
+                    "reason": cache_result.reason,
+                    "tier": route.tier.value,
+                },
+            )
+
+        # RAG compression
+        ctx = ""
+        if documents:
+            t_comp = time.perf_counter()
+            compressed = self.compressor.compress(query, documents, query_vec)
+            ctx = compressed.text
+            timings["compress_ms"] = (time.perf_counter() - t_comp) * 1000
+            timings["chunks_used"] = compressed.chunks_used
+
+        prompt = query if not ctx else f"Context:\n{ctx}\n\nQuestion: {query}"
+
+        t_llm = time.perf_counter()
+        response = self.llm.generate(route.model_name, prompt, system=system_prompt)
+        timings["llm_ms"] = (time.perf_counter() - t_llm) * 1000
+
+        self.cache.put(query, response, query_vec, intent=route.intent)
+        self.tuner.record(cache_result.similarity, was_hit=False)
+
+        timings["total_ms"] = (time.perf_counter() - t0) * 1000
+        return PipelineResponse(
+            text=response,
+            cache_hit=False,
+            model_used=route.model_name,
+            latency_ms=timings,
+            metadata={
+                "tier": route.tier.value,
+                "complexity": route.complexity_score,
+                "features": route.features,
+            },
+        )

package/src/semantic_layer/router.py

@@ -0,0 +1,128 @@
+"""Semantic router: complexity scoring and model tier selection."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+import numpy as np
+from numpy.typing import NDArray
+
+from .config import ModelTier, RouterConfig
+from .embeddings import EmbeddingService, FloatVector
+
+# Anchor embeddings for "simple" query prototypes (populated at startup)
+SIMPLE_ANCHORS = [
+    "hello",
+    "what time is it",
+    "define photosynthesis",
+    "translate hello to french",
+    "what is 2 plus 2",
+]
+
+COMPLEX_PATTERNS = re.compile(
+    r"(?i)\b(implement|architect|refactor|debug|optimize|prove|derive|"
+    r"multi.?step|recursive|distributed|benchmark|sql|regex|algorithm)\b"
+)
+CODE_PATTERNS = re.compile(r"```|<code>|def |class |function\s*\(|import ")
+
+
+@dataclass
+class RouteDecision:
+    tier: ModelTier
+    model_name: str
+    complexity_score: float
+    features: dict[str, float]
+    intent: str
+
+
+class SemanticRouter:
+    def __init__(
+        self,
+        config: RouterConfig | None = None,
+        embedder: EmbeddingService | None = None,
+    ) -> None:
+        self.config = config or RouterConfig()
+        self.embedder = embedder or EmbeddingService.get_instance()
+        self._anchor_embeddings: NDArray[np.float32] | None = None
+
+    def _ensure_anchors(self) -> NDArray[np.float32]:
+        if self._anchor_embeddings is None:
+            self._anchor_embeddings = self.embedder.embed(SIMPLE_ANCHORS)
+        return self._anchor_embeddings
+
+    def _length_feature(self, query: str) -> float:
+        token_estimate = len(query.split())
+        return min(1.0, token_estimate / 512.0)
+
+    def _structure_feature(self, query: str) -> float:
+        score = 0.0
+        if COMPLEX_PATTERNS.search(query):
+            score += 0.6
+        if CODE_PATTERNS.search(query):
+            score += 0.4
+        if query.count("?") > 1 or "\n" in query:
+            score += 0.2
+        return min(1.0, score)
+
+    def _embed_dispersion_feature(self, embedding: FloatVector) -> float:
+        anchors = self._ensure_anchors()
+        sims = anchors @ embedding  # (N,) cosine sims
+        max_sim = float(np.max(sims))
+        return 1.0 - max_sim  # high dispersion = dissimilar to simple anchors
+
+    @staticmethod
+    def _domain_feature(has_rag: bool, requires_tools: bool) -> float:
+        if requires_tools:
+            return 1.0
+        if has_rag:
+            return 0.6
+        return 0.0
+
+    @staticmethod
+    def _infer_intent(query: str, complexity: float) -> str:
+        if CODE_PATTERNS.search(query):
+            return "code"
+        if complexity >= 0.6:
+            return "reasoning"
+        if "?" in query:
+            return "qa"
+        return "general"
+
+    def route(
+        self,
+        query: str,
+        embedding: FloatVector | None = None,
+        has_rag: bool = False,
+        requires_tools: bool = False,
+    ) -> RouteDecision:
+        vec = embedding if embedding is not None else self.embedder.embed_one(query)
+        w = self.config.weights
+
+        features = {
+            "length": self._length_feature(query),
+            "structure": self._structure_feature(query),
+            "embed_disp": self._embed_dispersion_feature(vec),
+            "domain": self._domain_feature(has_rag, requires_tools),
+        }
+
+        complexity = (
+            w["length"] * features["length"]
+            + w["structure"] * features["structure"]
+            + w["embed_disp"] * features["embed_disp"]
+            + w["domain"] * features["domain"]
+        )
+
+        tier = ModelTier.SMALL if complexity < self.config.complexity_threshold else ModelTier.LARGE
+        model_name = (
+            self.config.small_model if tier == ModelTier.SMALL else self.config.large_model
+        )
+        intent = self._infer_intent(query, complexity)
+
+        return RouteDecision(
+            tier=tier,
+            model_name=model_name,
+            complexity_score=complexity,
+            features=features,
+            intent=intent,
+        )

package/src/semantic_layer/tuner.py

@@ -0,0 +1,72 @@
+"""Online threshold auto-tuner for cache similarity gate."""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections import deque
+from dataclasses import dataclass
+
+import numpy as np
+
+from .config import CacheConfig
+
+
+@dataclass
+class LookupEvent:
+    timestamp: float
+    similarity: float
+    was_hit: bool
+    was_correct: bool | None  # None = unlabeled, True/False from exploration
+
+
+class ThresholdAutoTuner:
+    EPS_FPR = 0.001  # 0.1% false positive rate ceiling
+    TAU_MIN = 0.85
+    TAU_MAX = 0.98
+    TAU_STEP = 0.005
+
+    def __init__(self, cache_config: CacheConfig, window_size: int = 1000) -> None:
+        self.cache_config = cache_config
+        self._events: deque[LookupEvent] = deque(maxlen=window_size)
+        self._lock = threading.Lock()
+
+    def record(
+        self,
+        similarity: float,
+        was_hit: bool,
+        was_correct: bool | None = None,
+    ) -> None:
+        with self._lock:
+            self._events.append(
+                LookupEvent(time.time(), similarity, was_hit, was_correct)
+            )
+
+    def _estimate_fpr(self, tau: float) -> float:
+        labeled = [e for e in self._events if e.was_hit and e.was_correct is not None]
+        if len(labeled) < 20:
+            return 0.0  # insufficient data — assume safe
+        would_hit = [e for e in labeled if e.similarity >= tau]
+        if not would_hit:
+            return 1.0  # tau excludes all labeled hits — treat as unsafe for selection
+        false = sum(1 for e in would_hit if not e.was_correct)
+        return false / len(would_hit)
+
+    def suggest_threshold(self) -> float:
+        with self._lock:
+            labeled = [e for e in self._events if e.was_hit and e.was_correct is not None]
+            if len(labeled) < 20:
+                return self.TAU_MAX
+            for tau in np.arange(self.TAU_MAX, self.TAU_MIN - self.TAU_STEP, -self.TAU_STEP):
+                labeled_at_tau = [e for e in labeled if e.similarity >= float(tau)]
+                if len(labeled_at_tau) < 20:
+                    continue
+                fpr = self._estimate_fpr(float(tau))
+                if fpr <= self.EPS_FPR:
+                    return float(tau)
+            return self.TAU_MIN
+
+    def apply(self) -> float:
+        new_tau = self.suggest_threshold()
+        self.cache_config.similarity_threshold = new_tau
+        return new_tau