devcouncil 0.3.1 → 0.4.0

package/src/semantic_layer/cache.py

@@ -0,0 +1,290 @@
+"""Semantic cache with FAISS backend, TTL, LRU eviction, and multi-gate validation."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import threading
+import time
+import uuid
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from typing import Any, Protocol
+
+import faiss
+import numpy as np
+from numpy.typing import NDArray
+
+from .config import CacheConfig
+from .embeddings import EmbeddingService, FloatVector
+
+FloatMatrix = NDArray[np.float32]
+
+
+class EmbedderProtocol(Protocol):
+    def embed_one(self, text: str) -> FloatVector: ...
+
+
+@dataclass
+class CacheEntry:
+    entry_id: str
+    query_text: str
+    query_hash: str
+    response_text: str
+    embedding: FloatVector
+    namespace: str
+    intent: str
+    created_at: float
+    last_accessed: float
+    access_count: int = 0
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    def is_expired(self, ttl_seconds: int, now: float | None = None) -> bool:
+        now = now or time.time()
+        return (now - self.created_at) > ttl_seconds
+
+
+@dataclass
+class CacheLookupResult:
+    hit: bool
+    response: str | None = None
+    similarity: float = 0.0
+    entry_id: str | None = None
+    reason: str = "miss"
+
+
+class SemanticCache:
+    """
+    FAISS IndexFlatIP + LRU metadata store.
+    IndexFlatIP with normalized vectors == cosine similarity.
+    """
+
+    def __init__(
+        self,
+        config: CacheConfig | None = None,
+        embedder: EmbedderProtocol | None = None,
+        dimension: int = 384,
+    ) -> None:
+        self.config = config or CacheConfig()
+        self.embedder = embedder or EmbeddingService.get_instance()
+        self.dimension = dimension
+        self._index: faiss.Index = faiss.IndexFlatIP(dimension)
+        self._entries: OrderedDict[str, CacheEntry] = OrderedDict()
+        self._id_to_faiss_row: dict[str, int] = {}
+        self._faiss_row_to_id: dict[int, str] = {}
+        self._lock = threading.RLock()
+        self._stats = {"hits": 0, "misses": 0, "evictions": 0, "false_positive_forced": 0}
+
+    @staticmethod
+    def _normalize_query(text: str) -> str:
+        return " ".join(text.lower().split())
+
+    @staticmethod
+    def _hash_query(text: str) -> str:
+        normalized = SemanticCache._normalize_query(text)
+        return hashlib.sha256(normalized.encode()).hexdigest()
+
+    def _evict_lru(self) -> None:
+        """Remove oldest-accessed entry when at capacity."""
+        if len(self._entries) < self.config.max_entries:
+            return
+        self._entries.popitem(last=False)
+        self._stats["evictions"] += 1
+        # Note: FAISS IndexFlatIP does not support deletion;
+        # stale rows are ignored during search (production: use IndexIDMap2)
+
+    def _search_faiss(
+        self, query_vec: FloatVector, k: int = 5
+    ) -> list[tuple[str, float]]:
+        if self._index.ntotal == 0:
+            return []
+
+        q = query_vec.reshape(1, -1).astype(np.float32)
+        scores, indices = self._index.search(q, min(k, self._index.ntotal))
+        results: list[tuple[str, float]] = []
+        now = time.time()
+
+        for score, idx in zip(scores[0], indices[0]):
+            if idx < 0:
+                continue
+            entry_id = self._faiss_row_to_id.get(int(idx))
+            if entry_id is None:
+                continue
+            entry = self._entries.get(entry_id)
+            if entry is None:
+                continue
+            if entry.is_expired(self.config.ttl_seconds, now):
+                continue
+            if entry.namespace != self.config.namespace:
+                continue
+            results.append((entry_id, float(score)))
+        return results
+
+    def lookup(
+        self,
+        query: str,
+        query_embedding: FloatVector | None = None,
+        intent: str | None = None,
+        force_miss: bool = False,
+    ) -> CacheLookupResult:
+        with self._lock:
+            query_hash = self._hash_query(query)
+
+            # Exact hash short-circuit
+            for entry in self._entries.values():
+                if (
+                    entry.query_hash == query_hash
+                    and entry.namespace == self.config.namespace
+                    and not entry.is_expired(self.config.ttl_seconds)
+                ):
+                    if intent is not None and entry.intent != intent:
+                        self._stats["misses"] += 1
+                        return CacheLookupResult(hit=False, similarity=1.0, reason="intent_mismatch")
+                    entry.last_accessed = time.time()
+                    entry.access_count += 1
+                    self._entries.move_to_end(entry.entry_id)
+                    self._stats["hits"] += 1
+                    return CacheLookupResult(
+                        hit=True,
+                        response=entry.response_text,
+                        similarity=1.0,
+                        entry_id=entry.entry_id,
+                        reason="exact_hash",
+                    )
+
+            if force_miss:
+                self._stats["misses"] += 1
+                return CacheLookupResult(hit=False, reason="forced_exploration")
+
+            vec = query_embedding if query_embedding is not None else self.embedder.embed_one(query)
+            candidates = self._search_faiss(vec, k=5)
+
+            if not candidates:
+                self._stats["misses"] += 1
+                return CacheLookupResult(hit=False, reason="empty_index")
+
+            best_id, best_sim = candidates[0]
+
+            # OOD gate
+            if best_sim < self.config.ood_threshold:
+                self._stats["misses"] += 1
+                return CacheLookupResult(hit=False, similarity=best_sim, reason="ood")
+
+            # Similarity gate
+            if best_sim < self.config.similarity_threshold:
+                self._stats["misses"] += 1
+                return CacheLookupResult(hit=False, similarity=best_sim, reason="below_threshold")
+
+            # Margin gate (ambiguous neighborhood)
+            if len(candidates) >= 2:
+                second_sim = candidates[1][1]
+                if (best_sim - second_sim) < self.config.margin_threshold:
+                    self._stats["misses"] += 1
+                    return CacheLookupResult(
+                        hit=False,
+                        similarity=best_sim,
+                        reason="insufficient_margin",
+                    )
+
+            entry = self._entries[best_id]
+
+            # Intent consistency gate
+            if intent is not None and entry.intent != intent:
+                self._stats["misses"] += 1
+                return CacheLookupResult(hit=False, similarity=best_sim, reason="intent_mismatch")
+
+            entry.last_accessed = time.time()
+            entry.access_count += 1
+            self._entries.move_to_end(entry.entry_id)
+            self._stats["hits"] += 1
+            return CacheLookupResult(
+                hit=True,
+                response=entry.response_text,
+                similarity=best_sim,
+                entry_id=entry.entry_id,
+                reason="semantic_hit",
+            )
+
+    def put(
+        self,
+        query: str,
+        response: str,
+        query_embedding: FloatVector | None = None,
+        intent: str = "general",
+        metadata: dict[str, Any] | None = None,
+    ) -> str:
+        with self._lock:
+            self._evict_lru()
+            vec = query_embedding if query_embedding is not None else self.embedder.embed_one(query)
+            entry_id = str(uuid.uuid4())
+            now = time.time()
+            entry = CacheEntry(
+                entry_id=entry_id,
+                query_text=query,
+                query_hash=self._hash_query(query),
+                response_text=response,
+                embedding=vec,
+                namespace=self.config.namespace,
+                intent=intent,
+                created_at=now,
+                last_accessed=now,
+                metadata=metadata or {},
+            )
+            row = self._index.ntotal
+            self._index.add(vec.reshape(1, -1).astype(np.float32))
+            self._id_to_faiss_row[entry_id] = row
+            self._faiss_row_to_id[row] = entry_id
+            self._entries[entry_id] = entry
+            return entry_id
+
+    @property
+    def hit_rate(self) -> float:
+        total = self._stats["hits"] + self._stats["misses"]
+        return self._stats["hits"] / total if total else 0.0
+
+    def stats(self) -> dict[str, Any]:
+        return {**self._stats, "hit_rate": self.hit_rate, "size": len(self._entries)}
+
+    def persist(self, path: str) -> None:
+        with self._lock:
+            faiss.write_index(self._index, f"{path}.faiss")
+            serializable = {
+                eid: {
+                    "query_text": e.query_text,
+                    "query_hash": e.query_hash,
+                    "response_text": e.response_text,
+                    "embedding": e.embedding.tolist(),
+                    "namespace": e.namespace,
+                    "intent": e.intent,
+                    "created_at": e.created_at,
+                    "last_accessed": e.last_accessed,
+                    "access_count": e.access_count,
+                    "metadata": e.metadata,
+                }
+                for eid, e in self._entries.items()
+            }
+            with open(f"{path}.json", "w", encoding="utf-8") as f:
+                json.dump({"entries": serializable, "row_map": self._faiss_row_to_id}, f)
+
+    def load(self, path: str) -> None:
+        with self._lock:
+            self._index = faiss.read_index(f"{path}.faiss")
+            with open(f"{path}.json", encoding="utf-8") as f:
+                data = json.load(f)
+            self._entries.clear()
+            self._faiss_row_to_id = {int(k): v for k, v in data["row_map"].items()}
+            self._id_to_faiss_row = {v: int(k) for k, v in self._faiss_row_to_id.items()}
+            for eid, raw in data["entries"].items():
+                self._entries[eid] = CacheEntry(
+                    entry_id=eid,
+                    query_text=raw["query_text"],
+                    query_hash=raw["query_hash"],
+                    response_text=raw["response_text"],
+                    embedding=np.array(raw["embedding"], dtype=np.float32),
+                    namespace=raw["namespace"],
+                    intent=raw["intent"],
+                    created_at=raw["created_at"],
+                    last_accessed=raw["last_accessed"],
+                    access_count=raw["access_count"],
+                    metadata=raw.get("metadata", {}),
+                )

package/src/semantic_layer/compressor.py

@@ -0,0 +1,137 @@
+"""Semantic RAG context compression with MMR diversification."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import numpy as np
+from numpy.typing import NDArray
+
+from .config import CompressorConfig
+from .embeddings import EmbeddingService, FloatVector
+
+
+@dataclass
+class CompressedContext:
+    text: str
+    chunks_used: int
+    chunks_total: int
+    estimated_tokens: int
+    scores: list[float]
+
+
+class SemanticCompressor:
+    def __init__(
+        self,
+        config: CompressorConfig | None = None,
+        embedder: EmbeddingService | None = None,
+    ) -> None:
+        self.config = config or CompressorConfig()
+        self.embedder = embedder or EmbeddingService.get_instance()
+
+    @staticmethod
+    def _estimate_tokens(text: str) -> int:
+        # Rough heuristic: ~4 chars per token for English
+        return max(1, len(text) // 4)
+
+    def chunk_document(self, document: str) -> list[str]:
+        """Split document into overlapping word-based chunks."""
+        words = document.split()
+        chunk_words = self.config.chunk_token_size  # treating as word proxy
+        overlap = self.config.chunk_overlap
+        if not words:
+            return []
+
+        chunks: list[str] = []
+        start = 0
+        while start < len(words):
+            end = min(len(words), start + chunk_words)
+            chunks.append(" ".join(words[start:end]))
+            if end >= len(words):
+                break
+            start = max(start + 1, end - overlap)
+        return chunks
+
+    def _mmr_select(
+        self,
+        query_vec: FloatVector,
+        chunk_vecs: NDArray[np.float32],
+        chunk_texts: list[str],
+        token_budget: int,
+    ) -> tuple[list[int], list[float]]:
+        """Maximal Marginal Relevance selection under token budget."""
+        n = len(chunk_texts)
+        if n == 0:
+            return [], []
+
+        relevance = chunk_vecs @ query_vec
+        selected: list[int] = []
+        scores: list[float] = []
+        tokens_used = 0
+        lam = self.config.mmr_lambda
+
+        candidate_mask = relevance >= self.config.min_chunk_score
+        candidates = [i for i in range(n) if candidate_mask[i]]
+
+        while candidates and len(selected) < self.config.top_k:
+            best_idx = -1
+            best_mmr = -float("inf")
+
+            for i in candidates:
+                chunk_tokens = self._estimate_tokens(chunk_texts[i])
+                if tokens_used + chunk_tokens > token_budget and selected:
+                    continue
+
+                rel = float(relevance[i])
+                redundancy = 0.0
+                if selected:
+                    redundancy = max(
+                        float(chunk_vecs[i] @ chunk_vecs[j]) for j in selected
+                    )
+                mmr = lam * rel - (1.0 - lam) * redundancy
+                if mmr > best_mmr:
+                    best_mmr = mmr
+                    best_idx = i
+
+            if best_idx < 0:
+                break
+
+            selected.append(best_idx)
+            scores.append(float(relevance[best_idx]))
+            tokens_used += self._estimate_tokens(chunk_texts[best_idx])
+            candidates.remove(best_idx)
+
+            if tokens_used >= token_budget:
+                break
+
+        return selected, scores
+
+    def compress(
+        self,
+        query: str,
+        documents: list[str],
+        query_embedding: FloatVector | None = None,
+        token_budget: int | None = None,
+    ) -> CompressedContext:
+        budget = token_budget or self.config.token_budget
+        all_chunks: list[str] = []
+        for doc in documents:
+            all_chunks.extend(self.chunk_document(doc))
+
+        if not all_chunks:
+            return CompressedContext(text="", chunks_used=0, chunks_total=0, estimated_tokens=0, scores=[])
+
+        q_vec = query_embedding if query_embedding is not None else self.embedder.embed_one(query)
+        chunk_vecs = self.embedder.embed(all_chunks)
+
+        selected_indices, scores = self._mmr_select(q_vec, chunk_vecs, all_chunks, budget)
+        selected_chunks = [all_chunks[i] for i in selected_indices]
+        combined = "\n\n---\n\n".join(selected_chunks)
+
+        return CompressedContext(
+            text=combined,
+            chunks_used=len(selected_chunks),
+            chunks_total=len(all_chunks),
+            estimated_tokens=self._estimate_tokens(combined),
+            scores=scores,
+        )

package/src/semantic_layer/config.py

@@ -0,0 +1,75 @@
+"""Central configuration for the semantic layer."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Literal
+
+
+class ModelTier(str, Enum):
+    SMALL = "small"  # 1B-3B parameters
+    LARGE = "large"  # 8B-70B parameters
+
+
+@dataclass(frozen=True)
+class EmbeddingConfig:
+    model_name: str = "sentence-transformers/all-MiniLM-L6-v2"
+    dimension: int = 384
+    device: Literal["cpu", "cuda"] = "cpu"  # CPU avoids VRAM contention with LLM
+    batch_size: int = 32
+    normalize: bool = True
+
+
+@dataclass
+class CacheConfig:
+    backend: Literal["faiss", "chroma"] = "faiss"
+    similarity_threshold: float = 0.92
+    ood_threshold: float = 0.75
+    margin_threshold: float = 0.03
+    ttl_seconds: int = 3600
+    max_entries: int = 10_000
+    namespace: str = "default"
+    exploration_rate: float = 0.02  # force miss for FPR calibration
+
+
+@dataclass
+class RouterConfig:
+    complexity_threshold: float = 0.45
+    small_model: str = "qwen2.5:1.5b"
+    large_model: str = "llama3.1:8b"
+    weights: dict[str, float] = field(
+        default_factory=lambda: {
+            "length": 0.25,
+            "structure": 0.30,
+            "embed_disp": 0.25,
+            "domain": 0.20,
+        }
+    )
+
+
+@dataclass
+class CompressorConfig:
+    token_budget: int = 2048
+    top_k: int = 8
+    chunk_token_size: int = 256
+    chunk_overlap: int = 32
+    min_chunk_score: float = 0.35
+    mmr_lambda: float = 0.7
+
+
+@dataclass
+class LLMConfig:
+    backend: Literal["ollama", "llama_cpp", "hf"] = "ollama"
+    base_url: str = "http://localhost:11434"
+    timeout_seconds: float = 120.0
+
+
+@dataclass
+class SemanticLayerConfig:
+    embedding: EmbeddingConfig = field(default_factory=EmbeddingConfig)
+    cache: CacheConfig = field(default_factory=CacheConfig)
+    router: RouterConfig = field(default_factory=RouterConfig)
+    compressor: CompressorConfig = field(default_factory=CompressorConfig)
+    llm: LLMConfig = field(default_factory=LLMConfig)
+    latency_budget_ms: float = 15.0

package/src/semantic_layer/embeddings.py

@@ -0,0 +1,69 @@
+"""Shared embedding service — singleton, low-latency, thread-safe."""
+
+from __future__ import annotations
+
+import threading
+import time
+from typing import Sequence, cast
+
+import numpy as np
+from numpy.typing import NDArray
+
+from .config import EmbeddingConfig
+
+FloatVector = NDArray[np.float32]
+FloatMatrix = NDArray[np.float32]
+
+
+class EmbeddingService:
+    """Lazy-loaded sentence-transformers embedder with L2 normalization."""
+
+    _instance: EmbeddingService | None = None
+    _lock = threading.Lock()
+
+    def __init__(self, config: EmbeddingConfig | None = None) -> None:
+        self.config = config or EmbeddingConfig()
+        self._model = None
+        self._model_lock = threading.Lock()
+
+    @classmethod
+    def get_instance(cls, config: EmbeddingConfig | None = None) -> EmbeddingService:
+        with cls._lock:
+            if cls._instance is None:
+                cls._instance = cls(config)
+            return cls._instance
+
+    def _load_model(self) -> None:
+        if self._model is not None:
+            return
+        with self._model_lock:
+            if self._model is not None:
+                return
+            from sentence_transformers import SentenceTransformer
+
+            self._model = SentenceTransformer(
+                self.config.model_name,
+                device=self.config.device,
+            )
+
+    def embed(self, texts: Sequence[str]) -> FloatMatrix:
+        """Embed a batch of texts. Returns (N, dim) float32 array."""
+        self._load_model()
+        assert self._model is not None
+
+        t0 = time.perf_counter()
+        vectors: FloatMatrix = self._model.encode(
+            list(texts),
+            batch_size=self.config.batch_size,
+            convert_to_numpy=True,
+            normalize_embeddings=self.config.normalize,
+            show_progress_bar=False,
+        ).astype(np.float32)
+        elapsed_ms = (time.perf_counter() - t0) * 1000
+        if elapsed_ms > 10:
+            # Log in production; kept silent here for brevity
+            pass
+        return vectors
+
+    def embed_one(self, text: str) -> FloatVector:
+        return cast(FloatVector, self.embed([text])[0])

package/src/semantic_layer/llm_backends.py

@@ -0,0 +1,99 @@
+"""Unified adapter for Ollama, llama.cpp, and Hugging Face backends."""
+
+from __future__ import annotations
+
+import abc
+from typing import Any, cast
+
+import httpx
+
+from .config import LLMConfig
+
+
+class LLMBackend(abc.ABC):
+    @abc.abstractmethod
+    def generate(self, model: str, prompt: str, system: str | None = None) -> str:
+        ...
+
+    def close(self) -> None:
+        """Release backend resources if any."""
+
+
+class OllamaBackend(LLMBackend):
+    def __init__(self, config: LLMConfig | None = None) -> None:
+        self.config = config or LLMConfig()
+        self._client = httpx.Client(base_url=self.config.base_url, timeout=self.config.timeout_seconds)
+
+    def generate(self, model: str, prompt: str, system: str | None = None) -> str:
+        payload: dict[str, Any] = {
+            "model": model,
+            "prompt": prompt,
+            "stream": False,
+        }
+        if system:
+            payload["system"] = system
+        resp = self._client.post("/api/generate", json=payload)
+        resp.raise_for_status()
+        return cast(str, resp.json()["response"])
+
+    def close(self) -> None:
+        self._client.close()
+
+
+class LlamaCppBackend(LLMBackend):
+    """OpenAI-compatible llama.cpp server (/v1/chat/completions)."""
+
+    def __init__(self, config: LLMConfig | None = None) -> None:
+        self.config = config or LLMConfig(base_url="http://localhost:8080")
+        self._client = httpx.Client(base_url=self.config.base_url, timeout=self.config.timeout_seconds)
+
+    def generate(self, model: str, prompt: str, system: str | None = None) -> str:
+        messages = []
+        if system:
+            messages.append({"role": "system", "content": system})
+        messages.append({"role": "user", "content": prompt})
+        resp = self._client.post(
+            "/v1/chat/completions",
+            json={"model": model, "messages": messages, "stream": False},
+        )
+        resp.raise_for_status()
+        return cast(str, resp.json()["choices"][0]["message"]["content"])
+
+    def close(self) -> None:
+        self._client.close()
+
+
+class HuggingFaceBackend(LLMBackend):
+    """Local HF pipeline — lazy load to avoid cold-start in semantic layer process."""
+
+    def __init__(self, config: LLMConfig | None = None) -> None:
+        self.config = config or LLMConfig()
+        self._pipelines: dict[str, Any] = {}
+
+    def _get_pipeline(self, model: str) -> Any:
+        if model not in self._pipelines:
+            from transformers import pipeline
+
+            self._pipelines[model] = pipeline(
+                "text-generation",
+                model=model,
+                device_map="auto",
+            )
+        return self._pipelines[model]
+
+    def generate(self, model: str, prompt: str, system: str | None = None) -> str:
+        pipe = self._get_pipeline(model)
+        full_prompt = f"{system}\n\n{prompt}" if system else prompt
+        out = pipe(full_prompt, max_new_tokens=512, do_sample=False)
+        return cast(str, out[0]["generated_text"])[len(full_prompt):]
+
+
+def create_backend(config: LLMConfig | None = None) -> LLMBackend:
+    cfg = config or LLMConfig()
+    if cfg.backend == "ollama":
+        return OllamaBackend(cfg)
+    if cfg.backend == "llama_cpp":
+        return LlamaCppBackend(cfg)
+    if cfg.backend == "hf":
+        return HuggingFaceBackend(cfg)
+    raise ValueError(f"Unknown backend: {cfg.backend}")