igel-qe-core 1.0.0

package/knowledge_base/core/confidence.py

@@ -0,0 +1,254 @@
+"""
+Confidence Scoring for Generated Test Cases
+============================================
+Adds 5-dimensional confidence scores to every test case generation.
+
+Usage:
+    from knowledge_base.core.confidence import score_generation
+    
+    confidence = score_generation(
+        query=query,
+        markdown_output=markdown,
+        python_output=python,
+        retrieved_chunks=chunks,
+        kg_entities=kg_entities,
+    )
+    
+    # confidence is dict with all 5 dimensions + overall
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from knowledge_base.eval.scorers import (
+    DomainScorer,
+    ExecutabilityScorer,
+    PublishScorer,
+    RetrievalScorer,
+    StructuralScorer,
+)
+
+
+class ConfidenceScorer:
+    """
+    Wrapper around evaluation scorers for runtime confidence scoring.
+    Lightweight initialization — no heavy dependencies.
+    """
+    
+    def __init__(self, kg_client=None):
+        """
+        Args:
+            kg_client: Optional KG client for domain validation
+        """
+        self.retrieval_scorer = RetrievalScorer()
+        self.domain_scorer = DomainScorer(kg_client=kg_client)
+        self.structural_scorer = StructuralScorer()
+        self.executability_scorer = ExecutabilityScorer()
+        self.publish_scorer = PublishScorer()
+    
+    def score(
+        self,
+        query: str,
+        markdown_output: str,
+        python_output: str | None,
+        retrieved_chunks: list,
+        retrieval_metadata: dict | None = None,
+        kg_entities: dict | None = None,
+    ) -> dict[str, Any]:
+        """
+        Score a generated test case across all 5 dimensions.
+        
+        Returns:
+            dict with structure:
+            {
+                "retrieval": {"score": 0.85, "feedback": [...]},
+                "domain": {"score": 0.90, "feedback": [...]},
+                "structural": {"score": 0.75, "feedback": [...]},
+                "executability": {"score": 0.80, "feedback": [...]},
+                "publish": {"score": 0.82, "feedback": [...]},
+                "overall": 0.82,
+                "passed": True,
+                "publish_recommendation": "PUBLISH: High confidence, ready for immediate use"
+            }
+        """
+        # Score each dimension
+        retrieval = self.retrieval_scorer.score(
+            query=query,
+            retrieved_chunks=retrieved_chunks,
+            retrieval_metadata=retrieval_metadata or {},
+        )
+        
+        domain = self.domain_scorer.score(
+            markdown_output=markdown_output,
+            python_output=python_output,
+            retrieved_chunks=retrieved_chunks,
+            kg_entities=kg_entities,
+        )
+        
+        structural = self.structural_scorer.score(
+            markdown_output=markdown_output,
+            python_output=python_output,
+        )
+        
+        executability = self.executability_scorer.score(
+            python_output=python_output,
+        )
+        
+        publish = self.publish_scorer.score(
+            retrieval=retrieval,
+            domain=domain,
+            structural=structural,
+            executability=executability,
+        )
+        
+        # Compute overall (same as publish composite)
+        overall = publish.score
+        
+        # Determine pass/fail
+        passed = (
+            retrieval.score >= 0.6 and
+            domain.score >= 0.6 and
+            structural.score >= 0.6 and
+            executability.score >= 0.6 and
+            overall >= 0.7
+        )
+        
+        # Extract publish recommendation
+        publish_recommendation = publish.feedback[0] if publish.feedback else "UNKNOWN"
+        
+        return {
+            "retrieval": round(retrieval.score, 3),
+            "domain": round(domain.score, 3),
+            "structural": round(structural.score, 3),
+            "executability": round(executability.score, 3),
+            "publish": round(publish.score, 3),
+            "overall": round(overall, 3),
+            "passed": passed,
+            "publish_recommendation": publish_recommendation,
+            "thresholds_met": {
+                "retrieval": retrieval.score >= 0.70,
+                "domain": domain.score >= 0.60,
+                "structural": structural.score >= 0.75,
+                "executability": executability.score >= 0.65,
+                "publish": publish.score >= 0.70,
+            },
+            "route": (
+                "auto_publish" if publish.score >= 0.70
+                else "human_review" if publish.score >= 0.40
+                else "reject"
+            ),
+            "details": {
+                "retrieval": {
+                    "score": retrieval.score,
+                    "feedback": retrieval.feedback,
+                    "metadata": retrieval.metadata,
+                },
+                "domain": {
+                    "score": domain.score,
+                    "feedback": domain.feedback,
+                    "metadata": domain.metadata,
+                },
+                "structural": {
+                    "score": structural.score,
+                    "feedback": structural.feedback,
+                },
+                "executability": {
+                    "score": executability.score,
+                    "feedback": executability.feedback,
+                },
+                "publish": {
+                    "score": publish.score,
+                    "feedback": publish.feedback,
+                    "metadata": publish.metadata,
+                },
+            },
+        }
+
+
+# Global scorer instance (initialized lazily)
+_scorer: ConfidenceScorer | None = None
+
+
+def get_scorer(kg_client=None) -> ConfidenceScorer:
+    """Get or create global scorer instance."""
+    global _scorer
+    if _scorer is None:
+        _scorer = ConfidenceScorer(kg_client=kg_client)
+    return _scorer
+
+
+def score_generation(
+    query: str,
+    markdown_output: str,
+    python_output: str | None,
+    retrieved_chunks: list,
+    retrieval_metadata: dict | None = None,
+    kg_entities: dict | None = None,
+    kg_client=None,
+) -> dict[str, Any]:
+    """
+    Convenience function: score a generation with default scorer.
+    
+    Returns confidence dict with all 5 dimensions.
+    """
+    scorer = get_scorer(kg_client=kg_client)
+    return scorer.score(
+        query=query,
+        markdown_output=markdown_output,
+        python_output=python_output,
+        retrieved_chunks=retrieved_chunks,
+        retrieval_metadata=retrieval_metadata,
+        kg_entities=kg_entities,
+    )
+
+
+def flatten_confidence_for_api(confidence: dict) -> dict:
+    """Return API-friendly confidence object with route and thresholds."""
+    return {
+        "retrieval": confidence.get("retrieval", 0.0),
+        "domain": confidence.get("domain", 0.0),
+        "structural": confidence.get("structural", 0.0),
+        "executability": confidence.get("executability", 0.0),
+        "publish": confidence.get("publish", 0.0),
+        "overall": confidence.get("overall", 0.0),
+        "passed": confidence.get("passed", False),
+        "thresholds_met": confidence.get("thresholds_met", {}),
+        "route": confidence.get("route", "human_review"),
+        "publish_recommendation": confidence.get("publish_recommendation", ""),
+    }
+
+
+def format_confidence_summary(confidence: dict) -> str:
+    """
+    Format confidence scores as human-readable summary.
+    
+    Example output:
+        Overall: 0.82 ✓ PASSED
+        
+        Retrieval:     0.85 ✓
+        Domain:        0.90 ✓
+        Structural:    0.75 ✓
+        Executability: 0.80 ✓
+        Publish:       0.82 ✓
+        
+        Recommendation: PUBLISH: High confidence, ready for immediate use
+    """
+    lines = []
+    
+    # Overall
+    status = "✓ PASSED" if confidence["passed"] else "✗ FAILED"
+    lines.append(f"Overall: {confidence['overall']:.2f} {status}")
+    lines.append("")
+    
+    # Dimensions
+    dimensions = ["retrieval", "domain", "structural", "executability", "publish"]
+    for dim in dimensions:
+        raw = confidence.get(dim, 0.0)
+        score = raw["score"] if isinstance(raw, dict) else float(raw or 0.0)
+        check = "✓" if score >= 0.6 else "✗"
+        lines.append(f"{dim.capitalize():15s}: {score:.2f} {check}")
+    
+    lines.append("")
+    lines.append(f"Recommendation: {confidence['publish_recommendation']}")
+    
+    return "\n".join(lines)

package/knowledge_base/core/logging_setup.py

@@ -0,0 +1,39 @@
+"""Structured logging setup."""
+
+from __future__ import annotations
+
+import json
+import logging
+import sys
+from datetime import datetime, timezone
+
+from knowledge_base.core.request_context import get_request_id
+
+
+class JsonFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        payload = {
+            "ts": datetime.now(timezone.utc).isoformat(),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+            "request_id": get_request_id() or getattr(record, "request_id", ""),
+        }
+        for key in ("endpoint", "duration_ms", "latency_ms", "agent", "status"):
+            if hasattr(record, key):
+                payload[key] = getattr(record, key)
+        if record.exc_info:
+            payload["exc_info"] = self.formatException(record.exc_info)
+        return json.dumps(payload, default=str)
+
+
+def configure_logging(level: str = "INFO", json_logs: bool = True) -> None:
+    root = logging.getLogger()
+    root.handlers.clear()
+    handler = logging.StreamHandler(sys.stdout)
+    if json_logs:
+        handler.setFormatter(JsonFormatter())
+    else:
+        handler.setFormatter(logging.Formatter("%(levelname)s %(name)s %(message)s"))
+    root.addHandler(handler)
+    root.setLevel(getattr(logging, level.upper(), logging.INFO))

package/knowledge_base/core/metrics.py

@@ -0,0 +1,63 @@
+"""Prometheus-style metrics (in-process)."""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections import defaultdict
+
+
+class MetricsRegistry:
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self.counters: dict[str, float] = defaultdict(float)
+        self.histograms: dict[str, list[float]] = defaultdict(list)
+
+    def inc(self, name: str, value: float = 1.0, labels: dict | None = None) -> None:
+        key = self._key(name, labels)
+        with self._lock:
+            self.counters[key] += value
+
+    def observe(self, name: str, value: float, labels: dict | None = None) -> None:
+        key = self._key(name, labels)
+        with self._lock:
+            self.histograms[key].append(value)
+            if len(self.histograms[key]) > 1000:
+                self.histograms[key] = self.histograms[key][-1000:]
+
+    def _key(self, name: str, labels: dict | None) -> str:
+        if not labels:
+            return name
+        parts = ",".join(f"{k}={v}" for k, v in sorted(labels.items()))
+        return f"{name}{{{parts}}}"
+
+    def render_prometheus(self) -> str:
+        lines: list[str] = []
+        with self._lock:
+            for key, val in sorted(self.counters.items()):
+                lines.append(f"# TYPE {key.split('{')[0]} counter")
+                lines.append(f"{key} {val}")
+            for key, vals in sorted(self.histograms.items()):
+                base = key.split("{")[0]
+                lines.append(f"# TYPE {base} summary")
+                if vals:
+                    lines.append(f"{key}_count {len(vals)}")
+                    lines.append(f"{key}_sum {sum(vals)}")
+        return "\n".join(lines) + "\n"
+
+
+metrics = MetricsRegistry()
+
+
+class Timer:
+    def __init__(self, metric: str, labels: dict | None = None) -> None:
+        self.metric = metric
+        self.labels = labels
+        self.start = 0.0
+
+    def __enter__(self):
+        self.start = time.perf_counter()
+        return self
+
+    def __exit__(self, *args):
+        metrics.observe(self.metric, (time.perf_counter() - self.start) * 1000, self.labels)