igel-qe-core 1.0.0

package/knowledge_base/eval/harness.py

@@ -0,0 +1,400 @@
+"""
+Evaluation Harness for IGEL KB System
+======================================
+Runs queries through the system and scores outputs across 5 dimensions.
+"""
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import asdict, dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from knowledge_base.eval.scorers import (
+    DomainScorer,
+    EvaluationResult,
+    ExecutabilityScorer,
+    PublishScorer,
+    RetrievalScorer,
+    StructuralScorer,
+)
+
+
+@dataclass
+class QueryInput:
+    """Single evaluation query."""
+    id: str
+    query: str
+    expected_features: list[str] | None = None
+    notes: str | None = None
+
+
+@dataclass
+class HarnessRun:
+    """Complete evaluation run results."""
+    run_id: str
+    timestamp: str
+    query_count: int
+    results: list[EvaluationResult]
+    avg_scores: dict[str, float]
+    pass_rate: float
+    duration_seconds: float
+    metadata: dict[str, Any]
+
+
+class EvaluationHarness:
+    """
+    Orchestrates evaluation runs:
+    1. Load queries from JSON
+    2. Run each query through system
+    3. Score outputs across 5 dimensions
+    4. Aggregate results
+    5. Save to database + JSON
+    """
+    
+    def __init__(
+        self,
+        kb_client,
+        kg_client=None,
+        output_dir: Path = Path("./eval_runs"),
+    ):
+        """
+        Args:
+            kb_client: Knowledge base API client (must have .generate() method)
+            kg_client: Optional KG client for domain validation
+            output_dir: Where to save evaluation results
+        """
+        self.kb_client = kb_client
+        self.kg_client = kg_client
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+        # Initialize scorers
+        self.retrieval_scorer = RetrievalScorer()
+        self.domain_scorer = DomainScorer(kg_client=kg_client)
+        self.structural_scorer = StructuralScorer()
+        self.executability_scorer = ExecutabilityScorer()
+        self.publish_scorer = PublishScorer()
+    
+    def load_queries(self, query_file: Path | str) -> list[QueryInput]:
+        """Load evaluation queries from JSON file."""
+        with open(query_file) as f:
+            data = json.load(f)
+        
+        return [
+            QueryInput(
+                id=q.get("id", str(i)),
+                query=q["query"],
+                expected_features=q.get("expected_features"),
+                notes=q.get("notes"),
+            )
+            for i, q in enumerate(data, start=1)
+        ]
+    
+    def run_query(self, query_input: QueryInput) -> tuple[dict, float]:
+        """
+        Run a single query through the KB system.
+        Returns (response_data, duration_seconds).
+        """
+        start = time.time()
+        
+        # Call KB API
+        response = self.kb_client.generate(query=query_input.query)
+        
+        duration = time.time() - start
+        
+        return response, duration
+    
+    def score_response(
+        self,
+        query_input: QueryInput,
+        response: dict,
+    ) -> EvaluationResult:
+        """Score a single query response across all 5 dimensions."""
+        
+        # Extract components from response
+        retrieved_chunks = response.get("retrieved_chunks", [])
+        retrieval_metadata = response.get("retrieval_metadata", {})
+        markdown_output = response.get("markdown", "")
+        python_output = response.get("python", None)
+        kg_entities = response.get("kg_entities", None)
+        
+        # Score each dimension
+        retrieval = self.retrieval_scorer.score(
+            query=query_input.query,
+            retrieved_chunks=retrieved_chunks,
+            retrieval_metadata=retrieval_metadata,
+        )
+        
+        domain = self.domain_scorer.score(
+            markdown_output=markdown_output,
+            python_output=python_output,
+            retrieved_chunks=retrieved_chunks,
+            kg_entities=kg_entities,
+        )
+        
+        structural = self.structural_scorer.score(
+            markdown_output=markdown_output,
+            python_output=python_output,
+        )
+        
+        executability = self.executability_scorer.score(
+            python_output=python_output,
+        )
+        
+        publish = self.publish_scorer.score(
+            retrieval=retrieval,
+            domain=domain,
+            structural=structural,
+            executability=executability,
+        )
+        
+        return EvaluationResult(
+            query=query_input.query,
+            retrieval=retrieval,
+            domain=domain,
+            structural=structural,
+            executability=executability,
+            publish=publish,
+        )
+    
+    def run_evaluation(
+        self,
+        query_file: Path | str,
+        run_id: str | None = None,
+        metadata: dict | None = None,
+    ) -> HarnessRun:
+        """
+        Run full evaluation:
+        1. Load queries
+        2. Execute each query
+        3. Score outputs
+        4. Aggregate results
+        5. Save to disk
+        """
+        if run_id is None:
+            run_id = datetime.now().strftime("%Y%m%d_%H%M%S")
+        
+        print(f"\n{'='*60}")
+        print(f"Starting Evaluation Run: {run_id}")
+        print(f"{'='*60}\n")
+        
+        queries = self.load_queries(query_file)
+        print(f"Loaded {len(queries)} queries from {query_file}\n")
+        
+        results = []
+        total_duration = 0.0
+        
+        for i, query_input in enumerate(queries, start=1):
+            print(f"[{i}/{len(queries)}] Processing: {query_input.query[:60]}...")
+            
+            try:
+                # Run query
+                response, duration = self.run_query(query_input)
+                total_duration += duration
+                
+                # Score response
+                eval_result = self.score_response(query_input, response)
+                results.append(eval_result)
+                
+                # Print summary
+                print(f"  ✓ Completed in {duration:.2f}s")
+                print(f"    Overall: {eval_result.overall_score:.2f} | "
+                      f"R:{eval_result.retrieval.score:.2f} "
+                      f"D:{eval_result.domain.score:.2f} "
+                      f"S:{eval_result.structural.score:.2f} "
+                      f"E:{eval_result.executability.score:.2f} "
+                      f"P:{eval_result.publish.score:.2f}")
+                
+                if not eval_result.passed:
+                    print(f"    ⚠ FAILED quality threshold")
+                
+            except Exception as e:
+                print(f"  ✗ Error: {e}")
+                # Continue with other queries
+        
+        # Aggregate results
+        avg_scores = {
+            "retrieval": sum(r.retrieval.score for r in results) / len(results),
+            "domain": sum(r.domain.score for r in results) / len(results),
+            "structural": sum(r.structural.score for r in results) / len(results),
+            "executability": sum(r.executability.score for r in results) / len(results),
+            "publish": sum(r.publish.score for r in results) / len(results),
+            "overall": sum(r.overall_score for r in results) / len(results),
+        }
+        
+        pass_rate = sum(1 for r in results if r.passed) / len(results)
+        
+        harness_run = HarnessRun(
+            run_id=run_id,
+            timestamp=datetime.now().isoformat(),
+            query_count=len(queries),
+            results=results,
+            avg_scores=avg_scores,
+            pass_rate=pass_rate,
+            duration_seconds=total_duration,
+            metadata=metadata or {},
+        )
+        
+        # Save results
+        self._save_run(harness_run)
+        if cfg.EVAL_ENABLED:
+            self._persist_run_to_db(harness_run, query_file)
+        
+        # Print summary
+        print(f"\n{'='*60}")
+        print(f"Evaluation Complete: {run_id}")
+        print(f"{'='*60}")
+        print(f"Queries: {len(queries)}")
+        print(f"Pass Rate: {pass_rate:.1%}")
+        print(f"Average Scores:")
+        for dim, score in avg_scores.items():
+            print(f"  {dim:20s}: {score:.3f}")
+        print(f"Total Duration: {total_duration:.1f}s")
+        print(f"{'='*60}\n")
+        
+        return harness_run
+    
+    def _save_run(self, run: HarnessRun) -> None:
+        """Save evaluation run to JSON file."""
+        output_file = self.output_dir / f"{run.run_id}.json"
+        
+        # Convert to serializable dict
+        run_dict = {
+            "run_id": run.run_id,
+            "timestamp": run.timestamp,
+            "query_count": run.query_count,
+            "avg_scores": run.avg_scores,
+            "pass_rate": run.pass_rate,
+            "duration_seconds": run.duration_seconds,
+            "metadata": run.metadata,
+            "results": [
+                {
+                    "query": r.query,
+                    "passed": r.passed,
+                    "overall_score": r.overall_score,
+                    "retrieval": asdict(r.retrieval),
+                    "domain": asdict(r.domain),
+                    "structural": asdict(r.structural),
+                    "executability": asdict(r.executability),
+                    "publish": asdict(r.publish),
+                }
+                for r in run.results
+            ],
+        }
+        
+        with open(output_file, "w") as f:
+            json.dump(run_dict, f, indent=2)
+        
+        print(f"Saved results to: {output_file}")
+
+    def _persist_run_to_db(self, run: HarnessRun, query_file: Path | str) -> None:
+        """Persist evaluation run to PostgreSQL when EVAL_ENABLED."""
+        try:
+            from knowledge_base.config import cfg
+            from knowledge_base.db.feedback_schema import insert_eval_run, insert_eval_results_batch
+
+            eval_uuid = insert_eval_run(
+                run_id=run.run_id,
+                run_type=run.metadata.get("type", "regression"),
+                avg_scores=run.avg_scores,
+                pass_rate=run.pass_rate,
+                query_count=run.query_count,
+                duration_seconds=run.duration_seconds,
+                system_version=run.metadata.get("phase"),
+                metadata={**run.metadata, "query_file": str(query_file)},
+            )
+
+            batch = []
+            queries = self.load_queries(query_file)
+            query_ids = {q.query: q.id for q in queries}
+
+            for r in run.results:
+                batch.append({
+                    "query_id": query_ids.get(r.query, "unknown"),
+                    "query": r.query,
+                    "generated_output": {},
+                    "retrieval_score": r.retrieval.score,
+                    "retrieval_feedback": r.retrieval.feedback,
+                    "domain_score": r.domain.score,
+                    "domain_feedback": r.domain.feedback,
+                    "structural_score": r.structural.score,
+                    "structural_feedback": r.structural.feedback,
+                    "executability_score": r.executability.score,
+                    "executability_feedback": r.executability.feedback,
+                    "publish_score": r.publish.score,
+                    "publish_feedback": r.publish.feedback,
+                    "overall_score": r.overall_score,
+                    "passed": r.passed,
+                    "duration_seconds": None,
+                    "metadata": {},
+                })
+
+            if batch:
+                insert_eval_results_batch(eval_uuid, batch)
+            print(f"Persisted eval run to database: {eval_uuid}")
+        except Exception as e:
+            print(f"WARNING: Failed to persist eval run to database: {e}")
+    
+    def compare_backends(
+        self,
+        query_file: Path | str,
+        backends: tuple[str, ...] = ("postgres", "neo4j"),
+    ) -> dict:
+        """Compare KG backend quality across queries (Phase 3)."""
+        from knowledge_base.config import cfg
+
+        queries = self.load_queries(query_file)
+        comparison = {"backends": backends, "results": {}}
+
+        original_backend = cfg.KG_BACKEND
+        for backend in backends:
+            import knowledge_base.config as config_module
+            config_module.cfg.KG_BACKEND = backend
+            run_id = f"backend_{backend}_{Path(query_file).stem}"
+            try:
+                run = self.run_evaluation(query_file, run_id=run_id, metadata={"backend": backend})
+                comparison["results"][backend] = {
+                    "pass_rate": run.pass_rate,
+                    "avg_scores": run.avg_scores,
+                }
+            except Exception as e:
+                comparison["results"][backend] = {"error": str(e)}
+
+        import knowledge_base.config as config_module
+        config_module.cfg.KG_BACKEND = original_backend
+        return comparison
+
+    def compare_runs(
+        self,
+        baseline_run_id: str,
+        current_run_id: str,
+    ) -> dict:
+        """Compare two evaluation runs to detect regressions."""
+        baseline_file = self.output_dir / f"{baseline_run_id}.json"
+        current_file = self.output_dir / f"{current_run_id}.json"
+        
+        with open(baseline_file) as f:
+            baseline = json.load(f)
+        with open(current_file) as f:
+            current = json.load(f)
+        
+        comparison = {
+            "baseline_run": baseline_run_id,
+            "current_run": current_run_id,
+            "delta_pass_rate": current["pass_rate"] - baseline["pass_rate"],
+            "delta_scores": {},
+        }
+        
+        for dim in baseline["avg_scores"]:
+            delta = current["avg_scores"][dim] - baseline["avg_scores"][dim]
+            comparison["delta_scores"][dim] = {
+                "baseline": baseline["avg_scores"][dim],
+                "current": current["avg_scores"][dim],
+                "delta": delta,
+                "regression": delta < -0.05,  # 5% drop = regression
+            }
+        
+        return comparison