igel-qe-core 1.0.0

package/knowledge_base/eval/__main__.py

@@ -0,0 +1,172 @@
+"""
+CLI entry point for running evaluations.
+
+Usage:
+    python -m knowledge_base.eval --baseline --queries queries_50.json
+    python -m knowledge_base.eval --run my_run --queries queries_50.json
+    python -m knowledge_base.eval --compare baseline my_run
+"""
+import argparse
+import sys
+from pathlib import Path
+
+# Add parent directory to path to allow imports
+sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+
+from knowledge_base.eval.baseline import BaselineManager
+from knowledge_base.eval.harness import EvaluationHarness
+
+
+def main():
+    parser = argparse.ArgumentParser(description="IGEL KB Evaluation Harness")
+    
+    # Mode selection
+    parser.add_argument(
+        "--baseline",
+        action="store_true",
+        help="Run evaluation and capture as baseline",
+    )
+    parser.add_argument(
+        "--run",
+        type=str,
+        metavar="RUN_ID",
+        help="Run evaluation with specified ID",
+    )
+    parser.add_argument(
+        "--compare",
+        nargs=2,
+        metavar=("BASELINE", "CURRENT"),
+        help="Compare two evaluation runs",
+    )
+    parser.add_argument(
+        "--list-baselines",
+        action="store_true",
+        help="List all saved baselines",
+    )
+    
+    # Configuration
+    parser.add_argument(
+        "--queries",
+        type=str,
+        default="knowledge_base/eval/queries_50.json",
+        help="Path to query file (default: queries_50.json)",
+    )
+    parser.add_argument(
+        "--kb-url",
+        type=str,
+        default="http://localhost:8000",
+        help="Knowledge base API URL (default: http://localhost:8000)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="./eval_runs",
+        help="Output directory for results (default: ./eval_runs)",
+    )
+    parser.add_argument(
+        "--baseline-dir",
+        type=str,
+        default="./eval_baselines",
+        help="Directory for baseline storage (default: ./eval_baselines)",
+    )
+    
+    args = parser.parse_args()
+    
+    # Initialize managers
+    baseline_mgr = BaselineManager(baseline_dir=Path(args.baseline_dir))
+    
+    if args.list_baselines:
+        baselines = baseline_mgr.list_baselines()
+        print(f"\nAvailable baselines ({len(baselines)}):")
+        for name in baselines:
+            print(f"  - {name}")
+        print()
+        return
+    
+    if args.compare:
+        baseline_name, current_name = args.compare
+        baseline = baseline_mgr.load_baseline(baseline_name)
+        current = baseline_mgr.load_baseline(current_name)
+        
+        # Create mock harness run for comparison
+        from knowledge_base.eval.harness import HarnessRun
+        current_run = HarnessRun(
+            run_id=current.run_id,
+            timestamp=current.timestamp,
+            query_count=current.query_count,
+            results=[],  # Not needed for comparison
+            avg_scores=current.avg_scores,
+            pass_rate=current.pass_rate,
+            duration_seconds=0,
+            metadata=current.metadata,
+        )
+        
+        comparison = baseline_mgr.compare_to_baseline(current_run, baseline_name)
+        
+        if comparison["overall_regression"]:
+            sys.exit(1)  # Exit with error on regression
+        return
+    
+    # Need KB client for evaluation runs
+    # For now, create a mock client — will be replaced with actual FastAPI client
+    print("WARNING: Using mock KB client. Implement actual API client for real runs.")
+    
+    class MockKBClient:
+        def generate(self, query: str) -> dict:
+            # Mock response
+            return {
+                "markdown": f"# Test Case\n\nGenerated for: {query}\n\n## Preconditions\n...",
+                "python": "import pytest\n\ndef test_example():\n    assert True",
+                "retrieved_chunks": [
+                    {"text": "chunk1", "rrf_score": 0.08},
+                    {"text": "chunk2", "rrf_score": 0.06},
+                ],
+                "retrieval_metadata": {},
+                "kg_entities": {
+                    "registry_keys": ["app.example.key"],
+                    "ui_paths": ["TC Setup > Example"],
+                },
+            }
+    
+    kb_client = MockKBClient()
+    harness = EvaluationHarness(
+        kb_client=kb_client,
+        kg_client=None,
+        output_dir=Path(args.output_dir),
+    )
+    
+    # Run evaluation
+    if args.baseline:
+        run_id = "baseline"
+    elif args.run:
+        run_id = args.run
+    else:
+        run_id = None  # Auto-generate timestamp
+    
+    harness_run = harness.run_evaluation(
+        query_file=args.queries,
+        run_id=run_id,
+        metadata={"mode": "baseline" if args.baseline else "standard"},
+    )
+    
+    # If baseline mode, capture it
+    if args.baseline:
+        baseline_mgr.capture_baseline(
+            harness_run,
+            name="baseline",
+            description="Initial baseline before Phase 0 changes",
+        )
+    
+    # Compare to active baseline if not in baseline mode
+    if not args.baseline:
+        try:
+            comparison = baseline_mgr.compare_to_baseline(harness_run)
+            if comparison["overall_regression"]:
+                print("\n⚠️  Regression detected compared to baseline!")
+                sys.exit(1)
+        except FileNotFoundError:
+            print("\nNo active baseline set. Run with --baseline first.")
+
+
+if __name__ == "__main__":
+    main()

package/knowledge_base/eval/baseline.py

@@ -0,0 +1,213 @@
+"""
+Baseline Management for Evaluation Runs
+========================================
+Capture, store, and compare against baseline evaluation results.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+
+
+@dataclass
+class BaselineSnapshot:
+    """Snapshot of baseline evaluation metrics."""
+    run_id: str
+    timestamp: str
+    avg_scores: dict[str, float]
+    pass_rate: float
+    query_count: int
+    metadata: dict
+
+
+class BaselineManager:
+    """
+    Manages baseline evaluation results:
+    - Capture initial baseline from current system
+    - Store in persistent location
+    - Load and compare against baselines
+    - Detect regressions
+    """
+    
+    def __init__(self, baseline_dir: Path = Path("./eval_baselines")):
+        self.baseline_dir = Path(baseline_dir)
+        self.baseline_dir.mkdir(parents=True, exist_ok=True)
+        self.active_baseline_file = self.baseline_dir / "ACTIVE_BASELINE.json"
+    
+    def capture_baseline(
+        self,
+        harness_run,
+        name: str = "baseline",
+        description: str | None = None,
+    ) -> BaselineSnapshot:
+        """
+        Capture a harness run as baseline.
+        
+        Args:
+            harness_run: HarnessRun object from evaluation
+            name: Baseline identifier (e.g., "phase0", "before-neo4j")
+            description: Human-readable description
+        """
+        snapshot = BaselineSnapshot(
+            run_id=harness_run.run_id,
+            timestamp=harness_run.timestamp,
+            avg_scores=harness_run.avg_scores,
+            pass_rate=harness_run.pass_rate,
+            query_count=harness_run.query_count,
+            metadata={
+                **harness_run.metadata,
+                "baseline_name": name,
+                "description": description,
+            },
+        )
+        
+        # Save to named file
+        baseline_file = self.baseline_dir / f"{name}.json"
+        with open(baseline_file, "w") as f:
+            json.dump(
+                {
+                    "run_id": snapshot.run_id,
+                    "timestamp": snapshot.timestamp,
+                    "avg_scores": snapshot.avg_scores,
+                    "pass_rate": snapshot.pass_rate,
+                    "query_count": snapshot.query_count,
+                    "metadata": snapshot.metadata,
+                },
+                f,
+                indent=2,
+            )
+        
+        print(f"✓ Baseline '{name}' captured: {baseline_file}")
+        
+        # If this is the first baseline, set as active
+        if not self.active_baseline_file.exists():
+            self.set_active_baseline(name)
+        
+        return snapshot
+    
+    def set_active_baseline(self, name: str) -> None:
+        """Set a baseline as the active comparison target."""
+        baseline_file = self.baseline_dir / f"{name}.json"
+        if not baseline_file.exists():
+            raise FileNotFoundError(f"Baseline '{name}' not found")
+        
+        with open(self.active_baseline_file, "w") as f:
+            json.dump({"active_baseline": name}, f, indent=2)
+        
+        print(f"✓ Active baseline set to: {name}")
+    
+    def load_baseline(self, name: str | None = None) -> BaselineSnapshot:
+        """
+        Load a baseline by name. If name is None, load active baseline.
+        """
+        if name is None:
+            # Load active baseline
+            if not self.active_baseline_file.exists():
+                raise FileNotFoundError("No active baseline set")
+            
+            with open(self.active_baseline_file) as f:
+                data = json.load(f)
+                name = data["active_baseline"]
+        
+        baseline_file = self.baseline_dir / f"{name}.json"
+        if not baseline_file.exists():
+            raise FileNotFoundError(f"Baseline '{name}' not found")
+        
+        with open(baseline_file) as f:
+            data = json.load(f)
+        
+        return BaselineSnapshot(**data)
+    
+    def compare_to_baseline(
+        self,
+        harness_run,
+        baseline_name: str | None = None,
+    ) -> dict:
+        """
+        Compare a harness run to baseline.
+        
+        Returns:
+            dict with comparison results and regression detection
+        """
+        baseline = self.load_baseline(baseline_name)
+        
+        comparison = {
+            "baseline_name": baseline.metadata.get("baseline_name", "unknown"),
+            "baseline_run_id": baseline.run_id,
+            "baseline_timestamp": baseline.timestamp,
+            "current_run_id": harness_run.run_id,
+            "current_timestamp": harness_run.timestamp,
+            "pass_rate": {
+                "baseline": baseline.pass_rate,
+                "current": harness_run.pass_rate,
+                "delta": harness_run.pass_rate - baseline.pass_rate,
+                "regression": harness_run.pass_rate < baseline.pass_rate - 0.05,
+            },
+            "scores": {},
+            "overall_regression": False,
+        }
+        
+        # Compare each dimension
+        for dim in baseline.avg_scores:
+            baseline_score = baseline.avg_scores[dim]
+            current_score = harness_run.avg_scores[dim]
+            delta = current_score - baseline_score
+            
+            comparison["scores"][dim] = {
+                "baseline": baseline_score,
+                "current": current_score,
+                "delta": delta,
+                "delta_pct": (delta / baseline_score * 100) if baseline_score > 0 else 0,
+                "regression": delta < -0.05,  # 5% drop = regression
+            }
+            
+            if comparison["scores"][dim]["regression"]:
+                comparison["overall_regression"] = True
+        
+        # Print comparison report
+        self._print_comparison(comparison)
+        
+        return comparison
+    
+    def _print_comparison(self, comparison: dict) -> None:
+        """Pretty-print comparison results."""
+        print(f"\n{'='*70}")
+        print(f"Baseline Comparison")
+        print(f"{'='*70}")
+        print(f"Baseline: {comparison['baseline_name']} ({comparison['baseline_run_id']})")
+        print(f"Current:  {comparison['current_run_id']}")
+        print(f"{'='*70}\n")
+        
+        # Pass rate
+        pr = comparison["pass_rate"]
+        status = "✗ REGRESSION" if pr["regression"] else "✓"
+        print(f"Pass Rate:    {pr['baseline']:.1%} → {pr['current']:.1%} "
+              f"({pr['delta']:+.1%}) {status}\n")
+        
+        # Scores
+        print(f"{'Dimension':<20} {'Baseline':>10} {'Current':>10} {'Delta':>10} {'Status':>10}")
+        print(f"{'-'*70}")
+        
+        for dim, data in comparison["scores"].items():
+            status = "✗ REGRESS" if data["regression"] else "✓"
+            print(f"{dim:<20} {data['baseline']:>10.3f} {data['current']:>10.3f} "
+                  f"{data['delta']:>+10.3f} {status:>10}")
+        
+        print(f"{'-'*70}")
+        
+        if comparison["overall_regression"]:
+            print(f"\n⚠️  OVERALL: REGRESSION DETECTED\n")
+        else:
+            print(f"\n✓ OVERALL: No regressions detected\n")
+        
+        print(f"{'='*70}\n")
+    
+    def list_baselines(self) -> list[str]:
+        """List all available baselines."""
+        baselines = [
+            f.stem for f in self.baseline_dir.glob("*.json")
+            if f.name != "ACTIVE_BASELINE.json"
+        ]
+        return sorted(baselines)