igel-qe-core 1.0.3 → 1.0.7

package/src/core/model_router.py

@@ -0,0 +1,246 @@
+"""model_router.py — Task-aware Azure OpenAI model selection.
+
+Design:
+  - Each task type maps to a default ModelConfig defined at module load.
+  - At runtime the router reads igel_generation_history from DB to find the
+    model with the highest average quality_score for that task type.
+  - DB preferences are used as a "learned override" only when >= MIN_SAMPLES
+    observations are present for that task+model combination AND the learned
+    model's avg_quality exceeds the default model's assumed quality floor.
+  - On ANY error (DB unavailable, missing key) the router falls back to
+    cfg.AZURE_DEPLOYMENT silently.
+  - The router intentionally does NOT read .ai/system/model_preferences.json;
+    the DB is always authoritative. model_preferences.json is only a display cache.
+
+Canary deployment:
+  Only TestCaseAgent wires into this router in the first sprint.
+  Other agents keep using cfg.AZURE_DEPLOYMENT directly until quality_score
+  data is confirmed stable.
+"""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import Optional
+
+from src.config import cfg
+
+logger = logging.getLogger(__name__)
+
+# Minimum sample count before a learned preference overrides the default
+MIN_SAMPLES = 10
+# Minimum quality score improvement required to switch from the default model
+MIN_QUALITY_LIFT = 0.05
+# Quality floor assumed for the default model when no history is present
+DEFAULT_QUALITY_FLOOR = 0.70
+
+
+@dataclass
+class ModelConfig:
+    deployment: str
+    api_version: str
+    max_tokens: int
+    temperature: float
+    task_type: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "deployment": self.deployment,
+            "api_version": self.api_version,
+            "max_tokens": self.max_tokens,
+            "temperature": self.temperature,
+            "task_type": self.task_type,
+        }
+
+
+# ── task-type registry ────────────────────────────────────────────────────────
+# Values here are applied when no learned preference exists in the DB.
+# All use cfg.AZURE_DEPLOYMENT so the default tracks the operator's configured model.
+
+def _default_configs() -> dict[str, ModelConfig]:
+    base = cfg.AZURE_DEPLOYMENT
+    ver = getattr(cfg, "AZURE_API_VERSION", "2024-02-01")
+    return {
+        "requirement_understanding": ModelConfig(
+            deployment=base, api_version=ver,
+            max_tokens=2048, temperature=0.2,
+            task_type="requirement_understanding",
+        ),
+        "scenario_generation": ModelConfig(
+            deployment=base, api_version=ver,
+            max_tokens=4096, temperature=0.4,
+            task_type="scenario_generation",
+        ),
+        "testcase_generation": ModelConfig(
+            deployment=base, api_version=ver,
+            max_tokens=4096, temperature=0.3,
+            task_type="testcase_generation",
+        ),
+        "script_generation": ModelConfig(
+            deployment=base, api_version=ver,
+            max_tokens=8192, temperature=0.2,
+            task_type="script_generation",
+        ),
+        "rca": ModelConfig(
+            deployment=base, api_version=ver,
+            max_tokens=2048, temperature=0.1,
+            task_type="rca",
+        ),
+        "healing": ModelConfig(
+            deployment=base, api_version=ver,
+            max_tokens=4096, temperature=0.2,
+            task_type="healing",
+        ),
+    }
+
+
+# ── router ────────────────────────────────────────────────────────────────────
+
+class ModelRouter:
+    """Route a task to the best available Azure OpenAI deployment.
+
+    Usage (TestCaseAgent example):
+        router = ModelRouter()
+        config = router.route("testcase_generation", context_size=3200, token_budget=4096)
+        response = azure_client.complete(deployment=config.deployment, max_tokens=config.max_tokens, ...)
+    """
+
+    def __init__(self) -> None:
+        self._defaults = _default_configs()
+        self._learned: dict[str, dict] = {}  # loaded lazily
+
+    # public API ──────────────────────────────────────────────────────────────
+
+    def route(
+        self,
+        task_type: str,
+        context_size: int = 0,
+        token_budget: Optional[int] = None,
+    ) -> ModelConfig:
+        """Return the best ModelConfig for the given task.
+
+        Falls back to cfg.AZURE_DEPLOYMENT on any error.
+        context_size and token_budget are currently informational (logged for
+        future dynamic selection but do not change the routing decision yet).
+        """
+        try:
+            return self._resolve(task_type, context_size, token_budget)
+        except Exception as e:
+            logger.warning("ModelRouter.route error for %s: %s — using default", task_type, e)
+            return self._fallback(task_type)
+
+    def record(
+        self,
+        jira_key: str,
+        action: str,
+        model_used: str,
+        task_type: str,
+        tokens_in: int,
+        tokens_out: int,
+        duration_ms: int,
+        quality_score: Optional[float],
+        routed_by: str = "router",
+    ) -> None:
+        """Insert a row into igel_generation_history.
+
+        Non-fatal: errors are logged but never raised.
+        """
+        try:
+            from src.db.client import get_conn
+            with get_conn() as conn:
+                with conn.cursor() as cur:
+                    cur.execute(
+                        """
+                        INSERT INTO igel_generation_history
+                          (jira_key, action, model_used, task_type,
+                           tokens_in, tokens_out, duration_ms, quality_score, routed_by)
+                        VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
+                        """,
+                        (jira_key, action, model_used, task_type,
+                         tokens_in, tokens_out, duration_ms, quality_score, routed_by),
+                    )
+                conn.commit()
+        except Exception as e:
+            logger.debug("ModelRouter.record failed (non-fatal): %s", e)
+
+    # internal ────────────────────────────────────────────────────────────────
+
+    def _resolve(self, task_type: str, context_size: int, token_budget: Optional[int]) -> ModelConfig:
+        base = self._defaults.get(task_type) or self._fallback(task_type)
+
+        # Attempt to find a learned improvement from the DB
+        learned = self._get_learned(task_type)
+        if learned:
+            learned_model = learned.get("recommended_model")
+            avg_quality = learned.get("avg_quality") or 0.0
+            call_count = learned.get("call_count") or 0
+
+            if (
+                learned_model
+                and learned_model != base.deployment
+                and call_count >= MIN_SAMPLES
+                and avg_quality >= DEFAULT_QUALITY_FLOOR + MIN_QUALITY_LIFT
+            ):
+                logger.info(
+                    "ModelRouter: task=%s → learned model=%s (avg_quality=%.3f, n=%d)",
+                    task_type, learned_model, avg_quality, call_count,
+                )
+                ver = getattr(cfg, "AZURE_API_VERSION", "2024-02-01")
+                return ModelConfig(
+                    deployment=learned_model,
+                    api_version=ver,
+                    max_tokens=base.max_tokens,
+                    temperature=base.temperature,
+                    task_type=task_type,
+                )
+
+        if context_size:
+            logger.debug("ModelRouter: task=%s context_size=%d → %s", task_type, context_size, base.deployment)
+        return base
+
+    def _get_learned(self, task_type: str) -> Optional[dict]:
+        """Load (and cache) the best model for task_type from DB."""
+        if task_type in self._learned:
+            return self._learned[task_type]
+        try:
+            from src.db.client import get_conn
+            with get_conn() as conn:
+                with conn.cursor() as cur:
+                    cur.execute(
+                        """
+                        SELECT model_used,
+                               AVG(quality_score) AS avg_quality,
+                               COUNT(*) AS call_count
+                        FROM igel_generation_history
+                        WHERE task_type = %s AND quality_score IS NOT NULL
+                        GROUP BY model_used
+                        ORDER BY avg_quality DESC
+                        LIMIT 1
+                        """,
+                        (task_type,),
+                    )
+                    row = cur.fetchone()
+            if row:
+                self._learned[task_type] = {
+                    "recommended_model": row[0],
+                    "avg_quality": float(row[1]) if row[1] is not None else None,
+                    "call_count": row[2],
+                }
+                return self._learned[task_type]
+        except Exception as e:
+            logger.debug("ModelRouter: could not load learned prefs for %s: %s", task_type, e)
+        return None
+
+    def _fallback(self, task_type: str) -> ModelConfig:
+        ver = getattr(cfg, "AZURE_API_VERSION", "2024-02-01")
+        return ModelConfig(
+            deployment=cfg.AZURE_DEPLOYMENT,
+            api_version=ver,
+            max_tokens=4096,
+            temperature=0.3,
+            task_type=task_type,
+        )
+
+    def invalidate_cache(self) -> None:
+        """Force next route() call to re-query DB preferences."""
+        self._learned.clear()

package/src/core/orchestration_contracts.py

@@ -0,0 +1,66 @@
+"""Core contracts for AI orchestration runtime."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+GateVerdict = Literal["allow", "review", "reject"]
+
+
+@dataclass
+class GenerationContext:
+    jira_key: str | None = None
+    test_reqid: str | None = None
+    task_type: str = "general"
+    agent_name: str = "unknown"
+    query: str = ""
+    intent: str = "synthesize"
+    feature_area: str = "Unknown"
+    version_scope: str = "unspecified"
+    query_categories: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class PolicyDecision:
+    verdict: GateVerdict
+    reason: str
+    details: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class GuardrailDecision:
+    verdict: GateVerdict
+    reason: str
+    details: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class HookTrace:
+    stage: str
+    name: str
+    status: Literal["ok", "warn", "error"]
+    message: str = ""
+    details: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class LLMRequest:
+    system_prompt: str
+    user_prompt: str
+    task_type: str
+    agent_name: str
+    temperature: float = 0.2
+    max_tokens: int = 4096
+    deployment: str | None = None
+    response_format: dict[str, Any] | None = None
+    context: GenerationContext = field(default_factory=GenerationContext)
+
+
+@dataclass
+class LLMResult:
+    content: str
+    model: str
+    traces: list[HookTrace] = field(default_factory=list)
+    policy: PolicyDecision | None = None
+    guardrail: GuardrailDecision | None = None

package/src/core/persistence_engine.py

@@ -0,0 +1,159 @@
+from __future__ import annotations
+
+import logging
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from src.models.test_requirement import TestCaseWithConfidence
+from src.core.traceability_engine import TraceabilityRecord
+from src.config.policy_config import PolicyConfig
+from src.db.client import get_conn
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class PersistenceResult:
+    success: bool
+    file_path: str
+    message: str
+
+
+class PersistenceEngine:
+    """Handles saving generated artifacts and tracking Execution Memory in Postgres."""
+    
+    def save_test_case(
+        self, 
+        tc_with_conf: TestCaseWithConfidence, 
+        traceability: TraceabilityRecord, 
+        output_root: Path, 
+        policy: PolicyConfig
+    ) -> PersistenceResult:
+        
+        tc = tc_with_conf.test_case
+        jira_key = traceability.jira_key
+        tc_ref = tc.tc_reference
+        folder = Path(policy.test_case_folder)
+        
+        # 1. Write File
+        try:
+            target_dir = output_root / folder
+            target_dir.mkdir(parents=True, exist_ok=True)
+            
+            # Use safe filename
+            safe_jira = jira_key.replace("-", "_").lower()
+            file_name = f"test_{safe_jira}_{tc_ref}.md"
+            target_path = target_dir / file_name
+            
+            # Format markdown
+            content = self._format_testcase_markdown(tc_with_conf, traceability)
+            with open(target_path, "w", encoding="utf-8") as f:
+                f.write(content)
+        except Exception as e:
+            logger.error(f"Failed to write test case file: {e}")
+            return PersistenceResult(False, "", f"File write error: {e}")
+            
+        # 2. Update Execution Memory (Postgres)
+        try:
+            with get_conn() as conn:
+                with conn.cursor() as cur:
+                    # Upsert generation_record
+                    cur.execute(
+                        """
+                        INSERT INTO igel_generation_records 
+                        (jira_key, tc_reference, generated_at, context_hash, confidence_score, status, artifact_path, metrics)
+                        VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+                        ON CONFLICT (jira_key, tc_reference) 
+                        DO UPDATE SET 
+                            generated_at = EXCLUDED.generated_at,
+                            context_hash = EXCLUDED.context_hash,
+                            confidence_score = EXCLUDED.confidence_score,
+                            status = EXCLUDED.status,
+                            metrics = EXCLUDED.metrics
+                        """,
+                        (
+                            jira_key, 
+                            tc_ref, 
+                            traceability.generated_at, 
+                            traceability.context_hash,
+                            tc.confidence_score,
+                            "pending_review" if not tc_with_conf.is_approved else "auto_approved",
+                            str(target_path),
+                            json.dumps(tc.confidence_metrics)
+                        )
+                    )
+                conn.commit()
+        except Exception as e:
+            logger.warning(f"Failed to update Execution Memory in DB (non-fatal): {e}")
+            
+        return PersistenceResult(True, str(target_path), "Saved successfully")
+
+
+    def save_automation_script(
+        self, 
+        code: str, 
+        target_path: Path, 
+        traceability: TraceabilityRecord, 
+        policy: PolicyConfig
+    ) -> PersistenceResult:
+        
+        try:
+            target_path.parent.mkdir(parents=True, exist_ok=True)
+            with open(target_path, "w", encoding="utf-8") as f:
+                f.write(code)
+        except Exception as e:
+            logger.error(f"Failed to write automation script: {e}")
+            return PersistenceResult(False, "", f"File write error: {e}")
+            
+        try:
+            with get_conn() as conn:
+                with conn.cursor() as cur:
+                    cur.execute(
+                        """
+                        INSERT INTO igel_automation_scripts 
+                        (jira_key, tc_reference, script_path, generated_at)
+                        VALUES (%s, %s, %s, %s)
+                        ON CONFLICT (jira_key, tc_reference) 
+                        DO UPDATE SET 
+                            script_path = EXCLUDED.script_path,
+                            generated_at = EXCLUDED.generated_at
+                        """,
+                        (traceability.jira_key, traceability.tc_reference, str(target_path), traceability.generated_at)
+                    )
+                conn.commit()
+        except Exception as e:
+            logger.warning(f"Failed to update script history in DB (non-fatal): {e}")
+            
+        return PersistenceResult(True, str(target_path), "Script saved successfully")
+
+    def _format_testcase_markdown(self, tc_with_conf: TestCaseWithConfidence, trace: TraceabilityRecord) -> str:
+        tc = tc_with_conf.test_case
+        lines = [
+            f"# Test Case: {tc.title}",
+            f"**Reference:** {tc.tc_reference}  ",
+            f"**Jira Key:** {trace.jira_key}  ",
+            f"**Priority:** {tc.priority.capitalize()}  ",
+            f"**Type:** {tc.test_type.capitalize()}  ",
+            f"**Approval Status:** {'✅ Auto-Approved' if tc_with_conf.is_approved else '⚠️ Pending Review'}  ",
+            "",
+            "## Confidence & Traceability",
+            f"- **Overall Confidence:** {tc.confidence_score * 100:.1f}%",
+            f"- **Context Sources:** {', '.join(trace.source_report.conflicts) if trace.source_report.conflicts else 'Verified'}",
+            "",
+            "## Preconditions"
+        ]
+        
+        for pre in tc.preconditions:
+            lines.append(f"- {pre}")
+            
+        lines.append("\n## Steps")
+        lines.append("| Step | Action | Data | Expected Result |")
+        lines.append("|---|---|---|---|")
+        
+        for step in tc.steps:
+            action = step.action.replace('\n', ' ')
+            data = step.data.replace('\n', ' ') if step.data else "N/A"
+            expected = step.expected_result.replace('\n', ' ')
+            lines.append(f"| {step.step_number} | {action} | {data} | {expected} |")
+            
+        return "\n".join(lines)

package/src/core/prompt_builder.py

@@ -0,0 +1,49 @@
+import json
+from typing import Dict, Any, Optional
+
+class PromptBuilder:
+    """
+    Constructs the final prompt by assembling minimal behavioral instructions,
+    templates, and the unified context package.
+    
+    This replaces the V1 anti-pattern of passing 50k tokens of static 
+    procedural rules to the LLM. Procedural rules belong in the Python runtime.
+    """
+    
+    @staticmethod
+    def build(
+        role: str,
+        behavioral_instructions: str,
+        template: str,
+        context_package: Dict[str, Any]
+    ) -> str:
+        """
+        Builds the final system prompt.
+        
+        Args:
+            role: The Persona (e.g., "You are a Senior IGEL QA Engineer.")
+            behavioral_instructions: Minimal rules (e.g., "Do not invent APIs", "Assume no IGEL knowledge")
+            template: The required output schema or structure.
+            context_package: The resolved, unified context to be used.
+            
+        Returns:
+            The assembled prompt string.
+        """
+        
+        prompt_parts = []
+        
+        # 1. Role
+        prompt_parts.append(f"# Role\n{role}")
+        
+        # 2. Behavioral Instructions
+        prompt_parts.append(f"# Instructions\n{behavioral_instructions}")
+        
+        # 3. Output Template
+        prompt_parts.append(f"# Required Format\n{template}")
+        
+        # 4. Context Package
+        prompt_parts.append("# Context Package")
+        prompt_parts.append("Use the following context strictly. Do not hallucinate outside of this context:")
+        prompt_parts.append(json.dumps(context_package, indent=2))
+        
+        return "\n\n".join(prompt_parts)

package/src/core/prompt_policy_engine.py

@@ -0,0 +1,35 @@
+"""Prompt policy engine applied before LLM dispatch.
+
+Phase bootstrap behavior:
+- If no project policy profile exists, safe built-in defaults are enforced.
+- Project profile support can be layered without disabling defaults.
+"""
+from __future__ import annotations
+
+import re
+
+from src.core.orchestration_contracts import LLMRequest, PolicyDecision
+from src.config.policy_config import get_policy
+
+class PromptPolicyEngine:
+    def evaluate(self, request: LLMRequest) -> PolicyDecision:
+        policy = get_policy()
+        
+        if len(request.system_prompt) + len(request.user_prompt) > policy.max_prompt_chars:
+            return PolicyDecision(
+                verdict="reject",
+                reason="Prompt length exceeds policy limit",
+                details={"limit": policy.max_prompt_chars},
+            )
+
+        blob = f"{request.system_prompt}\n{request.user_prompt}"
+        for pattern_str in policy.blocked_patterns:
+            p = re.compile(pattern_str, re.IGNORECASE)
+            if p.search(blob):
+                return PolicyDecision(
+                    verdict="reject",
+                    reason="Prompt violates blocked-pattern policy",
+                    details={"pattern": p.pattern},
+                )
+
+        return PolicyDecision(verdict="allow", reason="Policy checks passed")

package/src/core/traceability_engine.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import logging
+import hashlib
+from datetime import datetime, timezone
+from dataclasses import dataclass
+
+from src.core.context_engine import UnifiedContext, SourceReport
+
+logger = logging.getLogger(__name__)
+
+@dataclass
+class TraceabilityRecord:
+    tc_reference: str
+    jira_key: str
+    source_chunk_ids: list[str]       
+    business_rules_used: list[str]
+    source_attribution: dict
+    source_report: SourceReport       
+    generated_at: datetime
+    context_hash: str                  
+    os_version: str = "OS12"
+
+class TraceabilityEngine:
+    def build_matrix(
+        self, 
+        tc_reference: str, 
+        unified_context: UnifiedContext, 
+        business_rules: list[str]
+    ) -> TraceabilityRecord:
+        """
+        Pure Python, no LLM. Maps steps to chunks that were actually in context.
+        """
+        # Collect chunk IDs
+        source_chunk_ids = [str(getattr(rc.chunk, 'id', '')) for rc in unified_context.ranked_chunks if hasattr(rc.chunk, 'id')]
+        
+        # Build reproducible hash of context
+        hasher = hashlib.sha256()
+        for rc in unified_context.ranked_chunks:
+            hasher.update(rc.content.encode('utf-8'))
+        context_hash = hasher.hexdigest()
+        
+        record = TraceabilityRecord(
+            tc_reference=tc_reference,
+            jira_key=unified_context.jira_key,
+            source_chunk_ids=source_chunk_ids,
+            business_rules_used=business_rules,
+            source_attribution=unified_context.source_attribution,
+            source_report=unified_context.source_report,
+            generated_at=datetime.now(timezone.utc),
+            context_hash=context_hash,
+            os_version=unified_context.os_version
+        )
+        
+        logger.info(f"Generated TraceabilityRecord for {tc_reference}")
+        return record

package/{knowledge_base → src}/db/client.py

@@ -7,7 +7,7 @@ import psycopg2
 import psycopg2.pool
 from contextlib import contextmanager
 
-from knowledge_base.config import cfg
+from src.config import cfg
 
 logger = logging.getLogger(__name__)
 

package/{knowledge_base → src}/db/connector_sync.py

@@ -4,7 +4,7 @@ from __future__ import annotations
 
 from datetime import datetime, timezone
 
-from knowledge_base.db.client import get_conn
+from src.db.client import get_conn
 
 
 def get_last_sync(connector_name: str) -> datetime | None:

package/{knowledge_base → src}/db/feedback_schema.py

@@ -12,8 +12,8 @@ Never auto-edit based on raw feedback — human curation is mandatory gate.
 """
 from __future__ import annotations
 
-from knowledge_base.config import cfg
-from knowledge_base.db.client import get_conn
+from src.config import cfg
+from src.db.client import get_conn
 
 # ─────────────────────────────────────────────────────────────────────────────
 # Feedback Events Table