igel-qe-core 1.0.0

package/knowledge_base/eval/scorers.py

@@ -0,0 +1,494 @@
+"""
+5-Dimension Scoring for Test Case Generation Quality
+====================================================
+Each scorer returns a score [0.0, 1.0] and detailed feedback.
+"""
+from __future__ import annotations
+
+import ast
+import re
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class ScoreDimension:
+    """Single dimension score with feedback."""
+    name: str
+    score: float  # 0.0 to 1.0
+    feedback: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class EvaluationResult:
+    """Complete evaluation result with all 5 dimensions."""
+    query: str
+    retrieval: ScoreDimension
+    domain: ScoreDimension
+    structural: ScoreDimension
+    executability: ScoreDimension
+    publish: ScoreDimension
+    
+    @property
+    def overall_score(self) -> float:
+        """Weighted average: publish is composite of the other 4."""
+        return (
+            self.retrieval.score * 0.25 +
+            self.domain.score * 0.25 +
+            self.structural.score * 0.25 +
+            self.executability.score * 0.25
+        )
+    
+    @property
+    def passed(self) -> bool:
+        """Pass if all dimensions >= 0.6 and overall >= 0.7."""
+        return (
+            self.retrieval.score >= 0.6 and
+            self.domain.score >= 0.6 and
+            self.structural.score >= 0.6 and
+            self.executability.score >= 0.6 and
+            self.overall_score >= 0.7
+        )
+
+
+# ── Retrieval Confidence Scorer ──────────────────────────────────────────────
+
+class RetrievalScorer:
+    """Score retrieval quality: Did we find relevant chunks?"""
+    
+    def score(
+        self,
+        query: str,
+        retrieved_chunks: list,
+        retrieval_metadata: dict,
+    ) -> ScoreDimension:
+        """
+        Scoring criteria:
+        - Number of chunks retrieved (more = better signal, up to 10)
+        - RRF scores (higher = better relevance)
+        - Rerank scores if available
+        - Diversity (chunks from different documents)
+        """
+        feedback = []
+        score = 0.0
+        
+        if not retrieved_chunks:
+            return ScoreDimension(
+                name="retrieval_confidence",
+                score=0.0,
+                feedback=["CRITICAL: No chunks retrieved"],
+            )
+        
+        # Criterion 1: Number of chunks (max score 0.3)
+        chunk_count = len(retrieved_chunks)
+        chunk_score = min(chunk_count / 10.0, 1.0) * 0.3
+        score += chunk_score
+        feedback.append(f"Retrieved {chunk_count} chunks (score: {chunk_score:.2f})")
+        
+        # Criterion 2: RRF scores (max score 0.4)
+        rrf_scores = [
+            getattr(c, 'rrf_score', 0.0) for c in retrieved_chunks
+        ]
+        if rrf_scores:
+            avg_rrf = sum(rrf_scores) / len(rrf_scores)
+            # RRF scores typically range 0-0.1, normalize to 0-1
+            rrf_normalized = min(avg_rrf * 10, 1.0)
+            rrf_component = rrf_normalized * 0.4
+            score += rrf_component
+            feedback.append(
+                f"Avg RRF score: {avg_rrf:.4f} (normalized: {rrf_normalized:.2f}, "
+                f"component: {rrf_component:.2f})"
+            )
+        
+        # Criterion 3: Rerank scores if available (max score 0.3)
+        rerank_scores = [
+            getattr(c, 'rerank_score', None) for c in retrieved_chunks
+        ]
+        rerank_scores = [s for s in rerank_scores if s is not None]
+        if rerank_scores:
+            avg_rerank = sum(rerank_scores) / len(rerank_scores)
+            rerank_component = min(avg_rerank, 1.0) * 0.3
+            score += rerank_component
+            feedback.append(
+                f"Avg rerank score: {avg_rerank:.2f} (component: {rerank_component:.2f})"
+            )
+        else:
+            feedback.append("No reranking applied (missing 0.3 score)")
+            score += 0.0  # penalty for no reranking
+        
+        return ScoreDimension(
+            name="retrieval_confidence",
+            score=min(score, 1.0),
+            feedback=feedback,
+            metadata={
+                "chunk_count": chunk_count,
+                "rrf_scores": rrf_scores,
+                "rerank_scores": rerank_scores,
+            },
+        )
+
+
+# ── Domain Confidence Scorer ──────────────────────────────────────────────────
+
+class DomainScorer:
+    """Score domain validity: Are entities (registry keys, UI paths) validated?"""
+    
+    def __init__(self, kg_client=None):
+        """kg_client: optional KG client to verify entities against graph."""
+        self.kg_client = kg_client
+    
+    def score(
+        self,
+        markdown_output: str,
+        python_output: str | None,
+        retrieved_chunks: list,
+        kg_entities: dict | None = None,
+    ) -> ScoreDimension:
+        """
+        Scoring criteria:
+        - Registry keys present and verified against KG
+        - UI paths present and verified against KG
+        - No hallucinated entities
+        - Product consistency
+        """
+        feedback = []
+        score = 1.0  # start at 1.0, deduct for issues
+        
+        combined = markdown_output + "\n" + (python_output or "")
+        
+        # Extract registry keys
+        registry_keys = re.findall(r'\bapp\.[a-z0-9_]+(?:\.[a-z0-9_]+)*', combined, re.I)
+        registry_keys = list(set(registry_keys))
+        
+        # Extract UI paths (TC Setup > ...)
+        ui_paths = re.findall(
+            r'TC Setup\s*>\s*(?:[^>\n]+\s*>\s*)*[^>\n]+',
+            combined,
+            re.I
+        )
+        ui_paths = list(set(ui_paths))
+        
+        feedback.append(f"Found {len(registry_keys)} registry keys")
+        feedback.append(f"Found {len(ui_paths)} UI paths")
+        
+        # If KG client available, verify entities
+        if self.kg_client and kg_entities:
+            verified_keys = kg_entities.get('registry_keys', [])
+            verified_paths = kg_entities.get('ui_paths', [])
+            
+            # Check registry keys
+            unverified_keys = [
+                k for k in registry_keys
+                if k.lower() not in [v.lower() for v in verified_keys]
+            ]
+            if unverified_keys:
+                penalty = len(unverified_keys) * 0.1
+                score -= penalty
+                feedback.append(
+                    f"WARNING: {len(unverified_keys)} unverified registry keys "
+                    f"(penalty: -{penalty:.2f}): {unverified_keys[:3]}"
+                )
+            
+            # Check UI paths
+            unverified_paths = [
+                p for p in ui_paths
+                if not any(p.lower() in v.lower() or v.lower() in p.lower() for v in verified_paths)
+            ]
+            if unverified_paths:
+                penalty = len(unverified_paths) * 0.1
+                score -= penalty
+                feedback.append(
+                    f"WARNING: {len(unverified_paths)} unverified UI paths "
+                    f"(penalty: -{penalty:.2f})"
+                )
+        else:
+            # No KG verification — partial penalty
+            if registry_keys or ui_paths:
+                score -= 0.2
+                feedback.append(
+                    "WARNING: No KG verification available (penalty: -0.2)"
+                )
+        
+        # Check for common hallucination patterns
+        hallucination_patterns = [
+            r'http://example\.com',
+            r'TODO:',
+            r'PLACEHOLDER',
+            r'\[insert.*?\]',
+        ]
+        for pattern in hallucination_patterns:
+            if re.search(pattern, combined, re.I):
+                score -= 0.15
+                feedback.append(f"HALLUCINATION: Found pattern '{pattern}' (penalty: -0.15)")
+        
+        return ScoreDimension(
+            name="domain_confidence",
+            score=max(score, 0.0),
+            feedback=feedback,
+            metadata={
+                "registry_keys": registry_keys,
+                "ui_paths": ui_paths,
+            },
+        )
+
+
+# ── Structural Completeness Scorer ────────────────────────────────────────────
+
+class StructuralScorer:
+    """Score structural quality: Does TC have all required sections?"""
+    
+    def score(
+        self,
+        markdown_output: str,
+        python_output: str | None,
+    ) -> ScoreDimension:
+        """
+        Scoring criteria:
+        - Preconditions section exists
+        - Steps table exists with >= 3 steps
+        - Each step has test data and expected result
+        - Cleanup/teardown mentioned
+        - Markdown follows template structure
+        """
+        feedback = []
+        score = 0.0
+        
+        # Criterion 1: Preconditions (0.2)
+        if re.search(r'##\s*Preconditions?', markdown_output, re.I):
+            score += 0.2
+            feedback.append("✓ Preconditions section present (+0.2)")
+        else:
+            feedback.append("✗ Missing preconditions section")
+        
+        # Criterion 2: Steps table (0.3)
+        steps_match = re.search(
+            r'\|\s*#\s*\|\s*STEP\s*\|.*?\n\|[-|\s]+\n(.*?)(?=\n##|\Z)',
+            markdown_output,
+            re.DOTALL | re.I
+        )
+        if steps_match:
+            table_body = steps_match.group(1)
+            step_rows = [
+                row.strip() for row in table_body.split('\n')
+                if row.strip() and not row.strip().startswith('|---')
+            ]
+            step_count = len(step_rows)
+            
+            if step_count >= 3:
+                score += 0.3
+                feedback.append(f"✓ Steps table with {step_count} steps (+0.3)")
+            else:
+                partial = (step_count / 3) * 0.3
+                score += partial
+                feedback.append(
+                    f"✗ Only {step_count} steps (expected >= 3, partial: +{partial:.2f})"
+                )
+        else:
+            feedback.append("✗ Missing steps table")
+        
+        # Criterion 3: Expected results (0.2)
+        expected_count = len(re.findall(r'expected result', markdown_output, re.I))
+        if expected_count >= 3:
+            score += 0.2
+            feedback.append(f"✓ Expected results present ({expected_count} mentions, +0.2)")
+        else:
+            partial = (expected_count / 3) * 0.2
+            score += partial
+            feedback.append(
+                f"✗ Few expected results ({expected_count} mentions, +{partial:.2f})"
+            )
+        
+        # Criterion 4: Cleanup/teardown (0.15)
+        if re.search(r'\b(cleanup|teardown|reboot|reset)\b', markdown_output, re.I):
+            score += 0.15
+            feedback.append("✓ Cleanup mentioned (+0.15)")
+        else:
+            feedback.append("✗ No cleanup/teardown mentioned")
+        
+        # Criterion 5: Python structure if available (0.15)
+        if python_output:
+            has_steps = bool(re.findall(r'def\s+\w+_step\d+\s*\(', python_output))
+            has_cleanup = 'def.*cleanup' in python_output.lower()
+            
+            if has_steps:
+                score += 0.1
+                feedback.append("✓ Python has step functions (+0.1)")
+            if has_cleanup:
+                score += 0.05
+                feedback.append("✓ Python has cleanup (+0.05)")
+        
+        return ScoreDimension(
+            name="structural_completeness",
+            score=min(score, 1.0),
+            feedback=feedback,
+        )
+
+
+# ── Executability Scorer ──────────────────────────────────────────────────────
+
+class ExecutabilityScorer:
+    """Score Python script quality: Does it parse, import, pass lint?"""
+    
+    def score(
+        self,
+        python_output: str | None,
+    ) -> ScoreDimension:
+        """
+        Scoring criteria:
+        - Python parses (AST valid)
+        - Required imports present (pytest, allure, etc.)
+        - Fixture usage correct
+        - Registry key operations use proper helpers
+        - No bare asserts (use proper assertions)
+        """
+        feedback = []
+        score = 0.0
+        
+        if not python_output:
+            return ScoreDimension(
+                name="script_executability",
+                score=0.0,
+                feedback=["No Python output to score"],
+            )
+        
+        # Criterion 1: Valid Python syntax (0.3)
+        try:
+            ast.parse(python_output)
+            score += 0.3
+            feedback.append("✓ Valid Python syntax (+0.3)")
+        except SyntaxError as e:
+            feedback.append(f"✗ Syntax error: {e}")
+            return ScoreDimension(
+                name="script_executability",
+                score=0.0,
+                feedback=feedback,
+            )
+        
+        # Criterion 2: Required imports (0.25)
+        required_imports = ['pytest', 'allure']
+        present_imports = []
+        for imp in required_imports:
+            if f'import {imp}' in python_output or f'from {imp}' in python_output:
+                present_imports.append(imp)
+        
+        import_score = (len(present_imports) / len(required_imports)) * 0.25
+        score += import_score
+        if len(present_imports) == len(required_imports):
+            feedback.append(f"✓ All required imports present (+0.25)")
+        else:
+            missing = set(required_imports) - set(present_imports)
+            feedback.append(
+                f"✗ Missing imports: {missing} (+{import_score:.2f})"
+            )
+        
+        # Criterion 3: Fixture usage (0.2)
+        common_fixtures = ['ums_cred', 'device_cred', 'api_config']
+        used_fixtures = [f for f in common_fixtures if f in python_output]
+        if used_fixtures:
+            score += 0.2
+            feedback.append(f"✓ Uses fixtures: {used_fixtures} (+0.2)")
+        else:
+            feedback.append("✗ No standard fixtures used")
+        
+        # Criterion 4: Proper assertions (0.15)
+        bare_asserts = len(re.findall(r'\bassert\s+\w', python_output))
+        proper_asserts = len(re.findall(r'assert.*==|assert.*in|assert.*is', python_output))
+        
+        if proper_asserts >= bare_asserts * 0.8:  # 80% proper assertions
+            score += 0.15
+            feedback.append(
+                f"✓ Proper assertions ({proper_asserts}/{bare_asserts}, +0.15)"
+            )
+        else:
+            partial = (proper_asserts / max(bare_asserts, 1)) * 0.15
+            score += partial
+            feedback.append(
+                f"✗ Some bare asserts ({proper_asserts}/{bare_asserts}, +{partial:.2f})"
+            )
+        
+        # Criterion 5: Registry key helper usage (0.1)
+        if 'app.' in python_output:
+            # Should use get/set helpers, not raw registry operations
+            has_helpers = bool(re.search(r'\b(get|set)_registry\b', python_output))
+            if has_helpers:
+                score += 0.1
+                feedback.append("✓ Uses registry helpers (+0.1)")
+            else:
+                feedback.append("✗ Registry keys without helpers")
+        else:
+            # No registry keys = neutral (no penalty)
+            score += 0.1
+            feedback.append("✓ No registry operations needed (+0.1)")
+        
+        return ScoreDimension(
+            name="script_executability",
+            score=min(score, 1.0),
+            feedback=feedback,
+        )
+
+
+# ── Publish Confidence Scorer ─────────────────────────────────────────────────
+
+class PublishScorer:
+    """Composite score: Ready for human use without editing?"""
+    
+    def score(
+        self,
+        retrieval: ScoreDimension,
+        domain: ScoreDimension,
+        structural: ScoreDimension,
+        executability: ScoreDimension,
+    ) -> ScoreDimension:
+        """
+        Weighted composite:
+        - Retrieval: 25%
+        - Domain: 25%
+        - Structural: 25%
+        - Executability: 25%
+        
+        Thresholds:
+        - >= 0.85: Publish immediately
+        - 0.70-0.84: Publish with low-priority review
+        - 0.60-0.69: Human review required
+        - < 0.60: Block, fix before publish
+        """
+        composite = (
+            retrieval.score * 0.25 +
+            domain.score * 0.25 +
+            structural.score * 0.25 +
+            executability.score * 0.25
+        )
+        
+        feedback = []
+        
+        if composite >= 0.85:
+            feedback.append("PUBLISH: High confidence, ready for immediate use")
+        elif composite >= 0.70:
+            feedback.append("PUBLISH: Good confidence, low-priority review recommended")
+        elif composite >= 0.60:
+            feedback.append("REVIEW REQUIRED: Moderate confidence, human validation needed")
+        else:
+            feedback.append("BLOCK: Low confidence, requires fixes before publishing")
+        
+        # Flag specific issues
+        if retrieval.score < 0.6:
+            feedback.append("ISSUE: Poor retrieval quality")
+        if domain.score < 0.6:
+            feedback.append("ISSUE: Unverified or hallucinated entities")
+        if structural.score < 0.6:
+            feedback.append("ISSUE: Incomplete test case structure")
+        if executability.score < 0.6:
+            feedback.append("ISSUE: Script has errors or missing components")
+        
+        return ScoreDimension(
+            name="publish_confidence",
+            score=composite,
+            feedback=feedback,
+            metadata={
+                "retrieval": retrieval.score,
+                "domain": domain.score,
+                "structural": structural.score,
+                "executability": executability.score,
+            },
+        )

package/knowledge_base/exemplars/README.md

@@ -0,0 +1,7 @@
+# Few-shot exemplars for fidelity-first generation
+
+- **pytest_reference_snippet.py** — Reference pytest shape (`@allure.step`, `TCnnn_stepN`, `api_config`, UMS/WUMS, SSH, OCR).
+- **gold_manual_concise_snippet.md** — Short IGEL manual TC style (few steps, terse expected results).
+- **gold_manual_detailed_snippet.md** — Long TC style (many steps, full `TC Setup` paths).
+
+These are **style anchors**, not KB content. Transform mode still uses the user-pasted TC as primary truth.

package/knowledge_base/exemplars/gold_manual_concise_snippet.md

@@ -0,0 +1,8 @@
+# Example: concise official manual style (UMS device lifecycle)
+
+| Step | Test Data | Expected Result |
+|------|-----------|-----------------|
+| Register endpoint | Register in UMS (Radar). Check WUMS import. | Endpoint registered; license applied if configured. |
+| Check device info | Open device details: system + license. | Information displayed correctly. |
+| Assign profiles | Assign app + standard profiles; save now / on reboot. | Profiles applied per option; apps from portal if applicable. |
+| Reboot | Reboot via UMS. | Endpoint reboots. |

package/knowledge_base/exemplars/gold_manual_detailed_snippet.md

@@ -0,0 +1,7 @@
+# Example: detailed official manual style (browser policies)
+
+Steps preserve exact `TC Setup` navigation and verification URLs from IGEL QA.
+
+1. Open **TC Setup > Chromium Browser > Global Settings > General** — enable *Automatic browser restart on exit*. Expected: setting applied after policy sync.
+2. Open **TC Setup > Chromium Browser > Global Settings > Security & Encryption > JavaScript**. Navigate to `https://browserleaks.com/javascript` in session. Expected: behavior matches policy (enabled/disabled as configured).
+3. Use **chrome://policy** in session to confirm policies applied. Expected: configured keys visible and match TC Setup values.

package/knowledge_base/exemplars/pytest_reference_snippet.py

@@ -0,0 +1,37 @@
+import time, allure
+from core.api.UMS import UMS
+from core.ssh.ssh import SSHClient
+from core.ui.ui_automation_text import OcrUiInteractor
+from config.read_config import ums_cred, device_cred
+from core.ssh.my_logger import logger
+from core.api.ums_wums_api import UMSWUMSApi
+
+click = OcrUiInteractor()
+
+def get_ssh():
+    return SSHClient(host=device_cred["host"], user=device_cred["user"],
+                     pwd=device_cred["pwd"], port=device_cred["port"])
+
+def _assign(ums, wums, device, profile_key, default_name, api_config):
+    profile = ums.get_profile_details(api_config.get(profile_key, default_name))
+    wums.assign_object(device["id"], profile["id"], "profile")
+    return profile
+
+def _detach(ums, wums, device, profile_key, default_name, api_config):
+    profile = ums.get_profile_details(api_config.get(profile_key, default_name))
+    wums.detach_profile(device["id"], profile["id"])
+
+@allure.step("TC001 S1: Assign profile and verify UI")
+def TC001_step1(api_config, browser):
+    try:
+        ums  = UMS(ums_cred["base_url"], ums_cred["username"], ums_cred["password"])
+        wums = UMSWUMSApi(ums_cred["weburl"], ums_cred["username"], ums_cred["password"])
+        dev  = ums.get_vm_details(device_cred["hostname"])
+        _assign(ums, wums, dev, "profile_key", "Default-Profile", api_config)
+        return click.is_text_present_on_screen("ExpectedLabel", refresh_before_check=True)
+    except Exception as e:
+        logger.error(f"TC001_step1: {e}"); return False
+
+@allure.feature("Feature Area")
+@allure.severity(allure.severity_level.CRITICAL)
+def test_TC001_step1(api_config, browser_instance): assert TC001_step1(api_config, browser_instance)

package/knowledge_base/feedback/__init__.py

@@ -0,0 +1,10 @@
+"""Human feedback capture, curation, and application."""
+
+from knowledge_base.feedback.handler import apply_feedback, apply_promotion, capture_feedback, curate_feedback
+
+__all__ = [
+    "apply_feedback",
+    "apply_promotion",
+    "capture_feedback",
+    "curate_feedback",
+]

package/knowledge_base/feedback/analytics.py

@@ -0,0 +1,72 @@
+"""Feedback analytics — Phase 2 weekly failure pattern reports."""
+from __future__ import annotations
+
+from datetime import datetime, timedelta
+
+from knowledge_base.db.client import get_conn
+
+
+def generate_weekly_report(days: int = 7) -> dict:
+    """Aggregate feedback trends for the last N days."""
+    since = datetime.utcnow() - timedelta(days=days)
+
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                SELECT feedback_type, COUNT(*) FROM feedback_events
+                WHERE created_at >= %s
+                GROUP BY feedback_type ORDER BY COUNT(*) DESC
+                """,
+                (since,),
+            )
+            by_type = {row[0]: row[1] for row in cur.fetchall()}
+
+            cur.execute(
+                """
+                SELECT status, COUNT(*) FROM feedback_events
+                WHERE created_at >= %s
+                GROUP BY status ORDER BY COUNT(*) DESC
+                """,
+                (since,),
+            )
+            by_status = {row[0]: row[1] for row in cur.fetchall()}
+
+            cur.execute(
+                """
+                SELECT feedback_type, COUNT(*) FROM feedback_events
+                WHERE created_at >= %s AND status = 'pending'
+                GROUP BY feedback_type ORDER BY COUNT(*) DESC LIMIT 5
+                """,
+                (since,),
+            )
+            top_pending = [{"type": r[0], "count": r[1]} for r in cur.fetchall()]
+
+            cur.execute(
+                """
+                SELECT COUNT(*) FROM feedback_promotions
+                WHERE created_at >= %s AND status = 'applied'
+                """,
+                (since,),
+            )
+            applied_count = cur.fetchone()[0]
+
+    return {
+        "period_days": days,
+        "generated_at": datetime.utcnow().isoformat(),
+        "by_feedback_type": by_type,
+        "by_status": by_status,
+        "top_pending_patterns": top_pending,
+        "promotions_applied": applied_count,
+        "summary": _build_summary(by_type, by_status, top_pending),
+    }
+
+
+def _build_summary(by_type: dict, by_status: dict, top_pending: list) -> str:
+    total = sum(by_type.values())
+    pending = by_status.get("pending", 0)
+    lines = [f"Total feedback events (7d): {total}", f"Pending review: {pending}"]
+    if top_pending:
+        top = top_pending[0]
+        lines.append(f"Top failure pattern: {top['type']} ({top['count']} events)")
+    return " | ".join(lines)