igel-qe-core 1.0.5 → 1.0.7

package/{knowledge_base → src}/core/confidence.py

@@ -4,7 +4,7 @@ Confidence Scoring for Generated Test Cases
 Adds 5-dimensional confidence scores to every test case generation.
 
 Usage:
-    from knowledge_base.core.confidence import score_generation
+    from src.core.confidence import score_generation
     
     confidence = score_generation(
         query=query,
@@ -20,7 +20,7 @@ from __future__ import annotations
 
 from typing import Any
 
-from knowledge_base.eval.scorers import (
+from src.eval.scorers import (
     DomainScorer,
     ExecutabilityScorer,
     PublishScorer,

package/src/core/context_engine.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from typing import List
+
+from src.retrieval.retriever import retrieve, SearchResult
+from src.core.precedence import (
+    RankedChunk,
+    apply_precedence_filter,
+    detect_precedence_conflicts,
+)
+from src.config.policy_config import get_policy
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class SourceReport:
+    """What each source contributed to the context."""
+    jira_found: bool = False
+    jira_chunks: int = 0
+    confluence_found: bool = False
+    confluence_chunks: int = 0
+    postgres_kb_found: bool = False
+    postgres_kb_chunks: int = 0
+    igel_os12_docs_found: bool = False
+    igel_os12_docs_chunks: int = 0
+    community_docs_found: bool = False
+    community_docs_chunks: int = 0
+    conflicts: list[dict] = field(default_factory=list)
+
+
+@dataclass
+class UnifiedContext:
+    jira_key: str
+    ranked_chunks: list[RankedChunk]
+    source_report: SourceReport
+    source_attribution: dict
+    total_estimated_tokens: int
+    retrieval_quality_score: float
+    os_version: str = "OS12"
+
+
+class ContextEngine:
+    def build(self, jira_key: str, query: str) -> UnifiedContext:
+        policy = get_policy()
+        
+        # 1. Retrieve from all sources
+        raw_results = retrieve(query, top_k=policy.top_k_chunks * 2)
+        
+        # 2. Filter for OS12 content only
+        # We check product/metadata or assume it's valid if no 'OS11' explicit flag.
+        # Strict filter: if chunk content or metadata explicitly mentions OS11 and not OS12, drop it.
+        # Here we just blindly filter text if it heavily implies OS11.
+        os12_results = []
+        for r in raw_results:
+            text = (r.child_content + " " + r.parent_content).lower()
+            if "os11" in text and "os12" not in text:
+                continue
+            os12_results.append(r)
+        
+        # 3. Compute retrieval_quality_score
+        quality_score = 0.0
+        if os12_results:
+            quality_score = max(r.rerank_score or r.rrf_score or 0.0 for r in os12_results)
+            
+        # Optional: if quality_score < threshold, we might want to log it for the reviewer
+        if quality_score < 0.1:
+            logger.warning(f"Retrieval quality score {quality_score} is very low for query: {query}")
+        
+        # 4. Rank by precedence
+        # Retriever returned SearchResult objects, we can pass them to precedence as chunks
+        # since apply_precedence_filter expects objects with .metadata and .content
+        scores = {str(r.child_chunk_id): (r.rerank_score or r.rrf_score or 0.0) for r in os12_results}
+        
+        # We need to map SearchResult to a mock chunk object or just ensure it has what precedence needs
+        # SearchResult has .metadata, .parent_content
+        for r in os12_results:
+            if not hasattr(r, 'content'):
+                r.content = r.parent_content
+            if not hasattr(r, 'id'):
+                r.id = r.child_chunk_id
+        
+        filtered_chunks, attribution = apply_precedence_filter(
+            os12_results, 
+            top_k=policy.top_k_chunks, 
+            retrieval_scores=scores
+        )
+        
+        # Also let's re-run rank_chunks_by_precedence to get RankedChunk objects
+        from src.core.precedence import rank_chunks_by_precedence
+        ranked_chunks = rank_chunks_by_precedence(filtered_chunks, scores)
+        
+        # 5. Build SourceReport
+        report = SourceReport()
+        for rc in ranked_chunks:
+            stype = rc.source_type.name
+            if stype == "jira_tc":
+                report.jira_found = True
+                report.jira_chunks += 1
+            elif stype == "confluence":
+                report.confluence_found = True
+                report.confluence_chunks += 1
+            elif stype == "igel_docs" or stype == "kb_web":
+                report.igel_os12_docs_found = True
+                report.igel_os12_docs_chunks += 1
+            elif stype == "community":
+                report.community_docs_found = True
+                report.community_docs_chunks += 1
+            # Postgres KB might be mapped as gold_tc or other custom
+            elif stype == "gold_tc":
+                report.postgres_kb_found = True
+                report.postgres_kb_chunks += 1
+        
+        # 6. Detect conflicts
+        report.conflicts = detect_precedence_conflicts(filtered_chunks)
+        
+        # 7. Enforce max_context_tokens
+        # roughly 1 token = 4 chars
+        total_tokens = 0
+        final_ranked = []
+        for rc in ranked_chunks:
+            est_tokens = len(rc.content) // 4
+            if total_tokens + est_tokens > policy.max_context_tokens:
+                break
+            total_tokens += est_tokens
+            final_ranked.append(rc)
+            
+        return UnifiedContext(
+            jira_key=jira_key,
+            ranked_chunks=final_ranked,
+            source_report=report,
+            source_attribution=attribution,
+            total_estimated_tokens=total_tokens,
+            retrieval_quality_score=quality_score,
+            os_version=policy.igel_os_version
+        )

package/src/core/evidence_resolver.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Optional
+
+from src.core.context_engine import UnifiedContext
+from src.core.precedence import resolve_conflict, RankedChunk
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ResolvedContext:
+    chosen_truth: str
+    source: str
+    conflicting_sources: list[str]
+    resolution_rationale: str
+
+
+class EvidenceResolver:
+    """Resolves conflicting information before feeding context to Generation engines."""
+    
+    def resolve(self, unified_context: UnifiedContext) -> ResolvedContext:
+        """
+        Takes a UnifiedContext (which contains ranked_chunks and conflicts)
+        and outputs a single resolved truth.
+        """
+        if not unified_context.ranked_chunks:
+            return ResolvedContext(
+                chosen_truth="",
+                source="none",
+                conflicting_sources=[],
+                resolution_rationale="No chunks found."
+            )
+            
+        # The primary truth is simply the highest-precedence chunk's content, 
+        # since `ranked_chunks` is already sorted by composite_rank.
+        primary_chunk: RankedChunk = unified_context.ranked_chunks[0]
+        
+        chosen_truth = primary_chunk.content
+        source = primary_chunk.source_type.name
+        
+        conflicting_sources = []
+        
+        # If there are conflicts reported by context engine
+        if unified_context.source_report.conflicts:
+            for conflict in unified_context.source_report.conflicts:
+                s1 = conflict.get("source_1")
+                s2 = conflict.get("source_2")
+                if s1 and s1 not in conflicting_sources:
+                    conflicting_sources.append(s1)
+                if s2 and s2 not in conflicting_sources:
+                    conflicting_sources.append(s2)
+                    
+            # Ensure the primary source itself isn't listed as conflicting against itself
+            if source in conflicting_sources:
+                conflicting_sources.remove(source)
+                
+            rationale = (
+                f"Resolved {len(unified_context.source_report.conflicts)} conflicts. "
+                f"Preferred {source} due to higher precedence over {', '.join(conflicting_sources)}."
+            )
+        else:
+            rationale = f"No precedence conflicts detected. Used {source} as primary truth."
+            
+        return ResolvedContext(
+            chosen_truth=chosen_truth,
+            source=source,
+            conflicting_sources=conflicting_sources,
+            resolution_rationale=rationale
+        )

package/src/core/guardrails.py

@@ -0,0 +1,22 @@
+"""Agent-level guardrails with additive-only restrictions."""
+from __future__ import annotations
+
+from src.core.orchestration_contracts import GuardrailDecision, LLMRequest
+
+
+class AgentGuardrails:
+    def evaluate(self, request: LLMRequest) -> GuardrailDecision:
+        # Agent-specific minimal checks, additive to policy checks.
+        if request.task_type == "script_generation" and "pytest" not in request.user_prompt.lower():
+            return GuardrailDecision(
+                verdict="review",
+                reason="Script generation prompt missing explicit pytest intent",
+            )
+
+        if request.task_type == "requirement_understanding" and not request.context.jira_key:
+            return GuardrailDecision(
+                verdict="review",
+                reason="Missing jira_key in requirement context",
+            )
+
+        return GuardrailDecision(verdict="allow", reason="Guardrail checks passed")

package/src/core/hooks.py

@@ -0,0 +1,57 @@
+"""Pre/post hook interfaces for orchestration chain."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Protocol
+
+from src.core.orchestration_contracts import GenerationContext, HookTrace, LLMRequest
+
+
+class PreHook(Protocol):
+    name: str
+
+    def run(self, request: LLMRequest) -> HookTrace:
+        ...
+
+
+class PostHook(Protocol):
+    name: str
+
+    def run(self, context: GenerationContext, content: str) -> HookTrace:
+        ...
+
+
+@dataclass
+class BasicPreHook:
+    name: str = "basic_pre_hook"
+
+    def run(self, request: LLMRequest) -> HookTrace:
+        if not request.user_prompt.strip():
+            return HookTrace(stage="pre_hook", name=self.name, status="error", message="Empty user prompt")
+        return HookTrace(stage="pre_hook", name=self.name, status="ok")
+
+
+@dataclass
+class BasicPostHook:
+    name: str = "basic_post_hook"
+
+    def run(self, context: GenerationContext, content: str) -> HookTrace:
+        if not content.strip():
+            return HookTrace(stage="post_hook", name=self.name, status="error", message="Empty LLM content")
+        return HookTrace(stage="post_hook", name=self.name, status="ok", details={"chars": len(content)})
+
+
+class PreHookManager:
+    def __init__(self, hooks: list[PreHook] | None = None) -> None:
+        self.hooks = hooks or [BasicPreHook()]
+
+    def run(self, request: LLMRequest) -> list[HookTrace]:
+        return [h.run(request) for h in self.hooks]
+
+
+class PostHookManager:
+    def __init__(self, hooks: list[PostHook] | None = None) -> None:
+        self.hooks = hooks or [BasicPostHook()]
+
+    def run(self, context: GenerationContext, content: str) -> list[HookTrace]:
+        return [h.run(context, content) for h in self.hooks]

package/src/core/intent_service.py

@@ -0,0 +1,74 @@
+"""Unified intent service used by both pipeline and agentic flows."""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from src.generation.input_classifier import classify_input
+
+_FEATURE_MAP = {
+    "entra": "Entra ID SSO",
+    "sso": "Entra ID SSO",
+    "mfa": "Entra ID MFA",
+    "wifi": "Network WiFi 802.1x",
+    "802.1": "Network WiFi 802.1x",
+    "icg": "ICG Certificate Management",
+    "certificate": "ICG Certificate Management",
+    "chromium": "Chromium Browser",
+    "ums": "UMS Administration",
+    "profile": "UMS Profile Assignment",
+    "amd": "AMD Driver Integration",
+    "hub": "Device Onboarding IGEL Hub",
+}
+
+_VERSIONS = ("12.8", "12.9", "12.10")
+
+
+@dataclass
+class IntentResult:
+    intent: str
+    feature_area: str
+    version_scope: str
+    query_categories: list[str] = field(default_factory=list)
+
+
+class IntentService:
+    def classify(self, query: str, source_tc: str | None = None) -> IntentResult:
+        q = (query or "").strip()
+        low = q.lower()
+
+        cls = classify_input(q, source_tc)
+        intent = cls.mode.value
+
+        feature = "Unknown"
+        for kw, name in _FEATURE_MAP.items():
+            if kw in low:
+                feature = name
+                break
+
+        version_scope = "unspecified"
+        categories: list[str] = []
+        for v in _VERSIONS:
+            if v in q or f"os {v}" in low:
+                version_scope = v
+                categories.append("version_specific")
+                break
+        if version_scope == "unspecified":
+            categories.append("version_unspecified")
+
+        if any(w in low for w in ("fail", "error", "negative", "boundary")):
+            categories.append("negative_case")
+        if any(w in low for w in ("regression", "release", "change impact")):
+            categories.append("regression")
+        if any(w in low for w in ("debug", "troubleshoot", "root cause")):
+            categories.append("troubleshooting")
+        if any(w in low for w in ("registry", "profile", "configure", "setting")):
+            categories.append("configuration")
+        if intent == "synthesize" and feature == "Unknown":
+            categories.append("feature_unclear")
+
+        return IntentResult(
+            intent=intent,
+            feature_area=feature,
+            version_scope=version_scope,
+            query_categories=categories,
+        )

package/src/core/llm_executor.py

@@ -0,0 +1,138 @@
+"""Single mandatory LLM execution path.
+
+Execution order:
+  pre-hooks -> policy -> guardrails -> Azure OpenAI call -> post-hooks
+
+Precedence:
+  policy reject always wins.
+  guardrail may add restrictions but cannot override policy reject.
+"""
+from __future__ import annotations
+
+import logging
+import os
+import time
+
+from openai import APIConnectionError, APITimeoutError, AzureOpenAI, RateLimitError
+from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
+
+from src.config import cfg
+from src.core.guardrails import AgentGuardrails
+from src.core.hooks import PostHookManager, PreHookManager
+from src.core.orchestration_contracts import HookTrace, LLMRequest, LLMResult
+from src.core.prompt_policy_engine import PromptPolicyEngine
+from src.core.model_router import ModelRouter
+
+logger = logging.getLogger(__name__)
+
+
+class LLMExecutor:
+    def __init__(self) -> None:
+        self._client: AzureOpenAI | None = None
+        self.pre_hooks = PreHookManager()
+        self.post_hooks = PostHookManager()
+        self.policy = PromptPolicyEngine()
+        self.guardrails = AgentGuardrails()
+
+    def _get_client(self) -> AzureOpenAI:
+        if self._client is None:
+            self._client = AzureOpenAI(
+                api_key=cfg.AZURE_API_KEY,
+                azure_endpoint=cfg.AZURE_ENDPOINT,
+                api_version=cfg.AZURE_API_VERSION,
+            )
+        return self._client
+
+    @retry(
+        retry=retry_if_exception_type((RateLimitError, APIConnectionError, APITimeoutError)),
+        wait=wait_exponential(multiplier=1, min=2, max=30),
+        stop=stop_after_attempt(3),
+        reraise=True,
+    )
+    def _dispatch(self, request: LLMRequest) -> tuple[str, int, int]:
+        client = self._get_client()
+        kwargs = {
+            "model": request.deployment or cfg.AZURE_DEPLOYMENT,
+            "messages": [
+                {"role": "system", "content": request.system_prompt},
+                {"role": "user", "content": request.user_prompt},
+            ],
+            "max_tokens": request.max_tokens,
+            "temperature": request.temperature,
+        }
+        if request.response_format:
+            kwargs["response_format"] = request.response_format
+        response = client.chat.completions.create(**kwargs)
+        content = response.choices[0].message.content or ""
+        tokens_in = response.usage.prompt_tokens if response.usage else 0
+        tokens_out = response.usage.completion_tokens if response.usage else 0
+        return content, tokens_in, tokens_out
+
+    def execute(self, request: LLMRequest) -> LLMResult:
+        traces: list[HookTrace] = []
+
+        pre = self.pre_hooks.run(request)
+        traces.extend(pre)
+        if any(t.status == "error" for t in pre):
+            raise ValueError("Pre-hook validation failed")
+
+        policy = self.policy.evaluate(request)
+        traces.append(HookTrace(stage="policy", name="prompt_policy_engine", status="ok" if policy.verdict != "reject" else "error", message=policy.reason))
+        if policy.verdict == "reject":
+            raise ValueError(f"Policy rejection: {policy.reason}")
+
+        guard = self.guardrails.evaluate(request)
+        traces.append(HookTrace(stage="guardrail", name="agent_guardrails", status="ok" if guard.verdict == "allow" else "warn", message=guard.reason))
+        if guard.verdict == "reject":
+            raise ValueError(f"Guardrail rejection: {guard.reason}")
+
+        start_t = time.time()
+        content, tokens_in, tokens_out = self._dispatch(request)
+        duration_ms = int((time.time() - start_t) * 1000)
+
+        router = ModelRouter()
+        router.record(
+            jira_key=request.context.jira_key or "unknown",
+            action=request.context.intent or "unknown",
+            model_used=request.deployment or cfg.AZURE_DEPLOYMENT,
+            task_type=request.task_type,
+            tokens_in=tokens_in,
+            tokens_out=tokens_out,
+            duration_ms=duration_ms,
+            quality_score=None,
+            routed_by="executor"
+        )
+
+        post = self.post_hooks.run(request.context, content)
+        traces.extend(post)
+        if any(t.status == "error" for t in post):
+            raise ValueError("Post-hook validation failed")
+
+        return LLMResult(
+            content=content,
+            model=request.deployment or cfg.AZURE_DEPLOYMENT,
+            traces=traces,
+            policy=policy,
+            guardrail=guard,
+        )
+
+
+_executor: LLMExecutor | None = None
+
+
+def get_executor() -> LLMExecutor:
+    global _executor
+    if _executor is None:
+        _executor = LLMExecutor()
+    return _executor
+
+
+def executor_enabled_for(agent_name: str) -> bool:
+    """Canary gate for executor rollout.
+
+    Env var ORCH_EXECUTOR_ENABLED_AGENTS contains comma-separated agent names.
+    Defaults to TestCaseAgent + RequirementAgent.
+    """
+    raw = os.getenv("ORCH_EXECUTOR_ENABLED_AGENTS", "TestCaseAgent,RequirementAgent")
+    allowed = {a.strip() for a in raw.split(",") if a.strip()}
+    return agent_name in allowed

package/{knowledge_base → src}/core/logging_setup.py

@@ -7,7 +7,7 @@ import logging
 import sys
 from datetime import datetime, timezone
 
-from knowledge_base.core.request_context import get_request_id
+from src.core.request_context import get_request_id
 
 
 class JsonFormatter(logging.Formatter):