igel-qe-core 1.0.0

package/knowledge_base/retrieval/unified_query.py

@@ -0,0 +1,189 @@
+"""
+Unified query orchestrator — RAG + EKG + Web + MCP external sources.
+"""
+
+from __future__ import annotations
+
+import logging
+import random
+from dataclasses import dataclass, field
+from time import perf_counter
+
+from knowledge_base.config import cfg
+from knowledge_base.connectors.query_bridge import ExternalContext, query_mcp_sources
+from knowledge_base.retrieval.retriever import RetrievalBundle, retrieve_v3
+
+logger = logging.getLogger(__name__)
+
+
+def _empty_bundle(query: str, *, reason: str) -> RetrievalBundle:
+    """Build a safe empty fallback bundle for timeout/error paths."""
+    from knowledge_base.retrieval.knowledge_graph import KGContext
+    from knowledge_base.retrieval.retrieval_debug import RetrievalDebugReport
+
+    return RetrievalBundle(
+        results=[],
+        kg_context=KGContext("", [], 0, 0),
+        image_block="",
+        image_hits=[],
+        debug=RetrievalDebugReport(
+            query=query,
+            query_classification=reason,
+            total_vector_candidates=0,
+            total_fts_candidates=0,
+            rrf_merged_count=0,
+            reranker_applied=False,
+            post_rerank_count=0,
+            kg_nodes_found=0,
+            kg_edges_traversed=0,
+            final_chunk_count=0,
+            total_tokens_used=0,
+            token_budget=int(getattr(cfg, "CONTEXT_TOKEN_BUDGET", 8000)),
+            traces=[],
+            timing_ms={},
+        ),
+        web_hits=[],
+        web_block="",
+    )
+
+
+@dataclass
+class UnifiedQueryResult:
+    rag: RetrievalBundle
+    web_block: str = ""
+    web_hits: list = field(default_factory=list)
+    mcp: ExternalContext = field(default_factory=ExternalContext)
+    sources_summary: dict = field(default_factory=dict)
+    timing_ms: dict = field(default_factory=dict)
+
+    @property
+    def kg_context(self):
+        return self.rag.kg_context
+
+    @property
+    def results(self):
+        return self.rag.results
+
+    @property
+    def debug(self):
+        return self.rag.debug
+
+    @property
+    def precedence_conflicts(self):
+        return self.rag.precedence_conflicts
+
+    @property
+    def precedence_attribution(self):
+        return self.rag.precedence_attribution
+
+    @property
+    def combined_context_block(self) -> str:
+        parts = []
+        if self.rag.kg_context.text_block:
+            parts.append(self.rag.kg_context.text_block.strip())
+        if self.rag.image_block:
+            parts.append(self.rag.image_block.strip())
+        if self.web_block:
+            parts.append(self.web_block.strip())
+        if self.mcp.block:
+            parts.append(self.mcp.block.strip())
+        return "\n\n".join(p for p in parts if p)
+
+
+def _maybe_persist_trace(query: str, result: "UnifiedQueryResult") -> None:
+    rate = getattr(cfg, "RETRIEVAL_TRACE_SAMPLE_RATE", 0.0)
+    if rate <= 0 or random.random() > rate:
+        return
+    try:
+        from knowledge_base.core.request_context import get_request_id
+        from knowledge_base.db.client import get_conn
+        import json
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute(
+                    """
+                    INSERT INTO retrieval_traces (request_id, query, sources_summary, debug_json, timing_ms)
+                    VALUES (%s, %s, %s::jsonb, %s::jsonb, %s::jsonb)
+                    """,
+                    (
+                        get_request_id(),
+                        query[:2000],
+                        json.dumps(result.sources_summary),
+                        result.debug.to_json() if result.debug else "{}",
+                        json.dumps(result.timing_ms),
+                    ),
+                )
+    except Exception as exc:
+        logger.debug("Trace persistence skipped: %s", exc)
+
+
+def unified_query(
+    query: str,
+    *,
+    top_k: int = 8,
+    product: str | None = None,
+    chunk_type: str | None = None,
+) -> UnifiedQueryResult:
+    """Orchestrate RAG + EKG + Web + MCP for every user query."""
+    timings: dict[str, float] = {}
+    timeout = max(1.0, float(getattr(cfg, "QUERY_TIMEOUT_SECONDS", 60)))
+    deadline = perf_counter() + timeout
+
+    def _rag():
+        t0 = perf_counter()
+        bundle = retrieve_v3(query, top_k=top_k, product=product, chunk_type=chunk_type)
+        timings["rag_kg_web_ms"] = (perf_counter() - t0) * 1000
+        return bundle
+
+    def _mcp(*, timeout_seconds: float | None = None):
+        t0 = perf_counter()
+        ctx = query_mcp_sources(
+            query,
+            max_hits=getattr(cfg, "WEB_MAX_PAGES", 3),
+            timeout_seconds=timeout_seconds,
+        )
+        timings["mcp_ms"] = (perf_counter() - t0) * 1000
+        return ctx
+
+    bundle = _empty_bundle(query, reason="init")
+    mcp = ExternalContext()
+    try:
+        bundle = _rag()
+    except Exception as exc:
+        logger.exception("RAG retrieval failed: %s", exc)
+        bundle = _empty_bundle(query, reason="error")
+
+    if getattr(cfg, "QUERY_EXTERNAL_ENABLED", True):
+        remaining = deadline - perf_counter()
+        if remaining <= 0:
+            logger.warning("MCP query skipped due exhausted unified timeout budget")
+        else:
+            try:
+                mcp = _mcp(timeout_seconds=max(0.05, remaining))
+            except Exception as exc:
+                logger.warning("MCP query failed: %s", exc)
+
+    web_block = getattr(bundle, "web_block", "") or ""
+    web_hits = list(getattr(bundle, "web_hits", []) or [])
+
+    sources_summary = {
+        "rag_chunks": len(bundle.results),
+        "kg_nodes": bundle.kg_context.node_count,
+        "kg_edges": bundle.kg_context.edge_count,
+        "web_hits": len(web_hits),
+        "mcp_hits": len(mcp.hits),
+        "mcp_sources": mcp.sources_queried,
+        "reranker": bundle.debug.reranker_applied,
+        "kg_backend": getattr(cfg, "KG_BACKEND", "postgres"),
+    }
+
+    result = UnifiedQueryResult(
+        rag=bundle,
+        web_block=web_block,
+        web_hits=web_hits,
+        mcp=mcp,
+        sources_summary=sources_summary,
+        timing_ms=timings,
+    )
+    _maybe_persist_trace(query, result)
+    return result

package/knowledge_base/retrieval/web_retriever.py

@@ -0,0 +1,337 @@
+"""Live web retrieval — confidence-gated Stage 2 search (Firecrawl / DuckDuckGo / SearXNG)."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import math
+from dataclasses import dataclass, field
+from time import perf_counter
+from urllib.parse import urlparse
+
+from knowledge_base.config import cfg
+from knowledge_base.connectors.search_backends import get_configured_backends
+from knowledge_base.connectors.web_fetcher import fetch_url, is_safe_fetch_target
+from knowledge_base.parsers.html_parser import parse_html_page
+from knowledge_base.retrieval.reranker import rerank_web_pages
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class WebHit:
+    url: str
+    title: str
+    content: str
+    source_type: str
+    score: float = 0.5
+    metadata: dict = field(default_factory=dict)
+
+
+def _parse_domain_list(raw: str | tuple[str, ...] | None) -> tuple[str, ...]:
+    if not raw:
+        return ()
+    if isinstance(raw, tuple):
+        return tuple(s.strip() for s in raw if s and str(s).strip())
+    return tuple(s.strip() for s in str(raw).split(",") if s.strip())
+
+
+def _domains_from_sources() -> tuple[str, ...]:
+    """Resolve WEB_SOURCES aliases into host allowlist entries."""
+    raw_sources = getattr(cfg, "WEB_SOURCES", ())
+    if isinstance(raw_sources, str):
+        sources = [s.strip().lower() for s in raw_sources.split(",") if s.strip()]
+    else:
+        sources = [str(s).strip().lower() for s in raw_sources if str(s).strip()]
+
+    domains: list[str] = []
+    for source in sources:
+        if source == "kb_igel" and getattr(cfg, "KB_IGEL_ENABLED", True):
+            host = urlparse(str(getattr(cfg, "KB_IGEL_BASE_URL", "")).strip()).netloc.lower()
+            if host:
+                domains.append(host)
+        elif source == "community" and getattr(cfg, "COMMUNITY_ENABLED", True):
+            host = urlparse(str(getattr(cfg, "COMMUNITY_BASE_URL", "")).strip()).netloc.lower()
+            if host:
+                domains.append(host)
+
+    # Preserve order while deduplicating.
+    seen: set[str] = set()
+    out: list[str] = []
+    for d in domains:
+        if d not in seen:
+            out.append(d)
+            seen.add(d)
+    return tuple(out)
+
+
+def _fetch_page_content_httpx(url: str) -> tuple[str, str]:
+    """Fallback: httpx GET + HTML parse."""
+    cache_ttl = int(getattr(cfg, "WEB_CACHE_TTL_SECONDS", 3600))
+    result = fetch_url(url, cache_ttl=cache_ttl, strip_promoted=True)
+    if result.status_code != 200 or not result.html:
+        return url, ""
+
+    parsed = parse_html_page(result.html, url, source_type="web_search")
+    title = parsed.title or url
+    content = parsed.raw_text or ""
+    return title, content
+
+
+def _extract_crawl4ai_markdown(result: object) -> str:
+    """Normalize markdown across Crawl4AI versions."""
+    for attr in ("markdown", "fit_markdown", "cleaned_html"):
+        val = getattr(result, attr, None)
+        if isinstance(val, str) and val.strip():
+            return val.strip()
+    md_v2 = getattr(result, "markdown_v2", None)
+    if md_v2 is not None:
+        raw = getattr(md_v2, "raw_markdown", None) or getattr(md_v2, "fit_markdown", None)
+        if isinstance(raw, str) and raw.strip():
+            return raw.strip()
+    return ""
+
+
+async def _fetch_page_content_crawl4ai(url: str, query: str) -> tuple[str, str]:
+    """Fetch URL with Crawl4AI Playwright crawler (LLM-ready markdown)."""
+    try:
+        from crawl4ai import AsyncWebCrawler
+    except ImportError as e:
+        raise ImportError("crawl4ai not installed") from e
+
+    timeout = int(getattr(cfg, "CRAWL4AI_TIMEOUT", 30))
+    run_config = None
+    try:
+        from crawl4ai import CrawlerRunConfig
+
+        filter_kwargs: dict = {}
+        if query.strip():
+            try:
+                from crawl4ai.content_filter_strategy import BM25ContentFilter
+
+                filter_kwargs["content_filter"] = BM25ContentFilter(user_query=query)
+            except ImportError:
+                try:
+                    from crawl4ai.content_filter_strategy import PruningContentFilter
+
+                    filter_kwargs["content_filter"] = PruningContentFilter()
+                except ImportError:
+                    pass
+        run_config = CrawlerRunConfig(**filter_kwargs) if filter_kwargs else CrawlerRunConfig()
+    except ImportError:
+        run_config = None
+
+    async with AsyncWebCrawler(verbose=False) as crawler:
+        if run_config is not None:
+            result = await asyncio.wait_for(crawler.arun(url=url, config=run_config), timeout=timeout)
+        else:
+            result = await asyncio.wait_for(crawler.arun(url=url), timeout=timeout)
+
+    if not result or not getattr(result, "success", True):
+        return url, ""
+
+    title = getattr(result, "title", None) or url
+    markdown = _extract_crawl4ai_markdown(result)
+    return title, markdown
+
+
+def _fetch_page_content_playwright(url: str) -> tuple[str, str]:
+    from knowledge_base.connectors.playwright_fetcher import fetch_page_playwright, playwright_available
+
+    if not playwright_available():
+        raise ImportError("playwright not installed")
+    timeout_ms = int(getattr(cfg, "CRAWL4AI_TIMEOUT", 30)) * 1000
+    return fetch_page_playwright(url, timeout_ms=timeout_ms)
+
+
+def _fetch_page_content(url: str, query: str = "") -> tuple[str, str]:
+    """Fetch page content — Crawl4AI → Playwright → httpx fallback.
+
+    Crawl4AI uses an async crawler. To avoid conflicting with the running
+    uvicorn/FastAPI event loop, the coroutine is executed in a fresh thread
+    (which has no pre-existing loop) via ThreadPoolExecutor.
+    Playwright fallback is intentionally outside the CRAWL4AI_ENABLED guard
+    so it remains available when Crawl4AI is disabled or fails.
+    """
+    if getattr(cfg, "CRAWL4AI_ENABLED", True):
+        try:
+            import concurrent.futures
+            with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
+                return pool.submit(asyncio.run, _fetch_page_content_crawl4ai(url, query)).result()
+        except ImportError:
+            logger.debug("crawl4ai unavailable — trying Playwright fetcher")
+        except Exception as e:
+            logger.warning("crawl4ai fetch failed for %s: %s", url, e)
+
+    # Playwright fallback — runs independently of CRAWL4AI_ENABLED
+    try:
+        return _fetch_page_content_playwright(url)
+    except ImportError:
+        logger.debug("playwright unavailable — httpx fallback for %s", url)
+    except Exception as e:
+        logger.warning("playwright fetch failed for %s: %s — httpx fallback", url, e)
+
+    return _fetch_page_content_httpx(url)
+
+
+def retrieve_web_context(
+    query: str,
+    *,
+    max_pages: int | None = None,
+    local_top_score: float | None = None,
+) -> list[WebHit]:
+    """
+    Stage 2 live web search — only runs when local RAG confidence is below threshold.
+
+    Backends (WEB_SEARCH_BACKENDS):
+      - firecrawl → self-hosted Firecrawl search (primary)
+      - ddgs      → DuckDuckGo organic search (fallback)
+      - searxng   → Self-hosted SearXNG (optional)
+    """
+    if not getattr(cfg, "WEB_RETRIEVAL_ENABLED", False):
+        return []
+    if not query.strip():
+        return []
+
+    threshold = float(getattr(cfg, "CONFIDENCE_THRESHOLD", 0.75))
+    if local_top_score is not None and local_top_score >= threshold:
+        logger.info(
+            "confidence gate: local top score %.3f >= %.3f — skipping web search for %r",
+            local_top_score,
+            threshold,
+            query[:60],
+        )
+        return []
+
+    if local_top_score is not None:
+        logger.info(
+            "confidence gate: local top score %.3f < %.3f — escalating to web search for %r",
+            local_top_score,
+            threshold,
+            query[:60],
+        )
+    else:
+        logger.info("confidence gate: no local score — running web search for %r", query[:60])
+
+    max_pages = max_pages or int(getattr(cfg, "WEB_MAX_PAGES", 5))
+    min_score = float(getattr(cfg, "WEB_MIN_SCORE", 0.15))
+    include_domains = _parse_domain_list(getattr(cfg, "WEB_SEARCH_INCLUDE_DOMAINS", ()))
+    exclude_domains = _parse_domain_list(getattr(cfg, "WEB_SEARCH_EXCLUDE_DOMAINS", ()))
+    include_from_sources = _domains_from_sources()
+    if include_from_sources:
+        include_domains = tuple(dict.fromkeys((*include_domains, *include_from_sources)))
+
+    backends = get_configured_backends()
+    if not backends:
+        logger.warning("no web search backends configured (WEB_SEARCH_BACKENDS)")
+        return []
+
+    t0 = perf_counter()
+    seen_urls: set[str] = set()
+    candidates: list[tuple[str, str, str, str]] = []  # url, title, content, backend
+
+    per_backend = max(1, math.ceil(max_pages / max(len(backends), 1)))
+    for backend in backends:
+        try:
+            search_results = backend.search(
+                query,
+                max_results=per_backend,
+                include_domains=include_domains or None,
+                exclude_domains=exclude_domains or None,
+            )
+        except Exception as e:
+            logger.warning("web backend %s search failed: %s", backend.name, e)
+            continue
+
+        for sr in search_results:
+            if not is_safe_fetch_target(sr.url):
+                logger.warning("skipping unsafe web target from backend %s: %s", backend.name, sr.url)
+                continue
+            norm = sr.url.rstrip("/").lower()
+            if norm in seen_urls:
+                continue
+            seen_urls.add(norm)
+
+            # Firecrawl may return full markdown in snippet — skip re-fetch if substantial.
+            if sr.snippet and len(sr.snippet.strip()) >= 400 and sr.metadata.get("has_markdown"):
+                title = sr.title or sr.url
+                content = sr.snippet[:12000]
+            else:
+                title, content = _fetch_page_content(sr.url, query)
+                if not content and sr.snippet:
+                    content = sr.snippet
+            if not title:
+                title = sr.title or sr.url
+
+            if len(content.strip()) < 40 and not sr.snippet:
+                continue
+
+            candidates.append((sr.url, title, content[:12000], backend.name))
+
+    if not candidates:
+        logger.info("retrieve_web_context('%s') → 0 candidates", query[:50])
+        return []
+
+    pages_for_rerank = [(url, title, content) for url, title, content, _ in candidates]
+    ranked = rerank_web_pages(query, pages_for_rerank)
+    score_by_url = {url: score for url, score in ranked}
+
+    hits: list[WebHit] = []
+    backend_by_url = {url: backend for url, _, _, backend in candidates}
+    for url, title, content, backend in candidates:
+        score = float(score_by_url.get(url, 0.0))
+        if score < min_score:
+            continue
+        hits.append(
+            WebHit(
+                url=url,
+                title=title,
+                content=content,
+                source_type="web_search",
+                score=score,
+                metadata={
+                    "source_type": "web_search",
+                    "canonical_url": url,
+                    "web_source": backend_by_url.get(url, backend),
+                    "confidence_gate": "escalated",
+                    "local_top_score": local_top_score,
+                },
+            )
+        )
+
+    hits.sort(key=lambda h: h.score, reverse=True)
+    hits = hits[:max_pages]
+
+    elapsed = (perf_counter() - t0) * 1000
+    logger.info(
+        "retrieve_web_context('%s') → %d hits in %.0fms (backends=%s, gate=open)",
+        query[:50],
+        len(hits),
+        elapsed,
+        [b.name for b in backends],
+    )
+    return hits
+
+
+def format_web_block(hits: list[WebHit], *, max_chars: int = 6000) -> str:
+    """Format web hits for LLM context injection."""
+    if not hits:
+        return ""
+    lines = ["—— Live web sources (Firecrawl / organic search) ——"]
+    used = 0
+    for hit in hits:
+        backend = hit.metadata.get("web_source", hit.source_type)
+        block = (
+            f"\n### [{backend}] {hit.title}\n"
+            f"URL: {hit.url}\n"
+            f"Relevance: {hit.score:.2f}\n\n"
+            f"{hit.content[:4000]}"
+        )
+        if used + len(block) > max_chars:
+            remain = max_chars - used - 80
+            if remain > 200:
+                lines.append(block[:remain] + "\n[...truncated]")
+            break
+        lines.append(block)
+        used += len(block)
+    return "\n".join(lines)

package/knowledge_base/validators/__init__.py

@@ -0,0 +1 @@
+"""Post-generation validators for test case fidelity."""

package/knowledge_base/validators/step_fidelity.py

@@ -0,0 +1,88 @@
+"""
+Post-generation checks: preserve gold testcase step count and avoid hollow expected results.
+"""
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from knowledge_base.generation.manual_tc_parser import ParsedStep
+
+
+_VAGUE_PHRASES = (
+    "works as expected",
+    "verify success",
+    "test passes",
+    "everything works",
+    "as expected.",
+    "proper behavior",
+    "configuration is correct",
+)
+
+
+def count_pytest_steps(python_code: str) -> int:
+    """Count numbered step functions like `def TC001_step1` or `def QCL_4851_step2`."""
+    if not python_code:
+        return 0
+    # Primary: _stepN( before cleanup
+    n = len(re.findall(r"def\s+[A-Za-z0-9_]+_step\d+\s*\(", python_code))
+    if n:
+        return n
+    return len(re.findall(r"@allure\.step\(", python_code))
+
+
+def validate_transform_fidelity(
+    parsed_steps: list,
+    python_code: str | None,
+    markdown: str | None,
+) -> list[str]:
+    """
+    For TRANSFORM mode: core steps in generated Python should match parsed manual step count.
+    Allows one extra step for cleanup pattern.
+    """
+    warnings: list[str] = []
+    if not parsed_steps:
+        return warnings
+
+    target = len(parsed_steps)
+    if python_code:
+        got = count_pytest_steps(python_code)
+        if got == 0:
+            warnings.append("FIDELITY: No stepN functions detected in generated Python.")
+        elif got < target:
+            warnings.append(
+                f"FIDELITY: Generated {got} automation steps but manual TC has {target} — "
+                "possible dropped steps (review required)."
+            )
+        elif got > target + 1:
+            warnings.append(
+                f"FIDELITY: Generated {got} steps vs manual {target} — "
+                "possible invented steps (only +1 allowed for cleanup)."
+            )
+
+    if markdown:
+        low = markdown.lower()
+        for phrase in _VAGUE_PHRASES:
+            if phrase in low:
+                warnings.append(f"FIDELITY: Markdown contains vague phrase: {phrase!r}")
+                break
+
+    return warnings
+
+
+def validate_expand_preservation(
+    parsed_steps: list,
+    markdown: str | None,
+) -> list[str]:
+    """Heuristic: original step titles should still appear in expanded markdown."""
+    out: list[str] = []
+    if not parsed_steps or not markdown:
+        return out
+    for s in parsed_steps[:20]:
+        title = (s.step_title or s.test_data or "")[:40].strip()
+        if len(title) > 12 and title.lower() not in markdown.lower():
+            out.append(
+                f"FIDELITY_EXPAND: Original step fragment not found in output: {title[:60]!r}…"
+            )
+    return out[:5]