igel-qe-core 1.0.0

package/knowledge_base/connectors/testRail_connector.py

@@ -0,0 +1,25 @@
+"""TestRail connector — Phase 8 stub."""
+from __future__ import annotations
+
+from datetime import datetime
+
+from knowledge_base.config import cfg
+from knowledge_base.connectors.base import BaseConnector
+
+
+class TestRailConnector(BaseConnector):
+    name = "testrail"
+    enabled = cfg.TESTRAIL_ENABLED
+    supports_sync = False
+
+    def connect(self) -> bool:
+        return self.enabled
+
+    def sync(self, since: datetime | None = None) -> list[dict]:
+        return []
+
+    def ingest_to_kg(self, items: list[dict]) -> int:
+        return 0
+
+    def ingest_to_rag(self, items: list[dict]) -> int:
+        return 0

package/knowledge_base/connectors/web_fetcher.py

@@ -0,0 +1,280 @@
+"""Generic HTTP fetcher with in-memory cache for web retrieval."""
+
+from __future__ import annotations
+
+import logging
+import socket
+import time
+from ipaddress import ip_address
+from dataclasses import dataclass
+from threading import Lock
+from typing import Optional
+from urllib.parse import quote_plus, urljoin, urlparse
+
+from knowledge_base.config import cfg
+
+logger = logging.getLogger(__name__)
+
+_CACHE: dict[str, tuple[float, str]] = {}
+_CACHE_LOCK = Lock()
+_DEFAULT_HEADERS = {
+    "User-Agent": "IGEL-KB-Agent/1.0 (+https://kb.igel.com; internal-test-automation)",
+    "Accept": "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9,de;q=0.8",
+}
+
+
+@dataclass
+class FetchResult:
+    url: str
+    html: str
+    status_code: int
+    from_cache: bool = False
+
+
+def is_safe_fetch_target(url: str) -> bool:
+    """Block unsafe SSRF targets unless explicitly enabled by config."""
+    parsed = urlparse(url)
+    if parsed.scheme not in ("http", "https"):
+        return False
+
+    host = (parsed.hostname or "").strip().lower()
+    if not host:
+        return False
+
+    allow_private = bool(getattr(cfg, "WEB_ALLOW_PRIVATE_HOSTS", False))
+    if host in {"localhost"} or host.endswith(".localhost") or host.endswith(".local"):
+        return allow_private
+
+    try:
+        ip = ip_address(host)
+    except ValueError:
+        if allow_private:
+            return True
+        return _host_resolves_public_ips(host)
+
+    if allow_private:
+        return True
+    return _is_public_ip(ip)
+
+
+def _is_public_ip(ip) -> bool:
+    return not (
+        ip.is_private
+        or ip.is_loopback
+        or ip.is_link_local
+        or ip.is_multicast
+        or ip.is_reserved
+        or ip.is_unspecified
+    )
+
+
+def _host_resolves_public_ips(host: str) -> bool:
+    """Return True only when DNS answers resolve exclusively to public IPs."""
+    try:
+        infos = socket.getaddrinfo(host, None)
+    except Exception:
+        return False
+    if not infos:
+        return False
+    for info in infos:
+        addr = info[4][0]
+        try:
+            ip = ip_address(addr)
+        except ValueError:
+            return False
+        if not _is_public_ip(ip):
+            return False
+    return True
+
+
+def _cache_get(url: str, ttl: int) -> Optional[str]:
+    with _CACHE_LOCK:
+        entry = _CACHE.get(url)
+        if not entry:
+            return None
+        ts, html = entry
+        if time.time() - ts > ttl:
+            _CACHE.pop(url, None)
+            return None
+        return html
+
+
+def _cache_set(url: str, html: str) -> None:
+    with _CACHE_LOCK:
+        _CACHE[url] = (time.time(), html)
+        if len(_CACHE) > 256:
+            oldest = sorted(_CACHE.items(), key=lambda kv: kv[1][0])[:32]
+            for key, _ in oldest:
+                _CACHE.pop(key, None)
+
+
+def strip_promoted_content(html: str, patterns: list[str] | tuple[str, ...] | None = None) -> str:
+    """Remove ads, sponsored blocks, and promotional elements from HTML."""
+    if not html:
+        return html
+    try:
+        from bs4 import BeautifulSoup
+    except ImportError:
+        return html
+
+    try:
+        soup = BeautifulSoup(html, "lxml")
+    except Exception:
+        soup = BeautifulSoup(html, "html.parser")
+    for tag in soup.find_all(["aside", "ins"]):
+        tag.decompose()
+
+    promo_patterns = list(patterns or getattr(cfg, "STRIP_PROMOTED_PATTERNS", ()))
+    for pattern in promo_patterns:
+        p = (pattern or "").strip().lower()
+        if not p:
+            continue
+        for el in soup.find_all(True):
+            classes = el.get("class") or []
+            class_str = " ".join(str(c) for c in classes).lower()
+            el_id = (el.get("id") or "").lower()
+            if p in class_str or p in el_id:
+                el.decompose()
+    return str(soup)
+
+
+def fetch_url(
+    url: str,
+    *,
+    timeout: float = 20.0,
+    cache_ttl: int = 3600,
+    strip_promoted: bool = False,
+) -> FetchResult:
+    """Fetch a URL and return HTML using httpx with explicit safe-redirect handling."""
+    if not is_safe_fetch_target(url):
+        logger.warning("fetch_url blocked unsafe target: %s", url)
+        return FetchResult(url=url, html="", status_code=0)
+
+    cached = _cache_get(url, cache_ttl)
+    if cached is not None:
+        html = cached
+        if strip_promoted:
+            html = strip_promoted_content(html)
+        return FetchResult(url=url, html=html, status_code=200, from_cache=True)
+
+    html = ""
+    status = 0
+    try:
+        import httpx
+
+        with httpx.Client(follow_redirects=False, timeout=timeout, headers=_DEFAULT_HEADERS) as client:
+            current = url
+            max_redirects = 6
+            for _ in range(max_redirects + 1):
+                if not is_safe_fetch_target(current):
+                    logger.warning("fetch_url blocked unsafe target: %s", current)
+                    return FetchResult(url=url, html="", status_code=0)
+                resp = client.get(current)
+                if resp.is_redirect:
+                    location = resp.headers.get("location")
+                    if not location:
+                        status = resp.status_code
+                        html = resp.text
+                        break
+                    nxt = urljoin(current, location)
+                    if not is_safe_fetch_target(nxt):
+                        logger.warning("fetch_url blocked unsafe redirect target: %s -> %s", current, nxt)
+                        return FetchResult(url=url, html="", status_code=0)
+                    current = nxt
+                    continue
+                status = resp.status_code
+                html = resp.text
+                break
+            else:
+                logger.warning("fetch_url exceeded redirect limit for %s", url)
+                return FetchResult(url=url, html="", status_code=0)
+    except ImportError:
+        logger.warning("httpx is required for fetch_url but is not installed")
+        return FetchResult(url=url, html="", status_code=0)
+    except Exception as e:
+        logger.warning("fetch_url failed for %s: %s", url, e)
+        return FetchResult(url=url, html="", status_code=status or 0)
+
+    if status == 200 and html:
+        if strip_promoted:
+            html = strip_promoted_content(html)
+        _cache_set(url, html)
+    return FetchResult(url=url, html=html, status_code=status)
+
+
+def search_confluence_site(base_url: str, query: str, *, limit: int = 5) -> list[str]:
+    """
+    Search a Confluence/Scroll Sites KB (e.g. kb.igel.com) via REST API.
+    Returns list of page URLs.
+    """
+    base = base_url.rstrip("/")
+    cql = f'siteSearch ~ "{query.replace(chr(34), " ")}" AND type = page'
+    api = f"{base}/rest/api/content/search?cql={quote_plus(cql)}&limit={limit}"
+
+    try:
+        import httpx
+
+        with httpx.Client(follow_redirects=False, timeout=20.0, headers=_DEFAULT_HEADERS) as client:
+            current = api
+            data = None
+            max_redirects = 6
+            for _ in range(max_redirects + 1):
+                if not is_safe_fetch_target(current):
+                    logger.warning("Confluence search blocked unsafe target: %s", current)
+                    return _fallback_search_urls(base, query, limit=limit)
+                resp = client.get(current)
+                if resp.is_redirect:
+                    location = resp.headers.get("location")
+                    if not location:
+                        break
+                    nxt = urljoin(current, location)
+                    if not is_safe_fetch_target(nxt):
+                        logger.warning("Confluence search blocked unsafe redirect target: %s -> %s", current, nxt)
+                        return _fallback_search_urls(base, query, limit=limit)
+                    current = nxt
+                    continue
+
+                if resp.status_code != 200:
+                    logger.debug("Confluence search HTTP %s for %s", resp.status_code, current)
+                    return _fallback_search_urls(base, query, limit=limit)
+                data = resp.json()
+                break
+
+            if data is None:
+                logger.debug("Confluence search redirect resolution failed for %s", api)
+                return _fallback_search_urls(base, query, limit=limit)
+    except Exception as e:
+        logger.debug("Confluence search failed: %s", e)
+        return _fallback_search_urls(base, query, limit=limit)
+
+    urls: list[str] = []
+    for item in data.get("results", []):
+        links = item.get("_links", {})
+        webui = links.get("webui") or links.get("tinyui")
+        if webui:
+            urls.append(urljoin(base + "/", webui.lstrip("/")))
+        elif item.get("id"):
+            urls.append(f"{base}/pages/viewpage.action?pageId={item['id']}")
+    return urls[:limit]
+
+
+def _fallback_search_urls(base_url: str, query: str, *, limit: int) -> list[str]:
+    """Construct likely KB URLs from query keywords when REST search is unavailable."""
+    base = base_url.rstrip("/")
+    keywords = [w.lower() for w in query.split() if len(w) > 3][:4]
+    paths = [
+        "/en/igel-os-base-system",
+        "/en/universal-management-suite",
+        "/en/igel-cloud-gateway",
+        "/en/security-safety",
+        "/en/how-to-start-with-igel",
+    ]
+    urls: list[str] = []
+    for kw in keywords:
+        for path in paths:
+            if kw in path.replace("-", " "):
+                urls.append(f"{base}{path}")
+    if not urls:
+        urls.append(f"{base}/search?query={quote_plus(query)}")
+    return urls[:limit]

package/knowledge_base/core/__init__.py

@@ -0,0 +1,8 @@
+"""Core scoring and precedence rules."""
+
+from knowledge_base.core.confidence import flatten_confidence_for_api, score_generation
+
+__all__ = [
+    "flatten_confidence_for_api",
+    "score_generation",
+]

package/knowledge_base/core/cache.py

@@ -0,0 +1,98 @@
+"""Query/embedding cache with optional Redis backend."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+import threading
+import time
+from functools import lru_cache
+
+logger = logging.getLogger(__name__)
+
+_redis_client = None
+_redis_lock = threading.Lock()
+
+
+def _redis():
+    global _redis_client
+    if _redis_client is not None:
+        return _redis_client
+    with _redis_lock:
+        if _redis_client is not None:
+            return _redis_client
+        try:
+            from knowledge_base.config import cfg
+            if not getattr(cfg, "REDIS_URL", ""):
+                return None
+            import redis
+            _redis_client = redis.from_url(cfg.REDIS_URL, decode_responses=True)
+            _redis_client.ping()
+            return _redis_client
+        except Exception as exc:
+            logger.debug("Redis unavailable: %s", exc)
+            return None
+
+
+def _key(prefix: str, payload: str) -> str:
+    h = hashlib.sha256(payload.encode()).hexdigest()[:24]
+    return f"igel:kb:{prefix}:{h}"
+
+
+def cache_get(prefix: str, payload: str) -> str | None:
+    r = _redis()
+    if not r:
+        return None
+    try:
+        return r.get(_key(prefix, payload))
+    except Exception:
+        return None
+
+
+def cache_set(prefix: str, payload: str, value: str, ttl: int) -> None:
+    r = _redis()
+    if not r:
+        return
+    try:
+        r.setex(_key(prefix, payload), ttl, value)
+    except Exception:
+        pass
+
+
+@lru_cache(maxsize=512)
+def embed_query_cached(query: str) -> tuple[float, ...] | None:
+    """In-process LRU for embedding vectors (serialized as tuple)."""
+    return None
+
+
+def store_embedding_cache(query: str, vector: list[float]) -> None:
+    embed_query_cached.cache_clear()
+    # LRU handled by caller storing in Redis for cross-process
+    payload = json.dumps(vector)
+    from knowledge_base.config import cfg
+    cache_set("embed", query.strip().lower(), payload, getattr(cfg, "EMBED_CACHE_TTL", 3600))
+
+
+def get_embedding_cache(query: str) -> list[float] | None:
+    raw = cache_get("embed", query.strip().lower())
+    if not raw:
+        return None
+    try:
+        return json.loads(raw)
+    except Exception:
+        return None
+
+
+def get_retrieval_cache(key: str) -> dict | None:
+    raw = cache_get("retrieve", key)
+    if not raw:
+        return None
+    try:
+        return json.loads(raw)
+    except Exception:
+        return None
+
+
+def set_retrieval_cache(key: str, data: dict, ttl: int = 300) -> None:
+    cache_set("retrieve", key, json.dumps(data, default=str), ttl)

package/knowledge_base/core/circuit_breaker.py

@@ -0,0 +1,33 @@
+"""MCP circuit breaker for external connectors."""
+
+from __future__ import annotations
+
+import time
+import threading
+
+_lock = threading.Lock()
+_failures: dict[str, int] = {}
+_disabled_until: dict[str, float] = {}
+
+
+def record_failure(source: str, *, threshold: int = 3, cooldown_sec: int = 300) -> None:
+    with _lock:
+        _failures[source] = _failures.get(source, 0) + 1
+        if _failures[source] >= threshold:
+            _disabled_until[source] = time.time() + cooldown_sec
+
+
+def record_success(source: str) -> None:
+    with _lock:
+        _failures[source] = 0
+        _disabled_until.pop(source, None)
+
+
+def is_open(source: str) -> bool:
+    with _lock:
+        until = _disabled_until.get(source)
+        if until and time.time() < until:
+            return True
+        if until:
+            _disabled_until.pop(source, None)
+        return False