igel-qe-core 1.0.0

package/knowledge_base/connectors/query_bridge.py

@@ -0,0 +1,170 @@
+"""Live query bridge — MCP / external connectors at query time."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+from time import perf_counter
+
+from knowledge_base.config import cfg
+from knowledge_base.connectors.base import BaseConnector
+from knowledge_base.core.circuit_breaker import is_open, record_failure, record_success
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class ExternalHit:
+    source: str
+    title: str
+    url: str
+    content: str
+    score: float = 0.5
+    source_type: str = "external"
+
+
+@dataclass
+class ExternalContext:
+    hits: list[ExternalHit] = field(default_factory=list)
+    block: str = ""
+    sources_queried: list[str] = field(default_factory=list)
+    timing_ms: float = 0.0
+
+
+def _mcp_sources() -> list[str]:
+    raw = getattr(cfg, "QUERY_MCP_SOURCES", ("jira", "confluence"))
+    if isinstance(raw, str):
+        return [s.strip().lower() for s in raw.split(",") if s.strip()]
+    return [s.strip().lower() for s in raw if s.strip()]
+
+
+def _connector_for(name: str):
+    # Keep this registry aligned with sync_scheduler CONNECTORS.
+    if name == "jira":
+        from knowledge_base.connectors.jira_connector import JiraConnector
+        return JiraConnector()
+    if name == "confluence":
+        from knowledge_base.connectors.confluence_connector import ConfluenceConnector
+        return ConfluenceConnector()
+    if name == "bitbucket":
+        from knowledge_base.connectors.bitbucket_connector import BitbucketConnector
+        return BitbucketConnector()
+    if name == "teamcity":
+        from knowledge_base.connectors.teamcity_connector import TeamCityConnector
+        return TeamCityConnector()
+    if name == "testrail":
+        from knowledge_base.connectors.testRail_connector import TestRailConnector
+        return TestRailConnector()
+    if name == "servicenow":
+        from knowledge_base.connectors.serviceNow_connector import ServiceNowConnector
+        return ServiceNowConnector()
+    if name == "kb_igel":
+        from knowledge_base.connectors.kb_igel_connector import KbIgelConnector
+        return KbIgelConnector()
+    if name == "community":
+        from knowledge_base.connectors.community_connector import CommunityConnector
+        return CommunityConnector()
+    logger.warning("Unsupported MCP source configured: %s", name)
+    return None
+
+
+def _has_query_impl(connector: BaseConnector) -> bool:
+    """True when connector overrides query-time methods instead of base no-op stubs."""
+    return (
+        connector.__class__.search is not BaseConnector.search
+        or connector.__class__.fetch_page is not BaseConnector.fetch_page
+    )
+
+
+def query_mcp_sources(
+    query: str,
+    *,
+    max_hits: int = 3,
+    timeout_seconds: float | None = None,
+) -> ExternalContext:
+    if not getattr(cfg, "QUERY_EXTERNAL_ENABLED", True) or not query.strip():
+        return ExternalContext()
+
+    hits: list[ExternalHit] = []
+    queried: list[str] = []
+    t0 = perf_counter()
+    deadline = perf_counter() + timeout_seconds if timeout_seconds and timeout_seconds > 0 else None
+
+    for name in _mcp_sources():
+        if deadline is not None and perf_counter() >= deadline:
+            logger.warning("MCP query budget exhausted before source=%s", name)
+            break
+        if len(hits) >= max_hits:
+            break
+        if is_open(name):
+            continue
+        connector = _connector_for(name)
+        if connector is None or not getattr(connector, "enabled", False):
+            continue
+        if not _has_query_impl(connector):
+            continue
+        queried.append(name)
+        try:
+            source_success = False
+            remaining = None
+            if deadline is not None:
+                remaining = max(0.05, deadline - perf_counter())
+                if remaining <= 0:
+                    logger.warning("MCP query budget exhausted before search for source=%s", name)
+                    break
+
+            urls = connector.search(query, limit=max_hits, timeout=remaining)
+            if urls:
+                for url in urls[:max_hits]:
+                    if deadline is not None and perf_counter() >= deadline:
+                        logger.warning("MCP query budget exhausted while processing source=%s", name)
+                        break
+                    if len(hits) >= max_hits:
+                        break
+                    remaining = None
+                    if deadline is not None:
+                        remaining = max(0.05, deadline - perf_counter())
+                        if remaining <= 0:
+                            logger.warning("MCP query budget exhausted before fetch_page for source=%s", name)
+                            break
+
+                    page = connector.fetch_page(url, timeout=remaining)
+                    title = url
+                    if page:
+                        content = (page.get("raw_text") or page.get("content") or "").strip()[:6000]
+                        title = page.get("title") or url
+                        if content:
+                            source_success = True
+                            hits.append(ExternalHit(
+                                source=name,
+                                title=title,
+                                url=url,
+                                content=content,
+                                source_type=page.get("source_type", name),
+                            ))
+            elif not urls:
+                logger.debug("MCP source %s returned no URLs", name)
+
+            if source_success:
+                record_success(name)
+        except Exception as e:
+            record_failure(name)
+            logger.warning("MCP source %s query failed: %s", name, e)
+
+    block = _format_block(hits)
+    elapsed = (perf_counter() - t0) * 1000
+    return ExternalContext(hits=hits, block=block, sources_queried=queried, timing_ms=elapsed)
+
+
+def _format_block(hits: list[ExternalHit], *, max_chars: int = 4000) -> str:
+    if not hits:
+        return ""
+    lines = ["—— MCP / external connector sources ——"]
+    used = 0
+    for hit in hits:
+        block = f"\n### [{hit.source}] {hit.title}\nURL: {hit.url}\n{hit.content[:2500]}"
+        if used + len(block) > max_chars:
+            break
+        lines.append(block)
+        used += len(block)
+    return "\n".join(lines)

package/knowledge_base/connectors/search_backends.py

@@ -0,0 +1,368 @@
+"""Free/open-source web search backends for Stage 2 live web retrieval."""
+
+from __future__ import annotations
+
+import logging
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import quote_plus, urlparse
+
+from knowledge_base.config import cfg
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class WebSearchResult:
+    url: str
+    title: str
+    snippet: str = ""
+    backend: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+class SearchBackend(ABC):
+    name: str = "base"
+
+    @abstractmethod
+    def search(
+        self,
+        query: str,
+        *,
+        max_results: int = 5,
+        include_domains: tuple[str, ...] | None = None,
+        exclude_domains: tuple[str, ...] | None = None,
+    ) -> list[WebSearchResult]:
+        raise NotImplementedError
+
+
+def _normalize_url(url: str) -> str:
+    parsed = urlparse(url.strip())
+    path = parsed.path.rstrip("/") or "/"
+    return f"{parsed.scheme}://{parsed.netloc.lower()}{path}"
+
+
+def _host_matches(host: str, pattern: str) -> bool:
+    p = pattern.strip().lower().lstrip(".")
+    if not p:
+        return False
+    return host == p or host.endswith("." + p)
+
+
+def _domain_allowed(
+    url: str,
+    include_domains: tuple[str, ...] | None,
+    exclude_domains: tuple[str, ...] | None,
+) -> bool:
+    host = urlparse(url).netloc.lower()
+    if not host:
+        return False
+    if exclude_domains:
+        for pattern in exclude_domains:
+            if _host_matches(host, pattern):
+                return False
+    if include_domains:
+        return any(_host_matches(host, p) for p in include_domains if p.strip())
+    return True
+
+
+class DuckDuckGoBackend(SearchBackend):
+    """Organic web search via duckduckgo-search / ddgs (free, no API key)."""
+
+    name = "ddgs"
+
+    def search(
+        self,
+        query: str,
+        *,
+        max_results: int = 5,
+        include_domains: tuple[str, ...] | None = None,
+        exclude_domains: tuple[str, ...] | None = None,
+    ) -> list[WebSearchResult]:
+        if not query.strip():
+            return []
+
+        backend = getattr(cfg, "DDGS_BACKEND", "auto")
+        region = getattr(cfg, "DDGS_REGION", "wt-wt")
+        fetch_n = max(max_results * 3, max_results + 2)
+
+        raw_rows: list[dict[str, str]] = []
+        backends_to_try = [backend] if backend not in ("auto", "") else ["api", "html", "lite"]
+        last_err: Exception | None = None
+
+        try:
+            try:
+                from ddgs import DDGS
+            except ImportError:
+                from duckduckgo_search import DDGS
+
+            ddgs = DDGS()
+            search_fn = getattr(ddgs, "text", None)
+            if search_fn is None:
+                return []
+
+            for be in backends_to_try:
+                try:
+                    try:
+                        raw_rows = list(
+                            search_fn(
+                                query,
+                                region=region,
+                                safesearch="moderate",
+                                max_results=fetch_n,
+                                backend=be,
+                            )
+                            or []
+                        )
+                    except TypeError:
+                        raw_rows = list(
+                            search_fn(
+                                query,
+                                region=region,
+                                safesearch="moderate",
+                                max_results=fetch_n,
+                            )
+                            or []
+                        )
+                    if raw_rows:
+                        break
+                except Exception as e:
+                    last_err = e
+                    logger.debug("DuckDuckGo backend=%s failed: %s", be, e)
+                    continue
+
+            if not raw_rows and last_err is not None:
+                logger.warning("DuckDuckGo search failed after retries: %s", last_err)
+                return []
+        except Exception as e:
+            logger.warning("DuckDuckGo search failed: %s", e)
+            return []
+
+        results: list[WebSearchResult] = []
+        seen: set[str] = set()
+        for row in raw_rows:
+            url = (row.get("href") or row.get("url") or "").strip()
+            if not url or not url.startswith("http"):
+                continue
+            norm = _normalize_url(url)
+            if norm in seen:
+                continue
+            if not _domain_allowed(url, include_domains, exclude_domains):
+                continue
+            seen.add(norm)
+            results.append(
+                WebSearchResult(
+                    url=url,
+                    title=(row.get("title") or url)[:300],
+                    snippet=(row.get("body") or row.get("snippet") or "")[:2000],
+                    backend=self.name,
+                )
+            )
+            if len(results) >= max_results:
+                break
+        logger.info("DuckDuckGoBackend: %d results for %r", len(results), query[:60])
+        return results
+
+
+class SearXNGBackend(SearchBackend):
+    """Self-hosted SearXNG metasearch (free, configurable, no promoted results)."""
+
+    name = "searxng"
+
+    def __init__(self) -> None:
+        self.base_url = (getattr(cfg, "SEARXNG_BASE_URL", "") or "").rstrip("/")
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.base_url)
+
+    def search(
+        self,
+        query: str,
+        *,
+        max_results: int = 5,
+        include_domains: tuple[str, ...] | None = None,
+        exclude_domains: tuple[str, ...] | None = None,
+    ) -> list[WebSearchResult]:
+        if not self.enabled or not query.strip():
+            return []
+
+        api = f"{self.base_url}/search?q={quote_plus(query)}&format=json"
+        try:
+            import httpx
+
+            with httpx.Client(follow_redirects=True, timeout=20.0) as client:
+                resp = client.get(api)
+                if resp.status_code != 200:
+                    logger.warning("SearXNG HTTP %s for %s", resp.status_code, api)
+                    return []
+                data = resp.json()
+        except Exception as e:
+            logger.warning("SearXNG search failed: %s", e)
+            return []
+
+        results: list[WebSearchResult] = []
+        seen: set[str] = set()
+        for item in data.get("results", []):
+            url = (item.get("url") or "").strip()
+            if not url.startswith("http"):
+                continue
+            norm = _normalize_url(url)
+            if norm in seen:
+                continue
+            if not _domain_allowed(url, include_domains, exclude_domains):
+                continue
+            seen.add(norm)
+            results.append(
+                WebSearchResult(
+                    url=url,
+                    title=(item.get("title") or url)[:300],
+                    snippet=(item.get("content") or item.get("snippet") or "")[:2000],
+                    backend=self.name,
+                )
+            )
+            if len(results) >= max_results:
+                break
+        logger.info("SearXNGBackend: %d results for %r", len(results), query[:60])
+        return results
+
+
+class FirecrawlBackend(SearchBackend):
+    """Web search via self-hosted Firecrawl (search + optional markdown in results)."""
+
+    name = "firecrawl"
+
+    def __init__(self) -> None:
+        self.base_url = (getattr(cfg, "FIRECRAWL_BASE_URL", "") or "").rstrip("/")
+        self.api_key = getattr(cfg, "FIRECRAWL_API_KEY", "") or ""
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.base_url)
+
+    def _search_sdk(self, query: str, max_results: int) -> list[dict[str, Any]]:
+        from firecrawl import Firecrawl
+
+        init_kwargs: dict[str, Any] = {}
+        if self.api_key:
+            init_kwargs["api_key"] = self.api_key
+
+        app = None
+        for param in ("api_url", "base_url"):
+            try:
+                app = Firecrawl(**{param: self.base_url, **init_kwargs})
+                break
+            except TypeError:
+                continue
+        if app is None:
+            app = Firecrawl(**init_kwargs)
+
+        resp = app.search(query, limit=max_results)
+        if isinstance(resp, dict):
+            data = resp.get("data") or resp.get("results") or []
+        elif isinstance(resp, list):
+            data = resp
+        else:
+            data = getattr(resp, "data", None) or []
+        return list(data or [])
+
+    def _search_rest(self, query: str, max_results: int) -> list[dict[str, Any]]:
+        import httpx
+
+        url = f"{self.base_url}/v1/search"
+        headers = {"Content-Type": "application/json"}
+        if self.api_key:
+            headers["Authorization"] = f"Bearer {self.api_key}"
+        payload = {"query": query, "limit": max_results}
+        with httpx.Client(follow_redirects=True, timeout=60.0) as client:
+            resp = client.post(url, json=payload, headers=headers)
+            if resp.status_code != 200:
+                logger.warning("Firecrawl search HTTP %s: %s", resp.status_code, resp.text[:200])
+                return []
+            body = resp.json()
+        if isinstance(body, dict):
+            return list(body.get("data") or body.get("results") or [])
+        return []
+
+    def search(
+        self,
+        query: str,
+        *,
+        max_results: int = 5,
+        include_domains: tuple[str, ...] | None = None,
+        exclude_domains: tuple[str, ...] | None = None,
+    ) -> list[WebSearchResult]:
+        if not self.enabled or not query.strip():
+            return []
+
+        raw_items: list[dict[str, Any]] = []
+        try:
+            raw_items = self._search_sdk(query, max_results=max_results)
+        except ImportError:
+            logger.debug("firecrawl-py not installed — using REST search")
+        except Exception as e:
+            logger.debug("Firecrawl SDK search failed: %s — trying REST", e)
+
+        if not raw_items:
+            try:
+                raw_items = self._search_rest(query, max_results=max_results)
+            except Exception as e:
+                logger.warning("Firecrawl REST search failed: %s", e)
+                return []
+
+        results: list[WebSearchResult] = []
+        seen: set[str] = set()
+        for item in raw_items:
+            url = (item.get("url") or item.get("link") or "").strip()
+            if not url.startswith("http"):
+                continue
+            norm = _normalize_url(url)
+            if norm in seen:
+                continue
+            if not _domain_allowed(url, include_domains, exclude_domains):
+                continue
+            seen.add(norm)
+            snippet = (
+                item.get("markdown")
+                or item.get("description")
+                or item.get("snippet")
+                or item.get("content")
+                or ""
+            )
+            results.append(
+                WebSearchResult(
+                    url=url,
+                    title=(item.get("title") or url)[:300],
+                    snippet=str(snippet)[:8000],
+                    backend=self.name,
+                    metadata={"has_markdown": bool(item.get("markdown"))},
+                )
+            )
+            if len(results) >= max_results:
+                break
+
+        logger.info("FirecrawlBackend: %d results for %r", len(results), query[:60])
+        return results
+
+
+def get_configured_backends() -> list[SearchBackend]:
+    """Return enabled search backends from WEB_SEARCH_BACKENDS config."""
+    raw = getattr(cfg, "WEB_SEARCH_BACKENDS", ("ddgs",))
+    names = {n.strip().lower() for n in raw if n.strip()}
+    backends: list[SearchBackend] = []
+    if "firecrawl" in names:
+        fc = FirecrawlBackend()
+        if fc.enabled:
+            backends.append(fc)
+        else:
+            logger.debug("Firecrawl configured but FIRECRAWL_BASE_URL is empty — skipping")
+    if "ddgs" in names or "duckduckgo" in names:
+        backends.append(DuckDuckGoBackend())
+    if "searxng" in names:
+        sx = SearXNGBackend()
+        if sx.enabled:
+            backends.append(sx)
+        else:
+            logger.debug("SearXNG configured but SEARXNG_BASE_URL is empty — skipping")
+    return backends

package/knowledge_base/connectors/serviceNow_connector.py

@@ -0,0 +1,25 @@
+"""ServiceNow connector — Phase 8 stub."""
+from __future__ import annotations
+
+from datetime import datetime
+
+from knowledge_base.config import cfg
+from knowledge_base.connectors.base import BaseConnector
+
+
+class ServiceNowConnector(BaseConnector):
+    name = "servicenow"
+    enabled = cfg.SERVICENOW_ENABLED
+    supports_sync = False
+
+    def connect(self) -> bool:
+        return self.enabled
+
+    def sync(self, since: datetime | None = None) -> list[dict]:
+        return []
+
+    def ingest_to_kg(self, items: list[dict]) -> int:
+        return 0
+
+    def ingest_to_rag(self, items: list[dict]) -> int:
+        return 0

package/knowledge_base/connectors/sync_scheduler.py

@@ -0,0 +1,111 @@
+"""Connector sync scheduler with incremental state and daemon mode."""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import sys
+import time
+from datetime import datetime, timezone
+
+from knowledge_base.connectors.bitbucket_connector import BitbucketConnector
+from knowledge_base.connectors.community_connector import CommunityConnector
+from knowledge_base.connectors.confluence_connector import ConfluenceConnector
+from knowledge_base.connectors.jira_connector import JiraConnector
+from knowledge_base.connectors.kb_igel_connector import KbIgelConnector
+from knowledge_base.connectors.serviceNow_connector import ServiceNowConnector
+from knowledge_base.connectors.teamcity_connector import TeamCityConnector
+from knowledge_base.connectors.testRail_connector import TestRailConnector
+from knowledge_base.db.client import init_pool
+from knowledge_base.db.connector_sync import get_last_sync, update_sync_state
+
+logger = logging.getLogger(__name__)
+
+CONNECTORS = {
+    "jira": JiraConnector,
+    "confluence": ConfluenceConnector,
+    "bitbucket": BitbucketConnector,
+    "teamcity": TeamCityConnector,
+    "testrail": TestRailConnector,
+    "servicenow": ServiceNowConnector,
+    "kb_igel": KbIgelConnector,
+    "community": CommunityConnector,
+}
+
+
+def sync_source(name: str, *, dry_run: bool = False) -> dict:
+    cls = CONNECTORS.get(name)
+    if not cls:
+        raise ValueError(f"Unknown source: {name}")
+    connector = cls()
+    if not connector.enabled:
+        return {"source": name, "status": "disabled", "items": 0}
+    if not getattr(connector, "supports_sync", True):
+        return {"source": name, "status": "not_implemented", "items": 0}
+
+    if not connector.connect():
+        update_sync_state(name, status="connection_failed", advance_timestamp=False)
+        return {"source": name, "status": "connection_failed", "items": 0}
+
+    since = get_last_sync(name)
+    items = connector.sync(since=since)
+    if dry_run:
+        return {"source": name, "status": "dry_run", "items": len(items), "since": str(since)}
+
+    kg_count = connector.ingest_to_kg(items)
+    rag_count = connector.ingest_to_rag(items)
+    update_sync_state(name, status="synced")
+    return {"source": name, "status": "synced", "items": len(items), "kg": kg_count, "rag": rag_count}
+
+
+def sync_all(*, dry_run: bool = False) -> list[dict]:
+    results = []
+    for name in CONNECTORS:
+        try:
+            results.append(sync_source(name, dry_run=dry_run))
+        except Exception as e:
+            update_sync_state(name, status="error", advance_timestamp=False)
+            results.append({"source": name, "status": "error", "error": str(e)})
+    return results
+
+
+def main() -> int:
+    logging.basicConfig(level=logging.INFO)
+    parser = argparse.ArgumentParser(description="IGEL KB connector sync scheduler")
+    parser.add_argument("--source", default="all", help="Connector name or 'all'")
+    parser.add_argument("--dry-run", action="store_true")
+    parser.add_argument("--daemon", action="store_true", help="Run sync loop")
+    parser.add_argument("--interval", type=int, default=3600, help="Daemon interval seconds")
+    args = parser.parse_args()
+
+    if args.interval <= 0:
+        parser.error("--interval must be a positive integer")
+
+    init_pool()
+
+    def _run_once():
+        if args.source == "all":
+            return sync_all(dry_run=args.dry_run)
+        return [sync_source(args.source, dry_run=args.dry_run)]
+
+    if args.daemon:
+        while True:
+            try:
+                rows = _run_once()
+            except Exception as e:
+                logger.exception("sync daemon cycle failed for source=%s: %s", args.source, e)
+                if args.source != "all" and args.source in CONNECTORS:
+                    update_sync_state(args.source, status="error", advance_timestamp=False)
+                rows = [{"source": args.source, "status": "error", "error": str(e)}]
+            for r in rows:
+                print(r)
+            time.sleep(args.interval)
+        return 0
+
+    for r in _run_once():
+        print(r)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/knowledge_base/connectors/teamcity_connector.py

@@ -0,0 +1,25 @@
+"""TeamCity connector — Phase 8 stub."""
+from __future__ import annotations
+
+from datetime import datetime
+
+from knowledge_base.config import cfg
+from knowledge_base.connectors.base import BaseConnector
+
+
+class TeamCityConnector(BaseConnector):
+    name = "teamcity"
+    enabled = cfg.TEAMCITY_ENABLED
+    supports_sync = False
+
+    def connect(self) -> bool:
+        return self.enabled
+
+    def sync(self, since: datetime | None = None) -> list[dict]:
+        return []
+
+    def ingest_to_kg(self, items: list[dict]) -> int:
+        return 0
+
+    def ingest_to_rag(self, items: list[dict]) -> int:
+        return 0