igel-qe-core 1.0.0

package/knowledge_base/connectors/confluence_connector.py

@@ -0,0 +1,165 @@
+"""Confluence connector — REST API sync + KG ingest."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from urllib.parse import unquote_plus
+
+import httpx
+
+from knowledge_base.config import cfg
+from knowledge_base.connectors.base import BaseConnector
+
+logger = logging.getLogger(__name__)
+
+
+class ConfluenceConnector(BaseConnector):
+    name = "confluence"
+    enabled = cfg.CONFLUENCE_ENABLED
+
+    def connect(self) -> bool:
+        if not self.enabled:
+            return False
+        if not cfg.CONFLUENCE_URL:
+            logger.warning("CONFLUENCE_URL not set")
+            return False
+        auth = self._auth()
+        if auth and auth[0] and auth[1]:
+            return True
+        logger.warning("Confluence credentials missing (set CONFLUENCE_* or JIRA_* auth vars)")
+        return False
+
+    def _auth(self):
+        """Return httpx auth tuple, preferring dedicated Confluence credentials."""
+        if getattr(cfg, "CONFLUENCE_API_TOKEN", ""):
+            return (cfg.CONFLUENCE_USERNAME, cfg.CONFLUENCE_API_TOKEN)
+        if cfg.JIRA_API_TOKEN:
+            return (cfg.JIRA_USERNAME, cfg.JIRA_API_TOKEN)
+        return None
+
+    def search(self, query: str, limit: int = 5, timeout: float | None = None) -> list[str]:
+        if not self.connect():
+            return []
+        client_timeout = min(15.0, timeout) if timeout and timeout > 0 else 15.0
+        try:
+            with httpx.Client(timeout=client_timeout) as client:
+                resp = client.get(
+                    f"{cfg.CONFLUENCE_URL.rstrip('/')}/rest/api/content/search",
+                    params={"cql": f'text ~ "{query.replace(chr(34), "")}"', "limit": limit},
+                    auth=self._auth(),
+                )
+                resp.raise_for_status()
+                results = resp.json().get("results") or []
+                base = cfg.CONFLUENCE_URL.rstrip("/")
+                return [f"{base}{r.get('_links', {}).get('webui', '')}" for r in results]
+        except Exception as exc:
+            logger.warning("Confluence search failed: %s", exc)
+            return []
+
+    def fetch_page(self, url: str, timeout: float | None = None) -> dict | None:
+        """Fetch a single Confluence page by its webui URL and return a content dict."""
+        if not self.connect():
+            return None
+        import re
+        base = cfg.CONFLUENCE_URL.rstrip("/")
+        auth = self._auth()
+        client_timeout = min(15.0, timeout) if timeout and timeout > 0 else 15.0
+        page_id: str | None = None
+        # Cloud: /wiki/spaces/.../pages/123456/...
+        m = re.search(r"/pages/(\d+)", url)
+        if m:
+            page_id = m.group(1)
+        # Server/DC: ?pageId=123456
+        if not page_id:
+            m = re.search(r"pageId=(\d+)", url)
+            if m:
+                page_id = m.group(1)
+        try:
+            with httpx.Client(timeout=client_timeout) as client:
+                if not page_id:
+                    # Fall back: search by title derived from URL tail
+                    title = unquote_plus(url.rstrip("/").rsplit("/", 1)[-1])
+                    r = client.get(
+                        f"{base}/rest/api/content/search",
+                        params={"cql": f'title = "{title.replace(chr(34), "")}"', "limit": 1, "expand": "body.storage"},
+                        auth=auth,
+                    )
+                    r.raise_for_status()
+                    results = r.json().get("results") or []
+                    if not results:
+                        return None
+                    page_id = results[0]["id"]
+                resp = client.get(
+                    f"{base}/rest/api/content/{page_id}",
+                    params={"expand": "body.storage"},
+                    auth=auth,
+                )
+                resp.raise_for_status()
+                page = resp.json()
+                body = (page.get("body") or {}).get("storage") or {}
+                return {
+                    "title": page.get("title") or url,
+                    "url": url,
+                    "raw_text": (body.get("value") or "")[:8000],
+                    "source_type": "confluence",
+                }
+        except Exception as exc:
+            logger.warning("Confluence fetch_page failed for %s: %s", url, exc)
+            return None
+
+    def sync(self, since: datetime | None = None) -> list[dict]:
+        if not self.connect():
+            return []
+        cql = "type=page ORDER BY lastModified DESC"
+        if since:
+            cql = f"type=page AND lastModified >= '{since.strftime('%Y-%m-%d')}' ORDER BY lastModified DESC"
+        items: list[dict] = []
+        try:
+            with httpx.Client(timeout=30.0) as client:
+                resp = client.get(
+                    f"{cfg.CONFLUENCE_URL.rstrip('/')}/rest/api/content/search",
+                    params={"cql": cql, "limit": 30, "expand": "body.storage"},
+                    auth=self._auth(),
+                )
+                resp.raise_for_status()
+                base = cfg.CONFLUENCE_URL.rstrip("/")
+                for page in resp.json().get("results") or []:
+                    body = (page.get("body") or {}).get("storage") or {}
+                    items.append({
+                        "title": page.get("title"),
+                        "url": f"{base}{page.get('_links', {}).get('webui', '')}",
+                        "raw_text": (body.get("value") or "")[:8000],
+                        "source_type": "confluence",
+                    })
+        except Exception as exc:
+            logger.warning("Confluence sync failed: %s", exc)
+            raise
+        return items
+
+    def ingest_to_kg(self, items: list[dict]) -> int:
+        if not items:
+            return 0
+        from knowledge_base.db.client import get_conn
+        import json
+        count = 0
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                for item in items:
+                    title = item.get("title") or item.get("url")
+                    if not title:
+                        continue
+                    cur.execute(
+                        """
+                        INSERT INTO knowledge_graph_nodes (node_type, value, normalized, metadata)
+                        VALUES ('feature', %s, %s, %s::jsonb)
+                        ON CONFLICT (node_type, normalized) DO UPDATE SET metadata = EXCLUDED.metadata
+                        """,
+                        (title[:4000], title.lower()[:2000], json.dumps(item)),
+                    )
+                    count += 1
+        return count
+
+    def ingest_to_rag(self, items: list[dict]) -> int:
+        from knowledge_base.ingestion.web_ingest import ingest_web_items
+        return ingest_web_items(items, source_label="confluence")

package/knowledge_base/connectors/jira_connector.py

@@ -0,0 +1,174 @@
+"""JIRA connector — REST API sync + KG ingest."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+
+import httpx
+
+from knowledge_base.config import cfg
+from knowledge_base.connectors.base import BaseConnector
+
+logger = logging.getLogger(__name__)
+
+
+def _jira_text(value: object) -> str:
+    """Normalize Jira field payloads (plain text or ADF JSON) into readable text."""
+    if value is None:
+        return ""
+    if isinstance(value, str):
+        return value
+    if isinstance(value, (int, float, bool)):
+        return str(value)
+    if isinstance(value, list):
+        parts = [_jira_text(v) for v in value]
+        return "\n".join(p for p in parts if p)
+    if isinstance(value, dict):
+        txt = value.get("text")
+        if isinstance(txt, str) and txt.strip():
+            return txt
+        content = value.get("content")
+        if isinstance(content, list):
+            parts = [_jira_text(v) for v in content]
+            return "\n".join(p for p in parts if p)
+        val = value.get("value")
+        if isinstance(val, str):
+            return val
+    return ""
+
+
+class JiraConnector(BaseConnector):
+    name = "jira"
+    enabled = cfg.JIRA_ENABLED
+
+    def connect(self) -> bool:
+        if not self.enabled:
+            return False
+        if not cfg.JIRA_URL or not cfg.JIRA_USERNAME or not cfg.JIRA_API_TOKEN:
+            logger.warning("JIRA credentials missing")
+            return False
+        return True
+
+    def search(self, query: str, limit: int = 5, timeout: float | None = None) -> list[str]:
+        if not self.connect():
+            return []
+        jql = f'text ~ "{query.replace(chr(34), "")}" ORDER BY updated DESC'
+        client_timeout = min(15.0, timeout) if timeout and timeout > 0 else 15.0
+        try:
+            with httpx.Client(timeout=client_timeout) as client:
+                resp = client.get(
+                    f"{cfg.JIRA_URL.rstrip('/')}/rest/api/2/search",
+                    params={"jql": jql, "maxResults": limit, "fields": "summary"},
+                    auth=(cfg.JIRA_USERNAME, cfg.JIRA_API_TOKEN),
+                )
+                resp.raise_for_status()
+                issues = resp.json().get("issues") or []
+                return [
+                    f"{cfg.JIRA_URL.rstrip('/')}/browse/{i['key']}"
+                    for i in issues if i.get("key")
+                ]
+        except Exception as exc:
+            logger.warning("JIRA search failed: %s", exc)
+            return []
+
+    def sync(self, since: datetime | None = None) -> list[dict]:
+        if not self.connect():
+            return []
+        jql = "ORDER BY updated DESC"
+        if since:
+            jql = f'updated >= "{since.strftime("%Y-%m-%d %H:%M")}" ORDER BY updated DESC'
+        items: list[dict] = []
+        try:
+            with httpx.Client(timeout=30.0) as client:
+                resp = client.get(
+                    f"{cfg.JIRA_URL.rstrip('/')}/rest/api/2/search",
+                    params={"jql": jql, "maxResults": 50, "fields": "summary,description,status,components,fixVersions,issuetype"},
+                    auth=(cfg.JIRA_USERNAME, cfg.JIRA_API_TOKEN),
+                )
+                resp.raise_for_status()
+                for issue in resp.json().get("issues") or []:
+                    fields = issue.get("fields") or {}
+                    items.append({
+                        "key": issue.get("key"),
+                        "title": fields.get("summary") or issue.get("key"),
+                        "url": f"{cfg.JIRA_URL.rstrip('/')}/browse/{issue.get('key')}",
+                        "content": _jira_text(fields.get("description"))[:8000],
+                        "status": (fields.get("status") or {}).get("name"),
+                        "issue_type": (fields.get("issuetype") or {}).get("name"),
+                        "source_type": "jira",
+                    })
+        except Exception as exc:
+            logger.warning("JIRA sync failed: %s", exc)
+            raise
+        return items
+
+    def ingest_to_kg(self, items: list[dict]) -> int:
+        if not items:
+            return 0
+        from knowledge_base.db.client import get_conn
+        count = 0
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                for item in items:
+                    key = item.get("key") or item.get("title")
+                    if not key:
+                        continue
+                    cur.execute(
+                        """
+                        INSERT INTO knowledge_graph_nodes (node_type, value, normalized, metadata)
+                        VALUES ('defect', %s, %s, %s::jsonb)
+                        ON CONFLICT (node_type, normalized) DO UPDATE SET metadata = EXCLUDED.metadata
+                        """,
+                        (str(key), str(key).lower(), __import__("json").dumps(item)),
+                    )
+                    count += 1
+        return count
+
+    def fetch_page(self, url: str, timeout: float | None = None) -> dict | None:
+        """Fetch a single Jira issue by its browse URL and return a content dict."""
+        if not self.connect():
+            return None
+        # Extract issue key from URL tail (e.g. .../browse/PROJ-123)
+        key = url.rstrip("/").rsplit("/", 1)[-1]
+        if not key:
+            return None
+        client_timeout = min(15.0, timeout) if timeout and timeout > 0 else 15.0
+        try:
+            with httpx.Client(timeout=client_timeout) as client:
+                resp = client.get(
+                    f"{cfg.JIRA_URL.rstrip('/')}/rest/api/2/issue/{key}",
+                    params={"fields": "summary,description,status,issuetype,comment"},
+                    auth=(cfg.JIRA_USERNAME, cfg.JIRA_API_TOKEN),
+                )
+                resp.raise_for_status()
+                data = resp.json()
+                fields = data.get("fields") or {}
+                description = _jira_text(fields.get("description"))
+                comments = []
+                for c in ((fields.get("comment") or {}).get("comments") or [])[:3]:
+                    body = _jira_text(c.get("body"))
+                    if body.strip():
+                        comments.append(body)
+                comments_text = "\n".join(comments)
+                return {
+                    "title": fields.get("summary") or key,
+                    "url": url,
+                    "raw_text": f"{description}\n\n{comments_text}".strip()[:8000],
+                    "source_type": "jira",
+                }
+        except Exception as exc:
+            logger.warning("JIRA fetch_page failed for %s: %s", url, exc)
+            return None
+
+    def ingest_to_rag(self, items: list[dict]) -> int:
+        from knowledge_base.ingestion.web_ingest import ingest_web_items
+        web_items = []
+        for item in items:
+            web_items.append({
+                "url": item.get("url", ""),
+                "title": item.get("title", ""),
+                "raw_text": item.get("content", ""),
+                "source_type": "jira",
+            })
+        return ingest_web_items(web_items, source_label="jira")

package/knowledge_base/connectors/kb_igel_connector.py

@@ -0,0 +1,145 @@
+"""IGEL Knowledge Base (https://kb.igel.com) connector — crawl + RAG ingest."""
+
+from __future__ import annotations
+
+import logging
+from datetime import datetime
+from typing import Any
+from urllib.parse import urlparse
+
+from knowledge_base.config import cfg
+from knowledge_base.connectors.base import BaseConnector
+from knowledge_base.connectors.web_fetcher import fetch_url, search_confluence_site
+from knowledge_base.parsers.html_parser import parse_html_page
+
+logger = logging.getLogger(__name__)
+
+# Map common IGEL abbreviations / topics → kb.igel.com section paths
+_KB_TOPIC_PATHS: dict[str, str] = {
+    "ums": "/en/universal-management-suite/current",
+    "universal management suite": "/en/universal-management-suite/current",
+    "sso": "/en/how-to-configure-single-sign-on-sso-on-igel-os-12/current",
+    "single sign-on": "/en/how-to-configure-single-sign-on-sso-on-igel-os-12/current",
+    "entra": "/en/how-to-configure-single-sign-on-sso-on-igel-os-12/current",
+    "azure ad": "/en/how-to-configure-single-sign-on-sso-on-igel-os-12/current",
+    "okta": "/en/how-to-configure-single-sign-on-sso-on-igel-os-12/current",
+    "igel os": "/en/igel-os-base-system/current",
+    "icg": "/en/igel-cloud-gateway/current",
+    "cloud gateway": "/en/igel-cloud-gateway/current",
+    "pxe": "/en/how-to-deploy-igel-os-12-with-pxe/current",
+    "active directory": "/en/active-directory-kerberos-configuration-in-igel-os-12/current",
+    "kerberos": "/en/active-directory-kerberos-configuration-in-igel-os-12/current",
+    "profile": "/en/universal-management-suite/current",
+    "factory reset": "/en/igel-os-base-system/current",
+}
+
+
+class KbIgelConnector(BaseConnector):
+    """Fetch and ingest pages from the official IGEL Knowledge Base."""
+
+    name = "kb_igel"
+
+    def __init__(self) -> None:
+        self.base_url = getattr(cfg, "KB_IGEL_BASE_URL", "https://kb.igel.com").rstrip("/")
+        self.enabled = getattr(cfg, "KB_IGEL_ENABLED", True)
+
+    def connect(self) -> bool:
+        result = fetch_url(self.base_url, cache_ttl=300)
+        return result.status_code == 200 and len(result.html) > 100
+
+    def search(self, query: str, *, limit: int | None = None, timeout: float | None = None) -> list[str]:
+        max_pages = limit or getattr(cfg, "WEB_MAX_PAGES", 3)
+        q_lower = query.lower()
+        urls: list[str] = []
+
+        for topic, path in _KB_TOPIC_PATHS.items():
+            if topic in q_lower:
+                full = f"{self.base_url}{path}"
+                if full not in urls:
+                    urls.append(full)
+            if len(urls) >= max_pages:
+                return urls[:max_pages]
+
+        rest = search_confluence_site(self.base_url, query, limit=max_pages)
+        for u in rest:
+            if u not in urls and "/search?" not in u:
+                urls.append(u)
+            if len(urls) >= max_pages:
+                break
+        return urls[:max_pages]
+
+    def fetch_page(self, url: str, timeout: float | None = None) -> dict[str, Any] | None:
+        if not url.startswith("http"):
+            url = f"{self.base_url}/{url.lstrip('/')}"
+        host = urlparse(url).netloc
+        host_l = host.lower()
+        if host_l != "igel.com" and not host_l.endswith(".igel.com"):
+            logger.warning("kb_igel: refusing non-IGEL URL %s", url)
+            return None
+
+        fetch_timeout = min(20.0, timeout) if timeout and timeout > 0 else 20.0
+        result = fetch_url(
+            url,
+            cache_ttl=getattr(cfg, "WEB_CACHE_TTL_SECONDS", 3600),
+            timeout=fetch_timeout,
+        )
+        if result.status_code != 200 or not result.html:
+            return None
+
+        parsed = parse_html_page(result.html, url, source_type="kb_web")
+        return {
+            "url": url,
+            "title": parsed.title,
+            "sections": parsed.sections,
+            "raw_text": parsed.raw_text,
+            "source_type": "kb_web",
+            "product": "IGEL",
+        }
+
+    def sync(self, since: datetime | None = None) -> list[dict]:
+        """Sync seed pages from IGEL KB navigation (batch ingest)."""
+        seed_paths = [
+            "/en/igel-os-base-system",
+            "/en/universal-management-suite",
+            "/en/igel-cloud-gateway",
+            "/en/security-safety",
+            "/en/how-to-start-with-igel",
+            "/en/igel-apps",
+        ]
+        items: list[dict] = []
+        for path in seed_paths:
+            page = self.fetch_page(f"{self.base_url}{path}")
+            if page:
+                items.append(page)
+        logger.info("kb_igel sync: fetched %d seed pages", len(items))
+        return items
+
+    def ingest_to_kg(self, items: list[dict]) -> int:
+        if not items:
+            return 0
+        from knowledge_base.db.client import get_conn
+        import json
+        count = 0
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                for item in items:
+                    title = item.get("title") or item.get("url", "")
+                    product = item.get("product") or "IGEL"
+                    for entity in (title, product):
+                        if not entity:
+                            continue
+                        cur.execute(
+                            """
+                            INSERT INTO knowledge_graph_nodes (node_type, value, normalized, metadata)
+                            VALUES ('feature', %s, %s, %s::jsonb)
+                            ON CONFLICT (node_type, normalized) DO NOTHING
+                            """,
+                            (entity[:4000], entity.lower()[:2000], json.dumps({"source": "kb_igel"})),
+                        )
+                        count += 1
+        return count
+
+    def ingest_to_rag(self, items: list[dict]) -> int:
+        from knowledge_base.ingestion.web_ingest import ingest_web_items
+
+        return ingest_web_items(items, source_label="kb.igel.com")

package/knowledge_base/connectors/playwright_fetcher.py

@@ -0,0 +1,50 @@
+"""Playwright-based page fetcher — Crawl4AI-compatible fallback for Python 3.8."""
+
+from __future__ import annotations
+
+import logging
+import re
+from html import unescape
+
+logger = logging.getLogger(__name__)
+
+
+def _html_to_text(html: str) -> str:
+    text = re.sub(r"(?is)<(script|style|nav|footer|header|aside|ins)[^>]*>.*?</\1>", " ", html)
+    text = re.sub(r"(?is)<br\s*/?>", "\n", text)
+    text = re.sub(r"(?is)</?(p|div|li|h[1-6]|tr|td|th|article|section)[^>]*>", "\n", text)
+    text = re.sub(r"<[^>]+>", " ", text)
+    text = unescape(text)
+    text = re.sub(r"[ \t]+\n", "\n", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
+
+
+def fetch_page_playwright(url: str, *, timeout_ms: int = 30000) -> tuple[str, str]:
+    """
+    Fetch URL with headless Chromium via Playwright.
+    Returns (title, markdown-ish plain text).
+    """
+    from playwright.sync_api import sync_playwright
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        try:
+            page = browser.new_page()
+            page.goto(url, wait_until="domcontentloaded", timeout=timeout_ms)
+            page.wait_for_timeout(500)
+            title = page.title() or url
+            html = page.content()
+        finally:
+            browser.close()
+
+    return title[:300], _html_to_text(html)
+
+
+def playwright_available() -> bool:
+    try:
+        import playwright  # noqa: F401
+
+        return True
+    except ImportError:
+        return False