igel-qe-core 1.0.0

package/knowledge_base/graph_db/graphiti_client.py

@@ -0,0 +1,127 @@
+"""Graphiti temporal graph client — Phase 6."""
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from knowledge_base.config import cfg
+from knowledge_base.db.client import get_conn
+
+logger = logging.getLogger(__name__)
+
+_client: "GraphitiClient | None" = None
+
+_TEMPORAL_TABLE_SQL = """
+CREATE TABLE IF NOT EXISTS temporal_feature_changes (
+    id              UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    feature_name    TEXT NOT NULL,
+    version         TEXT NOT NULL,
+    change_type     TEXT NOT NULL DEFAULT 'enhanced',
+    description     TEXT,
+    affected_keys   JSONB DEFAULT '[]'::jsonb,
+    affected_paths  JSONB DEFAULT '[]'::jsonb,
+    created_at      TIMESTAMP DEFAULT NOW(),
+    UNIQUE(feature_name, version, change_type)
+);
+CREATE INDEX IF NOT EXISTS idx_temporal_feature ON temporal_feature_changes(feature_name);
+CREATE INDEX IF NOT EXISTS idx_temporal_version ON temporal_feature_changes(version);
+"""
+
+
+class GraphitiClient:
+    """Temporal graph client — Graphiti OSS when available, else PostgreSQL store."""
+
+    def __init__(self) -> None:
+        self._graphiti = None
+        self._use_graphiti = False
+        self._ensure_table()
+        if cfg.GRAPHITI_ENABLED:
+            try:
+                import graphiti_core  # noqa: F401
+                self._use_graphiti = True
+                logger.info("Graphiti: graphiti-core detected")
+            except ImportError:
+                logger.info("Graphiti: using PostgreSQL temporal fallback")
+
+    def _ensure_table(self) -> None:
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute(_TEMPORAL_TABLE_SQL)
+
+    def health_check(self) -> dict[str, Any]:
+        return {
+            "status": "healthy",
+            "backend": "graphiti" if self._use_graphiti else "postgresql_fallback",
+        }
+
+    def record_change(
+        self,
+        feature_name: str,
+        version: str,
+        change_type: str,
+        description: str,
+        affected_keys: list[str] | None = None,
+        affected_paths: list[str] | None = None,
+    ) -> None:
+        sql = """
+        INSERT INTO temporal_feature_changes
+            (feature_name, version, change_type, description, affected_keys, affected_paths)
+        VALUES (%s, %s, %s, %s, %s::jsonb, %s::jsonb)
+        ON CONFLICT (feature_name, version, change_type) DO UPDATE SET
+            description = EXCLUDED.description,
+            affected_keys = EXCLUDED.affected_keys,
+            affected_paths = EXCLUDED.affected_paths
+        """
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute(sql, (
+                    feature_name, version, change_type, description,
+                    json.dumps(affected_keys or []),
+                    json.dumps(affected_paths or []),
+                ))
+
+    def query_feature_changes(self, feature_name: str, version: str) -> dict[str, Any]:
+        sql = """
+        SELECT version, change_type, description, affected_keys, affected_paths
+        FROM temporal_feature_changes
+        WHERE feature_name ILIKE %s
+          AND (version = %s OR %s = 'unspecified')
+        ORDER BY version
+        """
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute(sql, (f"%{feature_name}%", version, version))
+                rows = cur.fetchall()
+
+        if not rows:
+            return {"status": "no_changes_found", "version_queried": version, "feature_changes": []}
+
+        changes = []
+        for row in rows:
+            changes.append({
+                "version": row[0],
+                "change_type": row[1],
+                "description": row[2],
+                "affected_registry_keys": row[3] if isinstance(row[3], list) else json.loads(row[3] or "[]"),
+                "affected_ui_paths": row[4] if isinstance(row[4], list) else json.loads(row[4] or "[]"),
+            })
+
+        return {
+            "status": "complete",
+            "version_queried": version,
+            "feature_changes": changes,
+            "deprecated_in_version": [],
+            "added_in_version": [],
+            "renamed_in_version": [],
+        }
+
+    def query_latest(self, feature_name: str) -> dict[str, Any]:
+        return self.query_feature_changes(feature_name, "12.10")
+
+
+def get_graphiti_client() -> GraphitiClient:
+    global _client
+    if _client is None:
+        _client = GraphitiClient()
+    return _client

package/knowledge_base/graph_db/kg_traversal.py

@@ -0,0 +1,192 @@
+"""Neo4j multi-hop KG traversal — Phase 3."""
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+
+from knowledge_base.config import cfg
+from knowledge_base.graph_db.neo4j_client import get_neo4j_client
+from knowledge_base.retrieval.knowledge_graph import KGContext
+
+logger = logging.getLogger(__name__)
+
+
+def retrieve_kg_context_neo4j(
+    query: str,
+    max_hops: int = 2,
+    max_nodes: int = 48,
+    feature_area: str | None = None,
+    seed_parent_chunk_ids: list[str] | None = None,
+) -> KGContext:
+    """Traverse Neo4j graph for registry keys, UI paths, and related entities."""
+    client = get_neo4j_client()
+    if not client.available:
+        logger.warning("Neo4j unavailable — returning empty KG context")
+        return KGContext(text_block="", node_ids=[], node_count=0, edge_count=0)
+
+    terms = _search_terms(query)
+    feature = feature_area or _infer_feature(query)
+    hops = max(0, min(int(max_hops), 3))
+
+    cypher = """
+    MATCH (f:Feature)
+    WHERE toLower(f.name) CONTAINS toLower($feature)
+       OR ANY(t IN $terms WHERE toLower(f.name) CONTAINS toLower(t))
+    OPTIONAL MATCH (f)-[:USES]->(rk:RegistryKey)
+    OPTIONAL MATCH (f)-[:CONFIGURED_VIA]->(up:UIPath)
+    RETURN f.name AS feature, collect(DISTINCT rk.key) AS keys,
+           collect(DISTINCT up.path) AS paths
+    LIMIT 1
+    """
+    rows = _run_query(client, cypher, {"feature": feature, "terms": terms[:8]})
+
+    keys: list[str] = []
+    paths: list[str] = []
+    features: list[str] = []
+
+    if rows:
+        row = rows[0]
+        features.append(row.get("feature") or feature)
+        keys = [k for k in (row.get("keys") or []) if k]
+        paths = [p for p in (row.get("paths") or []) if p]
+
+    if hops > 0 and features:
+        ext_cypher = """
+        MATCH (f:Feature {name: $feature})-[*1..$hops]-(related)
+        WHERE related:RegistryKey OR related:UIPath OR related:Feature
+        RETURN labels(related)[0] AS lbl,
+               coalesce(related.key, related.path, related.name) AS val
+        LIMIT $limit
+        """
+        ext_rows = _run_query(
+            client,
+            ext_cypher.replace("$hops", str(hops)),
+            {"feature": features[0], "limit": max_nodes},
+        )
+        for er in ext_rows:
+            lbl = er.get("lbl", "")
+            val = er.get("val")
+            if not val:
+                continue
+            if lbl == "RegistryKey" and val not in keys:
+                keys.append(val)
+            elif lbl == "UIPath" and val not in paths:
+                paths.append(val)
+
+    text_block = _format_block(keys, paths, features)
+    node_count = len(keys) + len(paths) + len(features)
+
+    return KGContext(
+        text_block=text_block,
+        node_ids=[f"neo4j:{v}" for v in (keys + paths + features)[:max_nodes]],
+        node_count=node_count,
+        edge_count=0,
+    )
+
+
+def traverse_failure_path(test_id: str) -> dict[str, Any]:
+    """RCA traversal: Testcase → Feature → RegistryKey → Defect."""
+    client = get_neo4j_client()
+    if not client.available:
+        return {"status": "insufficient_graph_data", "chain": [], "missing": ["neo4j"]}
+
+    cypher = """
+    MATCH path = (tc:Testcase)-[*1..3]-(n)
+    WHERE tc.id CONTAINS $test_id OR tc.name CONTAINS $test_id
+    RETURN [node IN nodes(path) | coalesce(node.name, node.key, node.id)] AS chain
+    LIMIT 5
+    """
+    rows = _run_query(client, cypher, {"test_id": test_id})
+    chains = [r.get("chain", []) for r in rows if r.get("chain")]
+    if not chains:
+        return {"status": "insufficient_graph_data", "chain": [], "missing": ["testcase_node"]}
+    return {"status": "complete", "root_cause_chain": chains[0], "all_paths": chains}
+
+
+def traverse_change_impact(changed_files: list[str]) -> list[dict]:
+    """Regression: changed files → features → testcases."""
+    client = get_neo4j_client()
+    if not client.available:
+        return []
+
+    results = []
+    for fpath in changed_files[:50]:
+        feature = _file_to_feature(fpath)
+        cypher = """
+        MATCH (feat:Feature)
+        WHERE toLower(feat.name) CONTAINS toLower($feature)
+        OPTIONAL MATCH (feat)-[:HAS_TESTCASE]->(tc:Testcase)
+        RETURN feat.name AS feature, collect(DISTINCT tc.id) AS test_ids
+        LIMIT 1
+        """
+        rows = _run_query(client, cypher, {"feature": feature})
+        for row in rows:
+            for tid in row.get("test_ids") or []:
+                if tid:
+                    results.append({
+                        "test_id": tid,
+                        "feature": row.get("feature"),
+                        "changed_file": fpath,
+                        "risk_score": 0.7,
+                    })
+    return results
+
+
+def _run_query(client, cypher: str, params: dict) -> list[dict]:
+    if not client.available:
+        return []
+    try:
+        with client._driver.session() as session:
+            result = session.run(cypher, params)
+            return [dict(r) for r in result]
+    except Exception as e:
+        logger.warning("Neo4j query failed: %s", e)
+        return []
+
+
+def _search_terms(query: str) -> list[str]:
+    keys = re.findall(r"app\.[a-z0-9_.]+", query, flags=re.I)
+    tokens = [t for t in re.split(r"[^\w\-./]+", query) if len(t) >= 3]
+    out: list[str] = []
+    for x in keys + tokens[:12]:
+        if x.lower() not in {y.lower() for y in out}:
+            out.append(x)
+    return out
+
+
+def _infer_feature(query: str) -> str:
+    q = query.lower()
+    if "entra" in q or "sso" in q:
+        return "Entra ID SSO"
+    if "wifi" in q or "802.1" in q:
+        return "Network WiFi 802.1x"
+    if "icg" in q or "certificate" in q:
+        return "ICG Certificate Management"
+    if "chromium" in q:
+        return "Chromium Browser"
+    return query[:80]
+
+
+def _file_to_feature(path: str) -> str:
+    name = path.split("/")[-1].replace("_", " ").replace(".py", "")
+    return name[:80]
+
+
+def _format_block(keys: list[str], paths: list[str], features: list[str]) -> str:
+    if not keys and not paths and not features:
+        return ""
+    lines = ["—— Knowledge graph entities (Neo4j) ——"]
+    if features:
+        lines.append("\n[Features & sections]")
+        for f in features[:10]:
+            lines.append(f"  • {f}")
+    if paths:
+        lines.append("\n[TC Setup / UI navigation paths]")
+        for p in paths[:20]:
+            lines.append(f"  • {p}")
+    if keys:
+        lines.append("\n[Registry keys]")
+        for k in keys[:30]:
+            lines.append(f"  • {k}")
+    return "\n".join(lines)

package/knowledge_base/graph_db/migrate_kg.py

@@ -0,0 +1,93 @@
+"""
+Migrate PostgreSQL knowledge graph to Neo4j (Phase 3).
+
+Usage:
+    python -m knowledge_base.graph_db.migrate_kg [--dry-run]
+"""
+from __future__ import annotations
+
+import argparse
+import logging
+
+from knowledge_base.db.client import get_conn, init_pool
+from knowledge_base.graph_db.neo4j_client import get_neo4j_client, should_write_neo4j
+from knowledge_base.graph_db.schema import apply_schema
+
+logger = logging.getLogger(__name__)
+
+_LABEL_MAP = {
+    "feature": "Feature",
+    "registry_key": "RegistryKey",
+    "ui_path": "UIPath",
+}
+
+
+def migrate(*, dry_run: bool = False) -> dict:
+    """Read PG KG tables and write to Neo4j."""
+    if not should_write_neo4j():
+        logger.warning("Neo4j migration skipped: set KG_SHADOW_MODE=true or KG_BACKEND=neo4j")
+        return {"nodes": 0, "edges": 0, "skipped": True}
+
+    client = get_neo4j_client()
+    if not client.available:
+        logger.error("Neo4j unavailable — cannot migrate")
+        return {"nodes": 0, "edges": 0, "error": "neo4j_unavailable"}
+
+    if dry_run:
+        logger.info("Dry run — counting PG nodes/edges only")
+    else:
+        apply_schema(client)
+
+    init_pool()
+    node_count = 0
+    edge_count = 0
+
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                "SELECT node_type, node_value, metadata FROM knowledge_graph_nodes"
+            )
+            nodes = cur.fetchall()
+
+            for node_type, node_value, metadata in nodes:
+                label = _LABEL_MAP.get(node_type, "Feature")
+                key_field = "name" if label == "Feature" else ("key" if label == "RegistryKey" else "path")
+                if not dry_run:
+                    client.upsert_entity(label, key_field, node_value, metadata or {})
+                node_count += 1
+
+            cur.execute(
+                """
+                SELECT e.relationship, n1.node_type, n1.node_value, n2.node_type, n2.node_value
+                FROM knowledge_graph_edges e
+                JOIN knowledge_graph_nodes n1 ON e.from_node_id = n1.id
+                JOIN knowledge_graph_nodes n2 ON e.to_node_id = n2.id
+                """
+            )
+            edges = cur.fetchall()
+
+            for rel, t1, v1, t2, v2 in edges:
+                l1 = _LABEL_MAP.get(t1, "Feature")
+                l2 = _LABEL_MAP.get(t2, "RegistryKey")
+                k1 = "name" if l1 == "Feature" else ("key" if l1 == "RegistryKey" else "path")
+                k2 = "name" if l2 == "Feature" else ("key" if l2 == "RegistryKey" else "path")
+                rel_type = rel.upper().replace(" ", "_")
+                if not dry_run:
+                    client.upsert_relationship(l1, k1, v1, rel_type, l2, k2, v2)
+                edge_count += 1
+
+    logger.info("Migration complete: %d nodes, %d edges (dry_run=%s)", node_count, edge_count, dry_run)
+    return {"nodes": node_count, "edges": edge_count, "dry_run": dry_run}
+
+
+def main() -> None:
+    logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s")
+    parser = argparse.ArgumentParser(description="Migrate PostgreSQL KG to Neo4j")
+    parser.add_argument("--dry-run", action="store_true")
+    args = parser.parse_args()
+    result = migrate(dry_run=args.dry_run)
+    print(result)
+
+
+if __name__ == "__main__":
+    main()

package/knowledge_base/graph_db/neo4j_client.py

@@ -0,0 +1,116 @@
+"""Neo4j client abstraction with graceful degradation when unavailable."""
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from knowledge_base.config import cfg
+
+logger = logging.getLogger(__name__)
+
+_client: "Neo4jClient | None" = None
+
+
+class Neo4jClient:
+    """Thin wrapper around the Neo4j driver. No-ops when driver is unavailable."""
+
+    def __init__(self) -> None:
+        self._driver = None
+        self._available = False
+        self._connect()
+
+    def _connect(self) -> None:
+        if not cfg.NEO4J_PASSWORD:
+            logger.info("Neo4j: NEO4J_PASSWORD not set — shadow mode disabled")
+            return
+        try:
+            from neo4j import GraphDatabase
+
+            self._driver = GraphDatabase.driver(
+                cfg.NEO4J_URI,
+                auth=(cfg.NEO4J_USER, cfg.NEO4J_PASSWORD),
+            )
+            self._driver.verify_connectivity()
+            self._available = True
+            logger.info("Neo4j: connected to %s", cfg.NEO4J_URI)
+        except ImportError:
+            logger.warning("Neo4j: neo4j driver not installed — pip install neo4j")
+        except Exception as e:
+            logger.warning("Neo4j: connection failed (%s) — shadow writes skipped", e)
+
+    @property
+    def available(self) -> bool:
+        return self._available and self._driver is not None
+
+    def health_check(self) -> dict[str, Any]:
+        if not self.available:
+            return {"status": "unavailable", "uri": cfg.NEO4J_URI}
+        try:
+            with self._driver.session() as session:
+                result = session.run("RETURN 1 AS ok")
+                ok = result.single()["ok"]
+            return {"status": "healthy" if ok == 1 else "degraded", "uri": cfg.NEO4J_URI}
+        except Exception as e:
+            return {"status": "error", "uri": cfg.NEO4J_URI, "error": str(e)}
+
+    def run_write(self, cypher: str, params: dict | None = None) -> None:
+        if not self.available:
+            return
+        with self._driver.session() as session:
+            session.run(cypher, params or {})
+
+    def upsert_entity(
+        self,
+        label: str,
+        key_field: str,
+        key_value: str,
+        properties: dict | None = None,
+    ) -> None:
+        """MERGE a node by key field."""
+        props = properties or {}
+        cypher = f"""
+        MERGE (n:{label} {{{key_field}: $key_value}})
+        SET n += $props
+        """
+        self.run_write(cypher, {"key_value": key_value, "props": props})
+
+    def upsert_relationship(
+        self,
+        from_label: str,
+        from_key: str,
+        from_value: str,
+        rel_type: str,
+        to_label: str,
+        to_key: str,
+        to_value: str,
+        rel_props: dict | None = None,
+    ) -> None:
+        cypher = f"""
+        MERGE (a:{from_label} {{{from_key}: $from_value}})
+        MERGE (b:{to_label} {{{to_key}: $to_value}})
+        MERGE (a)-[r:{rel_type}]->(b)
+        SET r += $rel_props
+        """
+        self.run_write(cypher, {
+            "from_value": from_value,
+            "to_value": to_value,
+            "rel_props": rel_props or {},
+        })
+
+    def close(self) -> None:
+        if self._driver:
+            self._driver.close()
+            self._driver = None
+            self._available = False
+
+
+def get_neo4j_client() -> Neo4jClient:
+    global _client
+    if _client is None:
+        _client = Neo4jClient()
+    return _client
+
+
+def should_write_neo4j() -> bool:
+    """True when Neo4j shadow or primary backend is active."""
+    return cfg.KG_BACKEND == "neo4j" or cfg.KG_SHADOW_MODE

package/knowledge_base/graph_db/schema.py

@@ -0,0 +1,22 @@
+"""Cypher schema definitions for IGEL EKG Neo4j graph (Phase 3)."""
+
+CONSTRAINTS = [
+    "CREATE CONSTRAINT feature_name IF NOT EXISTS FOR (f:Feature) REQUIRE f.name IS UNIQUE",
+    "CREATE CONSTRAINT registry_key_name IF NOT EXISTS FOR (r:RegistryKey) REQUIRE r.key IS UNIQUE",
+    "CREATE CONSTRAINT ui_path_path IF NOT EXISTS FOR (u:UIPath) REQUIRE u.path IS UNIQUE",
+    "CREATE CONSTRAINT testcase_id IF NOT EXISTS FOR (t:Testcase) REQUIRE t.id IS UNIQUE",
+    "CREATE CONSTRAINT version_name IF NOT EXISTS FOR (v:Version) REQUIRE v.version IS UNIQUE",
+]
+
+INDEXES = [
+    "CREATE INDEX feature_name_idx IF NOT EXISTS FOR (f:Feature) ON (f.name)",
+    "CREATE INDEX registry_key_idx IF NOT EXISTS FOR (r:RegistryKey) ON (r.key)",
+]
+
+
+def apply_schema(client) -> None:
+    """Apply constraints and indexes to Neo4j."""
+    if not client.available:
+        return
+    for stmt in CONSTRAINTS + INDEXES:
+        client.run_write(stmt)

package/knowledge_base/graph_db/temporal_ingest.py

@@ -0,0 +1,59 @@
+"""Version-aware temporal ingestion — Phase 6."""
+from __future__ import annotations
+
+import logging
+import re
+
+from knowledge_base.graph_db.graphiti_client import get_graphiti_client
+
+logger = logging.getLogger(__name__)
+
+_VERSION_RE = re.compile(r"IGEL\s*OS\s*(12\.\d+(?:\.\d+)?)", re.I)
+
+
+def extract_version_from_metadata(metadata: dict) -> str | None:
+    for key in ("version", "igel_version", "product_version"):
+        val = metadata.get(key)
+        if val and "12." in str(val):
+            return str(val).split()[0] if " " in str(val) else str(val)
+    title = metadata.get("title") or metadata.get("file_name") or ""
+    m = _VERSION_RE.search(str(title))
+    return m.group(1) if m else None
+
+
+def ingest_document_temporal(
+    document_title: str,
+    content: str,
+    version: str | None = None,
+    feature_name: str | None = None,
+) -> None:
+    """Create temporal edges for version-tagged documents."""
+    client = get_graphiti_client()
+    ver = version or "12.10"
+    feature = feature_name or document_title[:80]
+
+    keys = re.findall(r"\b(app\.[a-z0-9_.]+)\b", content, flags=re.I)
+    paths = re.findall(r"UMS[^>\n]{5,100}(?:>[^>\n]+)+", content)
+
+    client.record_change(
+        feature_name=feature,
+        version=ver,
+        change_type="document_ingested",
+        description=f"Ingested: {document_title[:200]}",
+        affected_keys=keys[:20],
+        affected_paths=paths[:10],
+    )
+    logger.debug("Temporal ingest: %s @ %s", feature, ver)
+
+
+def seed_default_versions() -> None:
+    """Seed known IGEL version evolution data."""
+    client = get_graphiti_client()
+    seeds = [
+        ("Entra ID SSO", "12.9", "enhanced", "Added MFA support to Entra ID SSO",
+         ["security.entra.mfa_enabled"], ["UMS Console > Profiles > Identity > Entra ID > MFA"]),
+        ("Entra ID SSO", "12.8", "introduced", "Initial Entra ID SSO support", [], []),
+        ("Network WiFi 802.1x", "12.10", "changed", "Revised network registry keys", [], []),
+    ]
+    for feature, ver, ctype, desc, keys, paths in seeds:
+        client.record_change(feature, ver, ctype, desc, keys, paths)

package/knowledge_base/ingestion/__init__.py

@@ -0,0 +1 @@
+"""Document and knowledge graph ingestion."""