igel-qe-core 1.0.0

package/knowledge_base/ingestion/kg_extractor.py

@@ -0,0 +1,358 @@
+"""
+Extract registry keys / TC Setup paths from kb_parent_chunks into knowledge_graph_*.
+
+Targets schema from ensure_knowledge_graph_tables(): UNIQUE(node_type,normalized) on nodes,
+UNIQUE(source_node_id,target_node_id,relation) on edges.
+"""
+
+from __future__ import annotations
+
+import argparse
+import logging
+import re
+
+from knowledge_base.config import cfg
+from knowledge_base.db.client import close_pool, get_conn, init_pool
+from knowledge_base.db.kg_introspect import clear_kg_layout_cache
+from knowledge_base.db.schema import ensure_knowledge_graph_tables
+
+logger = logging.getLogger(__name__)
+
+_REGISTRY_RE = re.compile(r"\b(app\.[a-z0-9]+(?:\.[a-z0-9_]+)+)", re.IGNORECASE)
+_TC_SETUP_RE = re.compile(
+    r"(?:TC\s*)?Setup\s*[>:]\s*((?:[^\n=]+)(?:\s*>\s*[^\n=]+)*(?:\s*=\s*[^\n]+)?)",
+    re.IGNORECASE,
+)
+
+
+def _upsert_node(cur, *, node_type: str, value: str, normalized: str, parent_chunk_id: str) -> str | None:
+    sql_ins = """
+    INSERT INTO knowledge_graph_nodes (node_type, value, normalized, parent_chunk_id)
+    VALUES (%s, %s, %s, %s::uuid)
+    ON CONFLICT (node_type, normalized) DO UPDATE SET
+        value = EXCLUDED.value,
+        parent_chunk_id = COALESCE(EXCLUDED.parent_chunk_id, knowledge_graph_nodes.parent_chunk_id)
+    RETURNING id::text
+    """
+    try:
+        cur.execute(
+            sql_ins,
+            (node_type, value[:4000], normalized[:2000], parent_chunk_id),
+        )
+        row = cur.fetchone()
+        return str(row[0]) if row else None
+    except Exception:
+        sel = """
+        SELECT id::text FROM knowledge_graph_nodes
+        WHERE node_type = %s AND normalized = %s LIMIT 1
+        """
+        cur.execute(sel, (node_type, normalized[:2000]))
+        found = cur.fetchone()
+        if found:
+            return str(found[0])
+        sql_plain = """
+        INSERT INTO knowledge_graph_nodes (node_type, value, normalized, parent_chunk_id)
+        VALUES (%s, %s, %s, %s::uuid) RETURNING id::text
+        """
+        cur.execute(
+            sql_plain,
+            (node_type, value[:4000], normalized[:2000], parent_chunk_id),
+        )
+        row = cur.fetchone()
+        return str(row[0]) if row else None
+
+
+def _link_edge(cur, src: str | None, tgt: str | None, relation: str) -> bool:
+    if not src or not tgt or src == tgt:
+        return False
+    sql = """
+    INSERT INTO knowledge_graph_edges (source_node_id, target_node_id, relation)
+    VALUES (%s::uuid, %s::uuid, %s)
+    ON CONFLICT (source_node_id, target_node_id, relation) DO NOTHING
+    """
+    try:
+        cur.execute(sql, (src, tgt, relation[:200]))
+        return cur.rowcount > 0
+    except Exception:
+        return False
+
+
+def extract_from_parents(
+    limit: int | None = None,
+    offset: int = 0,
+    document_ids: list[str] | None = None,
+) -> tuple[int, int]:
+    """Extract KG entities from parent chunks.
+
+    Args:
+        limit/offset: pagination for full-table scans.
+        document_ids: if provided, only process parent chunks belonging to
+                      these documents (incremental mode for ingest integration).
+    Returns:
+        (node_upsert_count, edge_insert_count)
+    """
+    ensure_knowledge_graph_tables()
+    clear_kg_layout_cache()
+
+    if document_ids:
+        sql = """
+            SELECT id::text, content, section_title FROM kb_parent_chunks
+            WHERE document_id = ANY(%s::uuid[])
+            ORDER BY created_at NULLS LAST, id
+        """
+        params: list = [document_ids]
+    else:
+        sql = """
+            SELECT id::text, content, section_title FROM kb_parent_chunks
+            ORDER BY created_at NULLS LAST, id
+        """
+        params = []
+        if limit is not None:
+            sql += " LIMIT %s OFFSET %s"
+            params.extend([limit, offset])
+
+    node_ops = 0
+    edges_ok = 0
+
+    with get_conn() as connection:
+        with connection.cursor() as cur:
+            cur.execute(sql, tuple(params))
+            rows = cur.fetchall()
+
+            for pid, content, title in rows:
+                n, e = _extract_single_parent(cur, pid, content, title)
+                node_ops += n
+                edges_ok += e
+
+    return node_ops, edges_ok
+
+
+def extract_for_document(document_id: str) -> tuple[int, int]:
+    """Extract KG entities for a single document (called during ingest)."""
+    return extract_from_parents(document_ids=[document_id])
+
+
+_VERSION_RE = re.compile(r"\b(?:IGEL OS|UMS|ICG|COSMOS|IMI)\s*[\d.]+(?:\s*[A-Z]{2})?\b", re.IGNORECASE)
+_ERROR_RE = re.compile(r"\b(?:ERR|ERROR|DEFECT|BUG)[-_ ]?\d{3,8}\b", re.IGNORECASE)
+_CLI_RE = re.compile(r"(?:^|\n)\s*(?:igel\s+)?(?:[a-z][a-z0-9_-]+\s+){1,6}[a-z0-9_-]+\s+(?:--[\w-]+(?:=\S+)?\s*)+", re.IGNORECASE)
+_CROSS_REF_RE = re.compile(r"(?:see|refer to|documented in|depends on)\s+([^\n.]{5,80})", re.IGNORECASE)
+
+
+def _llm_extract_entities(content: str, title: str | None) -> dict:
+    """LLM-assisted entity extraction for orphan parent chunks."""
+    try:
+        from knowledge_base.generation.test_generator import _call_gpt
+        prompt = f"""Extract structured entities from this IGEL documentation section.
+Return JSON only with keys: registry_keys (list), ui_paths (list), products (list),
+versions (list), error_codes (list), cli_commands (list), related_features (list).
+
+Section title: {title or 'unknown'}
+Content:
+{content[:3500]}"""
+        raw = _call_gpt(
+            "You extract IGEL test automation entities. Return valid JSON only.",
+            prompt,
+            max_tokens=800,
+            temperature=0.0,
+        )
+        import json
+        start = raw.find("{")
+        end = raw.rfind("}") + 1
+        if start >= 0 and end > start:
+            return json.loads(raw[start:end])
+    except Exception as exc:
+        logger.debug("LLM KG extraction failed: %s", exc)
+    return {}
+
+
+def enrich_orphan_parents(*, limit: int = 500) -> tuple[int, int]:
+    """Run LLM extraction on parent chunks with no KG entity."""
+    ensure_knowledge_graph_tables()
+    sql = """
+        SELECT p.id::text, p.content, p.section_title
+        FROM kb_parent_chunks p
+        WHERE NOT EXISTS (
+            SELECT 1 FROM knowledge_graph_nodes n WHERE n.parent_chunk_id = p.id
+        )
+        ORDER BY p.created_at NULLS LAST
+        LIMIT %s
+    """
+    node_ops = edges_ok = 0
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(sql, (limit,))
+            rows = cur.fetchall()
+            for pid, content, title in rows:
+                n, e = _extract_single_parent(cur, pid, content, title)
+                node_ops += n
+                edges_ok += e
+                if n == 0:
+                    entities = _llm_extract_entities(content, title)
+                    n2, e2 = _apply_llm_entities(cur, pid, title, entities)
+                    node_ops += n2
+                    edges_ok += e2
+    return node_ops, edges_ok
+
+
+def _apply_llm_entities(cur, pid: str, title: str | None, entities: dict) -> tuple[int, int]:
+    node_ops = edges_ok = 0
+    feature_name = (title or "").strip()[:240] or ("section_" + pid[:8])
+    fid = _upsert_node(
+        cur, node_type="feature", value=feature_name,
+        normalized=feature_name.lower()[:2000], parent_chunk_id=pid,
+    )
+    if fid:
+        node_ops += 1
+
+    mapping = [
+        ("registry_keys", "registry_key", "uses"),
+        ("ui_paths", "ui_path", "configured_via"),
+        ("products", "product", "documented_in"),
+        ("versions", "version", "documented_in"),
+        ("error_codes", "error_code", "related_to"),
+        ("cli_commands", "cli_command", "uses"),
+        ("related_features", "feature", "depends_on"),
+    ]
+    for key, node_type, relation in mapping:
+        for val in entities.get(key) or []:
+            if not val or not str(val).strip():
+                continue
+            val_s = str(val).strip()[:2000]
+            nid = _upsert_node(
+                cur, node_type=node_type, value=val_s,
+                normalized=val_s.lower()[:2000], parent_chunk_id=pid,
+            )
+            if nid:
+                node_ops += 1
+                if fid and _link_edge(cur, fid, nid, relation):
+                    edges_ok += 1
+    return node_ops, edges_ok
+
+
+def _extract_single_parent(cur, pid: str, content: str, title: str | None) -> tuple[int, int]:
+    """Process one parent chunk: create feature node + registry/ui_path nodes + edges."""
+    node_ops = 0
+    edges_ok = 0
+
+    blob = f"{title or ''}\n{content}"
+    feature_name = (title or "").strip()[:240] or ("section_" + pid[:8])
+
+    fid = _upsert_node(
+        cur,
+        node_type="feature",
+        value=feature_name,
+        normalized=feature_name.lower()[:2000],
+        parent_chunk_id=pid,
+    )
+    if fid:
+        node_ops += 1
+
+    for m in _REGISTRY_RE.findall(blob):
+        val = m.strip()
+        rid = _upsert_node(
+            cur,
+            node_type="registry_key",
+            value=val,
+            normalized=val.lower(),
+            parent_chunk_id=pid,
+        )
+        if rid:
+            node_ops += 1
+            if fid and _link_edge(cur, fid, rid, "uses"):
+                edges_ok += 1
+
+    for m in _TC_SETUP_RE.findall(blob):
+        path = m.strip()[:2000]
+        if len(path) < 8:
+            continue
+        uid = _upsert_node(
+            cur,
+            node_type="ui_path",
+            value=path,
+            normalized=path.lower()[:2000],
+            parent_chunk_id=pid,
+        )
+        if uid:
+            node_ops += 1
+            if fid and _link_edge(cur, fid, uid, "configured_via"):
+                edges_ok += 1
+
+    for m in _VERSION_RE.findall(blob):
+        val = m.strip()
+        vid = _upsert_node(cur, node_type="version", value=val, normalized=val.lower(), parent_chunk_id=pid)
+        if vid and fid and _link_edge(cur, fid, vid, "documented_in"):
+            node_ops += 1
+            edges_ok += 1
+
+    for m in _ERROR_RE.findall(blob):
+        val = m.strip()
+        eid = _upsert_node(cur, node_type="error_code", value=val, normalized=val.lower(), parent_chunk_id=pid)
+        if eid:
+            node_ops += 1
+            if fid and _link_edge(cur, fid, eid, "related_to"):
+                edges_ok += 1
+
+    for m in _CROSS_REF_RE.findall(blob):
+        ref = m.strip()[:2000]
+        rid = _upsert_node(cur, node_type="feature", value=ref, normalized=ref.lower()[:2000], parent_chunk_id=pid)
+        if rid and fid and _link_edge(cur, fid, rid, "depends_on"):
+            node_ops += 1
+            edges_ok += 1
+
+    _shadow_write_neo4j(feature_name, blob)
+
+    return node_ops, edges_ok
+
+
+def _shadow_write_neo4j(feature_name: str, blob: str) -> None:
+    """Dual-write extracted entities to Neo4j when shadow mode is enabled (Phase 3)."""
+    try:
+        from knowledge_base.graph_db.neo4j_client import get_neo4j_client, should_write_neo4j
+
+        if not should_write_neo4j():
+            return
+
+        client = get_neo4j_client()
+        if not client.available:
+            return
+
+        client.upsert_entity("Feature", "name", feature_name, {"source": "kg_extractor"})
+
+        for m in _REGISTRY_RE.findall(blob):
+            key = m.strip()
+            client.upsert_entity("RegistryKey", "key", key, {})
+            client.upsert_relationship(
+                "Feature", "name", feature_name,
+                "USES",
+                "RegistryKey", "key", key,
+            )
+
+        for m in _TC_SETUP_RE.findall(blob):
+            path = m.strip()[:2000]
+            if len(path) < 8:
+                continue
+            client.upsert_entity("UIPath", "path", path, {})
+            client.upsert_relationship(
+                "Feature", "name", feature_name,
+                "CONFIGURED_VIA",
+                "UIPath", "path", path,
+            )
+    except Exception as e:
+        logger.debug("Neo4j shadow write skipped: %s", e)
+
+
+def main() -> None:
+    logging.basicConfig(level=getattr(logging, cfg.LOG_LEVEL.upper(), logging.INFO))
+    parser = argparse.ArgumentParser(description="Extract KG entities from kb_parent_chunks.")
+    parser.add_argument("--limit", type=int, default=None)
+    parser.add_argument("--offset", type=int, default=0)
+    args = parser.parse_args()
+
+    init_pool()
+    n, e = extract_from_parents(limit=args.limit, offset=args.offset)
+    print(f"Done. Node upserts / inserts touched ≈ {n}, new edges inserted ≈ {e}")
+    close_pool()
+
+
+if __name__ == "__main__":
+    main()

package/knowledge_base/ingestion/web_ingest.py

@@ -0,0 +1,83 @@
+"""Ingest web-fetched pages into PostgreSQL vector store."""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+
+from knowledge_base.chunking.chunker import chunk_sections
+from knowledge_base.config import cfg
+from knowledge_base.db.client import init_pool
+from knowledge_base.db.schema import (
+    clear_document_chunks,
+    document_exists,
+    document_has_chunks,
+    ingest_document_atomic,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def ingest_web_items(items: list[dict], *, source_label: str = "web") -> int:
+    """Chunk and embed web pages into kb_documents/kb_chunks with dedup + atomic writes."""
+    if not items:
+        return 0
+
+    init_pool()
+    total_children = 0
+
+    for item in items:
+        url = item.get("url", "")
+        title = item.get("title", url)
+        sections = item.get("sections") or []
+        source_type = item.get("source_type", "kb_web")
+        product = item.get("product", "IGEL")
+
+        if not sections and item.get("raw_text"):
+            from knowledge_base.parsers.markdown_parser import Section
+
+            sections = [
+                Section(
+                    title=title,
+                    level=1,
+                    content=item["raw_text"],
+                    metadata={"canonical_url": url, "source_type": source_type},
+                )
+            ]
+
+        file_name = f"web:{source_label}:{hashlib.sha256(url.encode()).hexdigest()[:12]}"
+        file_hash = hashlib.sha256((url + title).encode()).hexdigest()
+
+        existing_id = document_exists(file_hash)
+        if existing_id and document_has_chunks(existing_id):
+            logger.debug("web_ingest skip duplicate: %s", url)
+            continue
+
+        parents, warnings = chunk_sections(sections, file_name, product)
+        for w in warnings[:5]:
+            logger.debug("web_ingest warning: %s", w)
+
+        from knowledge_base.embeddings.azure_embedder import embed_texts, format_for_pgvector
+
+        _, _, doc_children, _ = ingest_document_atomic(
+            file_name=file_name,
+            file_path=url,
+            file_type="html",
+            file_hash=file_hash,
+            product=product,
+            doc_metadata={
+                "source_type": source_type,
+                "canonical_url": url,
+                "title": title,
+                "web_source": source_label,
+            },
+            parents=parents,
+            existing_id=existing_id,
+            embed_fn=embed_texts,
+            format_fn=format_for_pgvector,
+            batch_size=cfg.BATCH_SIZE,
+        )
+        total_children += doc_children
+
+    logger.info("web_ingest: ingested %d child chunks from %d pages", total_children, len(items))
+    return total_children

package/knowledge_base/intelligence/__init__.py

@@ -0,0 +1 @@
+"""Execution intelligence package — Phase 7."""

package/knowledge_base/intelligence/execution_ingest.py

@@ -0,0 +1,34 @@
+"""Execution memory ingestion — Phase 7."""
+from __future__ import annotations
+
+import logging
+
+from knowledge_base.config import cfg
+from knowledge_base.memory.cognee_client import get_cognee_client
+from knowledge_base.memory.memory_types import MemoryType
+
+logger = logging.getLogger(__name__)
+
+
+def ingest_execution_results(results: list[dict], *, dry_run: bool = False) -> dict:
+    """
+    Parse CI results and store failure patterns in operational memory.
+    Each result: {test_id, status, failure_log, feature_area}
+    """
+    if not cfg.COGNEE_ENABLED:
+        return {"ingested": 0, "skipped": len(results), "reason": "COGNEE_ENABLED=false"}
+
+    client = get_cognee_client()
+    ingested = 0
+    for r in results:
+        if r.get("status") != "failed":
+            continue
+        content = f"Test {r.get('test_id')} failed: {(r.get('failure_log') or '')[:500]}"
+        feature = r.get("feature_area", "IGEL OS General")
+        if dry_run:
+            ingested += 1
+            continue
+        client.add_memory(content, MemoryType.EXECUTION, feature_area=feature)
+        ingested += 1
+
+    return {"ingested": ingested, "total": len(results)}

package/knowledge_base/intelligence/predictive_engine.py

@@ -0,0 +1,7 @@
+"""Predictive test failure engine — Phase 7 stub."""
+from __future__ import annotations
+
+
+def predict_failures(feature_area: str, limit: int = 10) -> list[dict]:
+    """Placeholder for predictive failure analysis."""
+    return []

package/knowledge_base/intelligence/risk_scorer.py

@@ -0,0 +1,8 @@
+"""Test risk scoring — Phase 7."""
+from __future__ import annotations
+
+
+def score_test_risk(test_id: str, *, failure_rate: float = 0.0, recency_days: int = 30) -> float:
+    """Score a test case by failure rate and recency (0.0–1.0)."""
+    recency_factor = max(0.1, 1.0 - (recency_days / 365))
+    return min(1.0, failure_rate * 0.7 + recency_factor * 0.3)

package/knowledge_base/memory/__init__.py

@@ -0,0 +1,6 @@
+"""Operational memory layer — Phase 4 Cognee integration."""
+
+from knowledge_base.memory.cognee_client import CogneeClient, get_cognee_client
+from knowledge_base.memory.memory_types import MemoryType
+
+__all__ = ["CogneeClient", "get_cognee_client", "MemoryType"]