igel-qe-core 1.0.0

package/knowledge_base/core/precedence.py

@@ -0,0 +1,354 @@
+"""
+Source Precedence Rules for IGEL KB System
+===========================================
+Enforces trust hierarchy when conflicting information appears in multiple sources.
+
+Precedence Order (Highest → Lowest):
+1. Gold TC (manual, reviewed, canonical test cases)
+2. Jira TC export (test cases from Jira, may be outdated)
+3. Confluence (team documentation, wiki pages)
+4. Official IGEL docs (reference manuals, admin guides)
+5. Screenshots (UI images with OCR)
+6. Execution history (past test runs, logs)
+
+Rules:
+- Higher precedence sources override lower ones for conflicting info
+- When merging: use highest-precedence source for each fact
+- Surface source attribution in output (which doc provided which info)
+- Flag conflicts for human review if precedence gap is small
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass
+class SourceType:
+    """Source type with precedence rank."""
+    name: str
+    rank: int              # Lower number = higher precedence
+    description: str
+    
+    def __lt__(self, other: SourceType) -> bool:
+        return self.rank < other.rank
+    
+    def __le__(self, other: SourceType) -> bool:
+        return self.rank <= other.rank
+
+
+# Define source types (precedence order)
+SOURCE_TYPES = {
+    "gold_tc": SourceType(
+        name="gold_tc",
+        rank=1,
+        description="Gold Test Case (manual, reviewed, canonical)",
+    ),
+    "jira_tc": SourceType(
+        name="jira_tc",
+        rank=2,
+        description="Jira Test Case Export",
+    ),
+    "confluence": SourceType(
+        name="confluence",
+        rank=3,
+        description="Confluence Documentation",
+    ),
+    "igel_docs": SourceType(
+        name="igel_docs",
+        rank=4,
+        description="Official IGEL Documentation",
+    ),
+    "kb_web": SourceType(
+        name="kb_web",
+        rank=4,
+        description="IGEL Knowledge Base (kb.igel.com)",
+    ),
+    "community": SourceType(
+        name="community",
+        rank=5,
+        description="IGEL Community / Forum Posts",
+    ),
+    "screenshot": SourceType(
+        name="screenshot",
+        rank=6,
+        description="Screenshot with OCR",
+    ),
+    "execution_history": SourceType(
+        name="execution_history",
+        rank=7,
+        description="Test Execution History",
+    ),
+}
+
+
+def infer_source_type(chunk_metadata: dict) -> SourceType:
+    """
+    Infer source type from chunk metadata.
+    
+    Looks at:
+    - metadata.product (e.g., "IGEL-GOLD-TC", "IGEL-JIRA", "IGEL-CONFLUENCE")
+    - metadata.file_path
+    - metadata.source_type (if explicitly tagged)
+    """
+    # Check explicit source_type tag
+    if "source_type" in chunk_metadata:
+        source_name = chunk_metadata["source_type"].lower()
+        if source_name in SOURCE_TYPES:
+            return SOURCE_TYPES[source_name]
+    product = chunk_metadata.get("product", "").lower()
+    if "gold" in product or "canonical" in product:
+        return SOURCE_TYPES["gold_tc"]
+    elif "jira" in product:
+        return SOURCE_TYPES["jira_tc"]
+    elif "confluence" in product or "wiki" in product:
+        return SOURCE_TYPES["confluence"]
+    elif "igel" in product or "docs" in product or "manual" in product:
+        return SOURCE_TYPES["igel_docs"]
+    elif "kb_web" in product or "kb.igel" in product:
+        return SOURCE_TYPES["kb_web"]
+    elif "community" in product or "forum" in product:
+        return SOURCE_TYPES["community"]
+    elif "screenshot" in product or "image" in product:
+        return SOURCE_TYPES["screenshot"]
+    elif "execution" in product or "history" in product or "log" in product:
+        return SOURCE_TYPES["execution_history"]
+    
+    # Check file path
+    file_path = chunk_metadata.get("file_path", "").lower()
+    if "gold" in file_path:
+        return SOURCE_TYPES["gold_tc"]
+    elif "jira" in file_path:
+        return SOURCE_TYPES["jira_tc"]
+    elif "confluence" in file_path or "wiki" in file_path:
+        return SOURCE_TYPES["confluence"]
+    elif "manual" in file_path or "guide" in file_path or "reference" in file_path:
+        return SOURCE_TYPES["igel_docs"]
+    elif "kb.igel" in file_path or file_path.startswith("web:kb"):
+        return SOURCE_TYPES["kb_web"]
+    elif "community" in file_path or file_path.startswith("web:community"):
+        return SOURCE_TYPES["community"]
+    elif "screenshot" in file_path or "image" in file_path:
+        return SOURCE_TYPES["screenshot"]
+    elif "execution" in file_path or "history" in file_path:
+        return SOURCE_TYPES["execution_history"]
+    
+    # Default to IGEL docs (safest middle ground)
+    return SOURCE_TYPES["igel_docs"]
+
+
+@dataclass
+class RankedChunk:
+    """Chunk with source precedence metadata."""
+    chunk: Any                  # Original chunk object
+    source_type: SourceType
+    retrieval_score: float      # RRF or rerank score
+    content: str
+    metadata: dict
+    
+    @property
+    def composite_rank(self) -> tuple[int, float]:
+        """
+        Composite ranking key: (source_rank, -retrieval_score).
+        Sorts by source precedence first, then retrieval score (higher better).
+        """
+        return (self.source_type.rank, -self.retrieval_score)
+
+
+def rank_chunks_by_precedence(
+    chunks: list,
+    retrieval_scores: dict | None = None,
+) -> list[RankedChunk]:
+    """
+    Rank chunks by source precedence, then retrieval score.
+    
+    Args:
+        chunks: List of chunk objects (must have .metadata attribute)
+        retrieval_scores: Optional dict mapping chunk ID to retrieval score
+    
+    Returns:
+        List of RankedChunk objects, sorted by precedence
+    """
+    ranked = []
+    
+    for chunk in chunks:
+        # Extract metadata
+        metadata = getattr(chunk, 'metadata', {})
+        if isinstance(metadata, str):
+            import json
+            try:
+                metadata = json.loads(metadata)
+            except:
+                metadata = {}
+        
+        # Infer source type
+        source_type = infer_source_type(metadata)
+        
+        # Get retrieval score
+        chunk_id = getattr(chunk, 'id', None)
+        retrieval_score = 0.0
+        if retrieval_scores and chunk_id:
+            retrieval_score = retrieval_scores.get(str(chunk_id), 0.0)
+        elif hasattr(chunk, 'rrf_score'):
+            retrieval_score = chunk.rrf_score
+        elif hasattr(chunk, 'rerank_score'):
+            retrieval_score = chunk.rerank_score
+        
+        # Get content
+        content = getattr(chunk, 'content', '') or getattr(chunk, 'parent_content', '') or getattr(chunk, 'child_content', '')
+        
+        ranked.append(RankedChunk(
+            chunk=chunk,
+            source_type=source_type,
+            retrieval_score=retrieval_score,
+            content=content,
+            metadata=metadata,
+        ))
+    
+    # Sort by composite rank (source precedence, then retrieval score)
+    ranked.sort(key=lambda rc: rc.composite_rank)
+    
+    return ranked
+
+
+def apply_precedence_filter(
+    chunks: list,
+    top_k: int = 10,
+    retrieval_scores: dict | None = None,
+) -> tuple[list, dict]:
+    """
+    Apply source precedence to filter and rank chunks.
+    
+    Strategy:
+    1. Rank all chunks by precedence
+    2. Select top K chunks
+    3. Within each precedence tier, prefer higher retrieval scores
+    4. Return filtered list + attribution metadata
+    
+    Returns:
+        (filtered_chunks, attribution_metadata)
+        
+        attribution_metadata: dict with source breakdown
+    """
+    # Rank chunks
+    ranked = rank_chunks_by_precedence(chunks, retrieval_scores)
+    
+    # Take top K
+    top_chunks = ranked[:top_k]
+    
+    # Build attribution metadata
+    source_counts = {}
+    for rc in top_chunks:
+        source_name = rc.source_type.name
+        source_counts[source_name] = source_counts.get(source_name, 0) + 1
+    
+    attribution = {
+        "total_chunks": len(top_chunks),
+        "source_breakdown": source_counts,
+        "highest_precedence": top_chunks[0].source_type.name if top_chunks else None,
+        "precedence_order": [rc.source_type.name for rc in top_chunks],
+    }
+    
+    # Extract original chunks
+    filtered = [rc.chunk for rc in top_chunks]
+    
+    return filtered, attribution
+
+
+def resolve_conflict(
+    sources: list[tuple[SourceType, Any]],
+) -> tuple[SourceType, Any] | None:
+    """
+    Resolve conflicting information by source precedence (lowest rank wins).
+
+    Args:
+        sources: List of (SourceType, value) tuples from different sources.
+
+    Returns:
+        Winning (SourceType, value) or None if empty.
+    """
+    if not sources:
+        return None
+    return min(sources, key=lambda pair: pair[0].rank)
+
+
+def detect_precedence_conflicts(
+    chunks: list,
+    conflict_threshold: int = 2,
+) -> list[dict]:
+    """
+    Detect conflicts where similar information appears in sources
+    with different precedence levels.
+    
+    Args:
+        chunks: List of chunks
+        conflict_threshold: Max precedence rank gap to consider conflict
+    
+    Returns:
+        List of conflict reports (for human review)
+    """
+    ranked = rank_chunks_by_precedence(chunks)
+    conflicts = []
+    
+    # Group by similar content (simplified: just check for registry keys/UI paths)
+    import re
+    
+    # Extract entities from each chunk
+    for i, rc1 in enumerate(ranked):
+        entities_1 = set(re.findall(r'app\.[a-z0-9_.]+', rc1.content.lower()))
+        entities_1.update(re.findall(r'TC Setup[^>\n]*(?:>[^>\n]*)*', rc1.content, re.I))
+        
+        for rc2 in ranked[i+1:]:
+            # Check precedence gap
+            rank_gap = abs(rc1.source_type.rank - rc2.source_type.rank)
+            if rank_gap > conflict_threshold:
+                continue  # Too far apart in precedence
+            
+            # Check for overlapping entities
+            entities_2 = set(re.findall(r'app\.[a-z0-9_.]+', rc2.content.lower()))
+            entities_2.update(re.findall(r'TC Setup[^>\n]*(?:>[^>\n]*)*', rc2.content, re.I))
+            
+            overlap = entities_1 & entities_2
+            if overlap:
+                conflicts.append({
+                    "entity": list(overlap)[0] if overlap else "unknown",
+                    "source_1": rc1.source_type.name,
+                    "source_2": rc2.source_type.name,
+                    "rank_gap": rank_gap,
+                    "chunk_1_id": getattr(rc1.chunk, 'id', None),
+                    "chunk_2_id": getattr(rc2.chunk, 'id', None),
+                    "recommendation": f"Prefer {rc1.source_type.name} over {rc2.source_type.name}",
+                })
+    
+    return conflicts
+
+
+def format_source_attribution(attribution: dict) -> str:
+    """
+    Format source attribution as human-readable text.
+    
+    Example:
+        Sources (10 chunks):
+        - Gold TC: 4 chunks
+        - IGEL Docs: 3 chunks
+        - Confluence: 2 chunks
+        - Jira TC: 1 chunk
+    """
+    lines = [f"Sources ({attribution['total_chunks']} chunks):"]
+    
+    # Sort by precedence
+    breakdown = attribution['source_breakdown']
+    sorted_sources = sorted(
+        breakdown.items(),
+        key=lambda x: SOURCE_TYPES[x[0]].rank
+    )
+    
+    for source_name, count in sorted_sources:
+        source_desc = SOURCE_TYPES[source_name].description
+        lines.append(f"  - {source_desc}: {count} chunk{'s' if count > 1 else ''}")
+    
+    if attribution.get('highest_precedence'):
+        highest = SOURCE_TYPES[attribution['highest_precedence']].description
+        lines.append(f"\nPrimary source: {highest}")
+    
+    return "\n".join(lines)

package/knowledge_base/core/request_context.py

@@ -0,0 +1,54 @@
+"""Request context and API security helpers."""
+
+from __future__ import annotations
+
+import hashlib
+import uuid
+from contextvars import ContextVar
+
+request_id_var: ContextVar[str] = ContextVar("request_id", default="")
+api_key_scope_var: ContextVar[str] = ContextVar("api_key_scope", default="read")
+
+
+def new_request_id() -> str:
+    rid = str(uuid.uuid4())
+    request_id_var.set(rid)
+    return rid
+
+
+def get_request_id() -> str:
+    return request_id_var.get() or ""
+
+
+def hash_api_key(key: str | None) -> str:
+    if not key:
+        return "anonymous"
+    return hashlib.sha256(key.encode()).hexdigest()[:16]
+
+
+def parse_api_keys(raw: str) -> dict[str, str]:
+    """
+    Parse API_KEYS env: key:scope,key2:scope2
+    scope in read|write|admin
+    """
+    out: dict[str, str] = {}
+    for part in (raw or "").split(","):
+        part = part.strip()
+        if not part:
+            continue
+        if ":" in part:
+            k, scope = part.split(":", 1)
+            out[k.strip()] = scope.strip().lower()
+        else:
+            out[part] = "admin"
+    return out
+
+
+def resolve_scope(api_key: str | None, keys: dict[str, str], legacy_key: str) -> str:
+    if not api_key:
+        return "anonymous"
+    if api_key in keys:
+        return keys[api_key]
+    if legacy_key and api_key == legacy_key:
+        return "admin"
+    return "invalid"

package/knowledge_base/db/client.py

@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+import logging
+import os
+
+import psycopg2
+import psycopg2.pool
+from contextlib import contextmanager
+
+from knowledge_base.config import cfg
+
+logger = logging.getLogger(__name__)
+
+_pool: psycopg2.pool.ThreadedConnectionPool | None = None
+
+
+def _pool_kwargs() -> dict:
+    sslmode = os.getenv("DB_SSLMODE", "prefer")
+    return {
+        "host": cfg.DB_HOST,
+        "port": cfg.DB_PORT,
+        "dbname": cfg.DB_NAME,
+        "user": cfg.DB_USER,
+        "password": cfg.DB_PASSWORD,
+        "connect_timeout": 10,
+        "keepalives": 1,
+        "keepalives_idle": 30,
+        "keepalives_interval": 10,
+        "keepalives_count": 5,
+        "sslmode": sslmode,
+    }
+
+
+def init_pool(minconn: int = 2, maxconn: int = 10) -> None:
+    global _pool
+    if _pool is not None:
+        return
+    _pool = psycopg2.pool.ThreadedConnectionPool(
+        minconn,
+        maxconn,
+        **_pool_kwargs(),
+    )
+
+
+def _ping(conn) -> None:
+    with conn.cursor() as cur:
+        cur.execute("SELECT 1")
+
+
+@contextmanager
+def get_conn(*, autocommit: bool = True):
+    """Get a pooled connection. Commits on success when autocommit=True."""
+    if _pool is None:
+        init_pool()
+    conn = _pool.getconn()
+    try:
+        try:
+            _ping(conn)
+        except Exception:
+            conn.close()
+            conn = _pool.getconn()
+            _ping(conn)
+        yield conn
+        if autocommit:
+            conn.commit()
+    except Exception:
+        try:
+            conn.rollback()
+        except Exception as rollback_exc:
+            logger.debug("Rollback failed while handling DB error: %s", rollback_exc)
+        raise
+    finally:
+        _pool.putconn(conn)
+
+
+@contextmanager
+def get_transaction():
+    """Yield a connection inside a single transaction (commit on success)."""
+    with get_conn(autocommit=False) as conn:
+        yield conn
+
+
+def test_connection() -> bool:
+    try:
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute("SELECT version();")
+                version = cur.fetchone()[0]
+                print(f"  Connected: {version[:60]}")
+        return True
+    except Exception as e:
+        print(f"  DB connection failed: {e}")
+        return False
+
+
+def close_pool() -> None:
+    global _pool
+    if _pool:
+        _pool.closeall()
+        _pool = None

package/knowledge_base/db/connector_sync.py

@@ -0,0 +1,43 @@
+"""Connector incremental sync state persistence."""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from knowledge_base.db.client import get_conn
+
+
+def get_last_sync(connector_name: str) -> datetime | None:
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                "SELECT last_sync_timestamp FROM connector_sync_state WHERE connector_name=%s",
+                (connector_name,),
+            )
+            row = cur.fetchone()
+            return row[0] if row and row[0] else None
+
+
+def update_sync_state(
+    connector_name: str,
+    *,
+    status: str = "synced",
+    cursor: str | None = None,
+    advance_timestamp: bool = True,
+) -> None:
+    now = datetime.now(timezone.utc)
+    last_sync_timestamp = now if advance_timestamp else None
+    with get_conn() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                INSERT INTO connector_sync_state (connector_name, last_sync_timestamp, cursor, status, updated_at)
+                VALUES (%s, %s, %s, %s, %s)
+                ON CONFLICT (connector_name) DO UPDATE SET
+                    last_sync_timestamp = COALESCE(EXCLUDED.last_sync_timestamp, connector_sync_state.last_sync_timestamp),
+                    cursor = EXCLUDED.cursor,
+                    status = EXCLUDED.status,
+                    updated_at = EXCLUDED.updated_at
+                """,
+                (connector_name, last_sync_timestamp, cursor, status, now),
+            )