igel-qe-core 1.0.0

package/knowledge_base/memory/cognee_client.py

@@ -0,0 +1,164 @@
+"""Cognee operational memory client — Phase 4 (with PostgreSQL fallback)."""
+from __future__ import annotations
+
+import json
+import logging
+import uuid
+from typing import Any
+
+from knowledge_base.config import cfg
+from knowledge_base.db.client import get_conn
+from knowledge_base.memory.memory_types import MemoryItem, MemoryType
+
+logger = logging.getLogger(__name__)
+
+_client: "CogneeClient | None" = None
+
+_MEMORY_TABLE_SQL = """
+CREATE TABLE IF NOT EXISTS operational_memory (
+    id              UUID PRIMARY KEY DEFAULT uuid_generate_v4(),
+    memory_type     TEXT NOT NULL,
+    content         TEXT NOT NULL,
+    feature_area    TEXT NOT NULL DEFAULT 'Unknown',
+    metadata        JSONB DEFAULT '{}'::jsonb,
+    curator_id      TEXT,
+    promotion_id    UUID,
+    created_at      TIMESTAMP DEFAULT NOW()
+);
+CREATE INDEX IF NOT EXISTS idx_op_memory_feature ON operational_memory(feature_area);
+CREATE INDEX IF NOT EXISTS idx_op_memory_type ON operational_memory(memory_type);
+"""
+
+
+class CogneeClient:
+    """
+    Operational memory client.
+    Uses Cognee OSS when COGNEE_ENABLED and package installed; otherwise PostgreSQL store.
+    """
+
+    def __init__(self) -> None:
+        self._cognee = None
+        self._use_cognee = False
+        self._ensure_table()
+        if cfg.COGNEE_ENABLED:
+            try:
+                import cognee  # noqa: F401
+                self._cognee = cognee
+                self._use_cognee = True
+                logger.info("Cognee: using cognee OSS backend")
+            except ImportError:
+                logger.info("Cognee: package not installed — using PostgreSQL fallback")
+
+    def _ensure_table(self) -> None:
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute(_MEMORY_TABLE_SQL)
+
+    @property
+    def available(self) -> bool:
+        return True
+
+    def health_check(self) -> dict[str, Any]:
+        return {
+            "status": "healthy",
+            "backend": "cognee" if self._use_cognee else "postgresql_fallback",
+        }
+
+    def add_memory(
+        self,
+        content: str,
+        memory_type: MemoryType | str,
+        feature_area: str = "Unknown",
+        metadata: dict | None = None,
+        curator_id: str | None = None,
+        promotion_id: str | None = None,
+    ) -> str:
+        """Add curated memory item (promotion gate enforced at ingest layer)."""
+        mtype = memory_type.value if isinstance(memory_type, MemoryType) else str(memory_type)
+        mem_id = str(uuid.uuid4())
+
+        if self._use_cognee and self._cognee:
+            try:
+                self._cognee.add(content, metadata={
+                    "memory_type": mtype,
+                    "feature_area": feature_area,
+                    **(metadata or {}),
+                })
+            except Exception as e:
+                logger.warning("Cognee add failed, using PG fallback: %s", e)
+
+        sql = """
+        INSERT INTO operational_memory (id, memory_type, content, feature_area, metadata, curator_id, promotion_id)
+        VALUES (%s::uuid, %s, %s, %s, %s::jsonb, %s, %s::uuid)
+        """
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute(sql, (
+                    mem_id, mtype, content, feature_area,
+                    json.dumps(metadata or {}),
+                    curator_id,
+                    promotion_id,
+                ))
+        return mem_id
+
+    def search(
+        self,
+        query: str,
+        memory_types: list[str] | None = None,
+        feature_area: str | None = None,
+        top_k: int = 5,
+        min_relevance: float = 0.5,
+    ) -> list[MemoryItem]:
+        """Search operational memory with strict feature_area isolation."""
+        clauses = ["content ILIKE %s OR feature_area ILIKE %s"]
+        params: list[Any] = [f"%{query[:100]}%", f"%{query[:50]}%"]
+
+        if memory_types:
+            clauses.append("memory_type = ANY(%s)")
+            params.append(memory_types)
+        if feature_area and feature_area != "Unknown":
+            clauses.append("feature_area ILIKE %s")
+            params.append(f"%{feature_area}%")
+
+        sql = f"""
+        SELECT id::text, memory_type, content, feature_area, metadata, curator_id, created_at
+        FROM operational_memory
+        WHERE ({clauses[0]})
+        {" AND " + " AND ".join(clauses[1:]) if len(clauses) > 1 else ""}
+        ORDER BY created_at DESC
+        LIMIT %s
+        """
+        params.append(top_k)
+
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute(sql, tuple(params))
+                rows = cur.fetchall()
+
+        items = []
+        for row in rows:
+            score = 0.8 if feature_area and feature_area.lower() in (row[3] or "").lower() else 0.6
+            if score < min_relevance:
+                continue
+            try:
+                mtype = MemoryType(row[1])
+            except ValueError:
+                mtype = MemoryType.PATTERN
+            items.append(MemoryItem(
+                memory_id=row[0],
+                memory_type=mtype,
+                content=row[2],
+                feature_area=row[3] or "Unknown",
+                relevance_score=score,
+                applied_at=row[6].isoformat() if row[6] else None,
+                curator=row[5],
+                metadata=row[4] if isinstance(row[4], dict) else {},
+            ))
+        return items
+
+
+def get_cognee_client() -> CogneeClient:
+    global _client
+    if _client is None:
+        _client = CogneeClient()
+    return _client

package/knowledge_base/memory/ingest_memory.py

@@ -0,0 +1,60 @@
+"""Ingest curated feedback promotions into operational memory — Phase 4."""
+from __future__ import annotations
+
+import logging
+
+from knowledge_base.db.feedback_schema import list_promotions
+from knowledge_base.memory.cognee_client import get_cognee_client
+from knowledge_base.memory.memory_types import MemoryType
+
+logger = logging.getLogger(__name__)
+
+
+def ingest_curated_feedback(*, limit: int = 50, dry_run: bool = False) -> dict:
+    """
+    Ingest only APPROVED+applied feedback from feedback_promotions.
+    Cognee NEVER receives raw feedback_events — curation is mandatory.
+    """
+    promotions = list_promotions(status="applied", limit=limit)
+    client = get_cognee_client()
+    ingested = 0
+    skipped = 0
+
+    for promo in promotions:
+        content = promo.get("curator_notes") or ""
+        promoted = promo.get("promoted_content") or {}
+        if isinstance(promoted, dict):
+            content = content or promoted.get("curator_notes") or str(promoted)
+
+        if not content.strip():
+            skipped += 1
+            continue
+
+        feature_area = _infer_feature(promo.get("query", ""))
+        if dry_run:
+            ingested += 1
+            continue
+
+        client.add_memory(
+            content=content,
+            memory_type=MemoryType.FEEDBACK,
+            feature_area=feature_area,
+            metadata={"feedback_event_id": promo["feedback_event_id"], "promotion_id": promo["id"]},
+            curator_id=promo.get("curator_id"),
+            promotion_id=promo["id"],
+        )
+        ingested += 1
+
+    logger.info("Memory ingest: %d ingested, %d skipped", ingested, skipped)
+    return {"ingested": ingested, "skipped": skipped, "dry_run": dry_run}
+
+
+def _infer_feature(query: str) -> str:
+    q = query.lower()
+    if "entra" in q or "sso" in q:
+        return "Entra ID SSO"
+    if "wifi" in q:
+        return "Network WiFi 802.1x"
+    if "icg" in q or "cert" in q:
+        return "ICG Certificate Management"
+    return "IGEL OS General"

package/knowledge_base/memory/memory_types.py

@@ -0,0 +1,23 @@
+"""Memory category definitions — Phase 4."""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+
+class MemoryType(str, Enum):
+    FEEDBACK = "FeedbackMemory"
+    EXECUTION = "ExecutionMemory"
+    PATTERN = "PatternMemory"
+
+
+@dataclass
+class MemoryItem:
+    memory_id: str
+    memory_type: MemoryType
+    content: str
+    feature_area: str
+    relevance_score: float = 0.0
+    applied_at: str | None = None
+    curator: str | None = None
+    metadata: dict | None = None

package/knowledge_base/models/test_requirement.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+from typing import List, Optional, Dict, Any, Literal
+from pydantic import BaseModel, Field
+from datetime import datetime
+
+# ==========================================
+# 1. Requirement Understanding
+# ==========================================
+class RequirementUnderstanding(BaseModel):
+    """AI extracted metadata from a Jira Requirement."""
+    actors: List[str] = Field(default_factory=list, description="Users, systems, or roles involved.")
+    entities: List[str] = Field(default_factory=list, description="Core business objects (e.g., 'Device', 'Profile').")
+    business_rules: List[str] = Field(default_factory=list, description="Explicit rules stated in the requirement.")
+    validations: List[str] = Field(default_factory=list, description="Required data or state validations.")
+    constraints: List[str] = Field(default_factory=list, description="Technical or business limitations.")
+    dependencies: List[str] = Field(default_factory=list, description="Pre-requisites or downstream impacts.")
+
+# ==========================================
+# 2. Xray Core Models (Requirement -> Scenario)
+# ==========================================
+class TestStep(BaseModel):
+    """Represents a single step in an Xray Test (IGEL Scenario)."""
+    step_number: int
+    action: str
+    test_data: Optional[str] = None
+    expected_result: Optional[str] = None
+
+class TestRequirement(BaseModel):
+    """Represents an Xray Test Issue (IGEL Test Requirement)."""
+    jira_key: str
+    summary: str
+    description: Optional[str] = None
+    labels: List[str] = Field(default_factory=list)
+    components: List[str] = Field(default_factory=list)
+    steps: List[TestStep] = Field(default_factory=list)
+    understanding: Optional[RequirementUnderstanding] = None
+
+# ==========================================
+# 3. Business Rules Layer
+# ==========================================
+class BusinessRule(BaseModel):
+    """Generated business rule covering a specific aspect of a scenario."""
+    rule_type: Literal["positive", "negative", "edge"]
+    description: str
+
+class ScenarioWithRules(BaseModel):
+    """A scenario mapped to its exhaustive business rules."""
+    scenario: TestStep
+    business_rules: List[BusinessRule]
+
+# ==========================================
+# 4. Generated Artifacts
+# ==========================================
+class ManualTestStep(BaseModel):
+    """A detailed step for a manual test case execution."""
+    step_number: int
+    action: str
+    data: str
+    expected_result: str
+
+class GeneratedTestCase(BaseModel):
+    """A concrete test case generated from a business rule."""
+    tc_reference: str
+    business_rule: BusinessRule
+    title: str
+    test_type: Literal["positive", "negative", "edge"]
+    priority: Literal["critical", "high", "medium", "low"]
+    preconditions: List[str] = Field(default_factory=list)
+    steps: List[ManualTestStep] = Field(default_factory=list)
+    
+class ConfidenceMetrics(BaseModel):
+    """Deterministic hybrid confidence score breakdown."""
+    coverage_score: float = 0.0
+    traceability_score: float = 0.0
+    completeness_score: float = 0.0
+    historical_similarity_score: float = 0.0
+    automation_feasibility_score: float = 0.0
+    total_score: float = 0.0
+    
+class TestCaseWithConfidence(BaseModel):
+    """Test case bundled with its validation and confidence metrics."""
+    test_case: GeneratedTestCase
+    confidence: ConfidenceMetrics
+    is_approved: bool = False

package/knowledge_base/parsers/csv_parser.py

@@ -0,0 +1,126 @@
+"""
+CSV / Excel Parser
+Strategy:
+  - Each sheet = one logical document group
+  - Rows are grouped into chunks by logical context (not arbitrary row counts)
+  - Column headers are always preserved in every chunk (critical for context)
+  - Defect/Jira CSVs: group by module/component column if present
+"""
+from __future__ import annotations
+import re
+from pathlib import Path
+from knowledge_base.parsers.markdown_parser import Section
+
+
+_GROUP_COLUMNS = ["module", "component", "area", "feature", "category", "type", "status"]
+_ROWS_PER_CHUNK = 20   # max rows per chunk when no grouping column found
+
+
+def parse_csv(file_path: Path) -> list[Section]:
+    import pandas as pd
+    suffix = file_path.suffix.lower()
+
+    if suffix in (".xlsx", ".xls"):
+        xl = pd.ExcelFile(str(file_path))
+        sections: list[Section] = []
+        for sheet_name in xl.sheet_names:
+            df = xl.parse(sheet_name)
+            sections.extend(_dataframe_to_sections(df, f"{file_path.stem} - {sheet_name}"))
+        return sections
+
+    elif suffix == ".csv":
+        try:
+            df = pd.read_csv(str(file_path), encoding="utf-8")
+        except UnicodeDecodeError:
+            df = pd.read_csv(str(file_path), encoding="latin-1")
+        return _dataframe_to_sections(df, file_path.stem)
+
+    raise ValueError(f"Unsupported tabular type: {suffix}")
+
+
+def _dataframe_to_sections(df, source_name: str) -> list[Section]:
+    import pandas as pd
+
+    # Clean up: drop fully empty rows/columns
+    df = df.dropna(how="all").dropna(axis=1, how="all")
+    df = df.fillna("")
+    df.columns = [str(c).strip() for c in df.columns]
+
+    if df.empty:
+        return []
+
+    # Find grouping column
+    group_col = _find_group_column(df.columns.tolist())
+
+    if group_col:
+        return _group_by_column(df, group_col, source_name)
+    else:
+        return _chunk_by_rows(df, source_name)
+
+
+def _find_group_column(columns: list[str]) -> str | None:
+    col_lower = {c.lower(): c for c in columns}
+    for key in _GROUP_COLUMNS:
+        if key in col_lower:
+            return col_lower[key]
+    return None
+
+
+def _group_by_column(df, group_col: str, source_name: str) -> list[Section]:
+    sections: list[Section] = []
+    headers = df.columns.tolist()
+    header_line = "| " + " | ".join(headers) + " |"
+    divider = "| " + " | ".join(["---"] * len(headers)) + " |"
+
+    for group_val, group_df in df.groupby(group_col, sort=False):
+        if group_df.empty:
+            continue
+        rows_md = _rows_to_markdown(group_df, headers)
+        content = f"{header_line}\n{divider}\n{rows_md}"
+        sections.append(Section(
+            title=f"{source_name} - {group_col}: {group_val}",
+            level=2,
+            content=content,
+            metadata={
+                "is_table": True,
+                "row_count": len(group_df),
+                "group_by": group_col,
+                "group_value": str(group_val),
+            },
+        ))
+
+    return sections
+
+
+def _chunk_by_rows(df, source_name: str) -> list[Section]:
+    sections: list[Section] = []
+    headers = df.columns.tolist()
+    header_line = "| " + " | ".join(headers) + " |"
+    divider = "| " + " | ".join(["---"] * len(headers)) + " |"
+
+    for start in range(0, len(df), _ROWS_PER_CHUNK):
+        chunk_df = df.iloc[start:start + _ROWS_PER_CHUNK]
+        rows_md = _rows_to_markdown(chunk_df, headers)
+        content = f"{header_line}\n{divider}\n{rows_md}"
+        end = min(start + _ROWS_PER_CHUNK, len(df))
+        sections.append(Section(
+            title=f"{source_name} (rows {start+1}-{end})",
+            level=2,
+            content=content,
+            metadata={
+                "is_table": True,
+                "row_count": len(chunk_df),
+                "row_start": start + 1,
+                "row_end": end,
+            },
+        ))
+
+    return sections
+
+
+def _rows_to_markdown(df, headers: list[str]) -> str:
+    lines = []
+    for _, row in df.iterrows():
+        cells = [str(row[col]).strip().replace("\n", " ") for col in headers]
+        lines.append("| " + " | ".join(cells) + " |")
+    return "\n".join(lines)

package/knowledge_base/parsers/doc_parser.py

@@ -0,0 +1,176 @@
+"""
+DOC / DOCX Parser
+Strategy:
+  .docx → python-docx (best fidelity, preserves headings/tables)
+  .doc  → mammoth → fallback → win32com (Windows COM via installed MS Word)
+"""
+from __future__ import annotations
+import re
+from pathlib import Path
+from knowledge_base.parsers.markdown_parser import Section
+
+
+def parse_doc(file_path: Path) -> list[Section]:
+    suffix = file_path.suffix.lower()
+    if suffix == ".docx":
+        return _parse_docx(file_path)
+    elif suffix == ".doc":
+        return _parse_doc_binary(file_path)
+    raise ValueError(f"Unsupported doc type: {suffix}")
+
+
+# ── DOCX ─────────────────────────────────────────────────────────────────────
+
+def _parse_docx(path: Path) -> list[Section]:
+    from docx import Document
+    doc = Document(str(path))
+
+    sections: list[Section] = []
+    current_title = path.stem
+    current_level = 0
+    current_paragraphs: list[str] = []
+
+    for para in doc.paragraphs:
+        style_name = para.style.name if para.style else ""
+        text = para.text.strip()
+
+        if not text:
+            continue
+
+        # Detect headings by style
+        heading_match = re.match(r"Heading (\d+)", style_name)
+        if heading_match:
+            # Flush current section
+            content = "\n\n".join(current_paragraphs).strip()
+            if content:
+                sections.append(Section(
+                    title=current_title,
+                    level=current_level,
+                    content=content,
+                ))
+            current_title = text
+            current_level = int(heading_match.group(1))
+            current_paragraphs = []
+        else:
+            current_paragraphs.append(text)
+
+    # Flush last section
+    content = "\n\n".join(current_paragraphs).strip()
+    if content:
+        sections.append(Section(title=current_title, level=current_level, content=content))
+
+    # Extract tables and append as extra sections
+    for i, table in enumerate(doc.tables):
+        md_table = _docx_table_to_markdown(table)
+        if md_table:
+            sections.append(Section(
+                title=f"Table {i+1}",
+                level=3,
+                content=md_table,
+                metadata={"is_table": True},
+            ))
+
+    return sections if sections else [Section(title=path.stem, level=0, content=_docx_full_text(doc))]
+
+
+def _docx_full_text(doc) -> str:
+    return "\n\n".join(p.text.strip() for p in doc.paragraphs if p.text.strip())
+
+
+def _docx_table_to_markdown(table) -> str:
+    rows = []
+    for row in table.rows:
+        cells = [cell.text.strip().replace("\n", " ") for cell in row.cells]
+        rows.append(cells)
+    if not rows:
+        return ""
+    cols = max(len(r) for r in rows)
+    rows = [r + [""] * (cols - len(r)) for r in rows]
+    header = "| " + " | ".join(rows[0]) + " |"
+    divider = "| " + " | ".join(["---"] * cols) + " |"
+    body = ["| " + " | ".join(r) + " |" for r in rows[1:]]
+    return "\n".join([header, divider] + body)
+
+
+# ── DOC (binary) ──────────────────────────────────────────────────────────────
+
+def _parse_doc_binary(path: Path) -> list[Section]:
+    text = ""
+
+    # Attempt 1: mammoth (handles some .doc files)
+    try:
+        import mammoth
+        with open(str(path), "rb") as f:
+            result = mammoth.extract_raw_text(f)
+        text = result.value.strip()
+        if len(text) > 200:
+            return _text_to_sections(text, path.stem)
+    except Exception:
+        pass
+
+    # Attempt 2: Windows COM (requires MS Word installed)
+    try:
+        import win32com.client
+        import pythoncom
+        pythoncom.CoInitialize()
+        word = win32com.client.Dispatch("Word.Application")
+        word.Visible = False
+        doc_obj = word.Documents.Open(str(path.resolve()))
+        text = doc_obj.Content.Text.strip()
+        doc_obj.Close(False)
+        word.Quit()
+        if text:
+            return _text_to_sections(text, path.stem)
+    except Exception:
+        pass
+
+    # Attempt 3: read raw bytes and extract printable text (last resort)
+    try:
+        raw = path.read_bytes()
+        # DOC files store text in the Word Binary Format — extract ASCII runs
+        text = _extract_text_from_binary(raw)
+        if len(text) > 100:
+            return _text_to_sections(text, path.stem)
+    except Exception:
+        pass
+
+    raise RuntimeError(f"Could not parse .doc file: {path.name}. "
+                       "Try converting to .docx using Microsoft Word or LibreOffice first.")
+
+
+def _extract_text_from_binary(raw: bytes) -> str:
+    """Extract readable text runs from DOC binary format (crude but works for plain content)."""
+    import re
+    # Extract ASCII sequences of 4+ printable chars
+    chunks = re.findall(rb"[\x20-\x7E]{4,}", raw)
+    text = " ".join(c.decode("ascii", errors="ignore") for c in chunks)
+    # Remove excessive whitespace
+    text = re.sub(r"\s{3,}", "\n\n", text)
+    return text.strip()
+
+
+def _text_to_sections(text: str, source_name: str) -> list[Section]:
+    """Convert flat text into sections by detecting paragraph boundaries."""
+    paragraphs = [p.strip() for p in re.split(r"\n{2,}", text) if p.strip()]
+    sections: list[Section] = []
+    current_title = source_name
+    current_paras: list[str] = []
+
+    for para in paragraphs:
+        # Heuristic: short line that looks like a heading
+        if len(para) < 80 and not para.endswith(".") and para[0].isupper():
+            if current_paras:
+                sections.append(Section(
+                    title=current_title,
+                    level=2,
+                    content="\n\n".join(current_paras),
+                ))
+            current_title = para
+            current_paras = []
+        else:
+            current_paras.append(para)
+
+    if current_paras:
+        sections.append(Section(title=current_title, level=2, content="\n\n".join(current_paras)))
+
+    return sections if sections else [Section(title=source_name, level=0, content=text)]