igel-qe-core 1.0.3 → 1.0.7

package/src/project/init_check.py

@@ -0,0 +1,214 @@
+"""init_check.py — Pre-flight connectivity checker for igel-qe init.
+
+Failure semantics:
+  REQUIRED (hard block — init fails if these are down):
+    - PostgreSQL + pgvector extension
+    - Azure OpenAI (embedding + LLM endpoint)
+
+  OPTIONAL (soft warn — init continues; integration is just disabled):
+    - Jira
+    - Confluence
+    - Bitbucket
+    - Xray
+
+Security:
+  - Error messages are sanitised before output: auth headers, tokens, passwords
+    are stripped from any exception text and replaced with [REDACTED].
+  - The summary JSON never echoes back credential values.
+
+Output: single JSON object to stdout, suitable for Node.js IPC.
+"""
+from __future__ import annotations
+
+import json
+import re
+import sys
+import time
+from pathlib import Path
+
+_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
+if str(_PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(_PROJECT_ROOT))
+
+from src.config import cfg  # noqa: E402
+
+# ── secret sanitiser ──────────────────────────────────────────────────────────
+
+_SECRET_PATTERNS = [
+    re.compile(r"(Authorization:\s*(?:Bearer|Basic)\s+)\S+", re.IGNORECASE),
+    re.compile(r"(api[_-]?key\s*=\s*)[^\s,&]+", re.IGNORECASE),
+    re.compile(r"(token\s*=\s*)[^\s,&]+", re.IGNORECASE),
+    re.compile(r"(password\s*=\s*)[^\s,&]+", re.IGNORECASE),
+    re.compile(r"(secret\s*=\s*)[^\s,&]+", re.IGNORECASE),
+]
+
+
+def _sanitise(text: str) -> str:
+    for pattern in _SECRET_PATTERNS:
+        text = pattern.sub(r"\g<1>[REDACTED]", text)
+    # Also strip any value that looks like a JWT or long base64 token
+    text = re.sub(r"ey[A-Za-z0-9_\-]{20,}\.[A-Za-z0-9_\-]+\.[A-Za-z0-9_\-]+", "[REDACTED_JWT]", text)
+    return text
+
+
+def _safe_error(exc: Exception) -> str:
+    return _sanitise(str(exc))
+
+
+# ── individual checks ─────────────────────────────────────────────────────────
+
+def _check_postgres() -> dict:
+    """Required: PostgreSQL connection + pgvector extension."""
+    t0 = time.perf_counter()
+    try:
+        from src.db.client import get_conn
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                cur.execute("SELECT 1")
+                cur.execute("SELECT extname FROM pg_extension WHERE extname = 'vector'")
+                has_vector = cur.fetchone() is not None
+        ms = round((time.perf_counter() - t0) * 1000)
+        if not has_vector:
+            return {"status": "fail", "message": "pgvector extension not installed. Run: CREATE EXTENSION vector;", "ms": ms}
+        return {"status": "ok", "message": "PostgreSQL + pgvector ready", "ms": ms}
+    except Exception as e:
+        return {"status": "fail", "message": _safe_error(e), "ms": round((time.perf_counter() - t0) * 1000)}
+
+
+def _check_azure_openai() -> dict:
+    """Required: Azure OpenAI endpoint reachable with valid credentials."""
+    t0 = time.perf_counter()
+    if not cfg.AZURE_API_KEY or not cfg.AZURE_ENDPOINT:
+        return {"status": "fail", "message": "AZURE_API_KEY or AZURE_ENDPOINT not set", "ms": 0}
+    try:
+        import httpx
+        with httpx.Client(timeout=10.0) as client:
+            resp = client.get(
+                f"{cfg.AZURE_ENDPOINT.rstrip('/')}/openai/models?api-version={cfg.AZURE_API_VERSION}",
+                headers={"api-key": cfg.AZURE_API_KEY},
+            )
+            # 200 or 401 both mean the endpoint is reachable; 401 means bad key
+            if resp.status_code == 401:
+                return {"status": "fail", "message": "Azure OpenAI credentials rejected (401)", "ms": round((time.perf_counter() - t0) * 1000)}
+            resp.raise_for_status()
+        return {"status": "ok", "message": "Azure OpenAI endpoint reachable", "ms": round((time.perf_counter() - t0) * 1000)}
+    except Exception as e:
+        return {"status": "fail", "message": _safe_error(e), "ms": round((time.perf_counter() - t0) * 1000)}
+
+
+def _check_jira() -> dict:
+    """Optional: Jira connectivity check."""
+    t0 = time.perf_counter()
+    if not cfg.JIRA_ENABLED:
+        return {"status": "skipped", "message": "JIRA_ENABLED=false", "ms": 0}
+    try:
+        from src.connectors.jira_connector import JiraConnector
+        c = JiraConnector()
+        ok = c.connect()
+        ms = round((time.perf_counter() - t0) * 1000)
+        return {"status": "ok" if ok else "warn", "message": "Jira connected" if ok else "Jira connection failed — check credentials", "ms": ms}
+    except Exception as e:
+        return {"status": "warn", "message": _safe_error(e), "ms": round((time.perf_counter() - t0) * 1000)}
+
+
+def _check_confluence() -> dict:
+    """Optional: Confluence connectivity check."""
+    t0 = time.perf_counter()
+    if not cfg.CONFLUENCE_ENABLED:
+        return {"status": "skipped", "message": "CONFLUENCE_ENABLED=false", "ms": 0}
+    try:
+        from src.connectors.confluence_connector import ConfluenceConnector
+        c = ConfluenceConnector()
+        ok = c.connect()
+        ms = round((time.perf_counter() - t0) * 1000)
+        return {"status": "ok" if ok else "warn", "message": "Confluence connected" if ok else "Confluence connection failed", "ms": ms}
+    except Exception as e:
+        return {"status": "warn", "message": _safe_error(e), "ms": round((time.perf_counter() - t0) * 1000)}
+
+
+def _check_bitbucket() -> dict:
+    """Optional: Bitbucket connectivity check."""
+    t0 = time.perf_counter()
+    if not cfg.BITBUCKET_ENABLED:
+        return {"status": "skipped", "message": "BITBUCKET_ENABLED=false", "ms": 0}
+    try:
+        from src.connectors.bitbucket_connector import BitbucketConnector
+        c = BitbucketConnector()
+        ok = c.connect()
+        ms = round((time.perf_counter() - t0) * 1000)
+        return {"status": "ok" if ok else "warn", "message": "Bitbucket connected" if ok else "Bitbucket connection failed", "ms": ms}
+    except Exception as e:
+        return {"status": "warn", "message": _safe_error(e), "ms": round((time.perf_counter() - t0) * 1000)}
+
+
+def _check_xray() -> dict:
+    """Optional: Xray connectivity check."""
+    t0 = time.perf_counter()
+    if not cfg.XRAY_ENABLED:
+        return {"status": "skipped", "message": "XRAY_ENABLED=false", "ms": 0}
+    try:
+        from src.connectors.xray_connector import XrayConnector
+        c = XrayConnector()
+        ok = c.connect()
+        ms = round((time.perf_counter() - t0) * 1000)
+        return {"status": "ok" if ok else "warn", "message": "Xray connected" if ok else "Xray authentication failed", "ms": ms}
+    except Exception as e:
+        return {"status": "warn", "message": _safe_error(e), "ms": round((time.perf_counter() - t0) * 1000)}
+
+
+# ── runner ────────────────────────────────────────────────────────────────────
+
+def run_checks() -> dict:
+    """Run all checks and return a structured summary.
+
+    Returns:
+        {
+          "ready": bool,         # True only if all REQUIRED checks pass
+          "checks": { ... },     # per-check results
+          "required_failed": [],  # list of failed required check names
+          "warnings": [],        # list of optional check names that warned
+        }
+    """
+    required = {
+        "postgres": _check_postgres,
+        "azure_openai": _check_azure_openai,
+    }
+    optional = {
+        "jira": _check_jira,
+        "confluence": _check_confluence,
+        "bitbucket": _check_bitbucket,
+        "xray": _check_xray,
+    }
+
+    results: dict[str, dict] = {}
+    required_failed: list[str] = []
+    warnings: list[str] = []
+
+    for name, fn in required.items():
+        result = fn()
+        results[name] = result
+        if result["status"] == "fail":
+            required_failed.append(name)
+
+    for name, fn in optional.items():
+        result = fn()
+        results[name] = result
+        if result["status"] == "warn":
+            warnings.append(name)
+
+    return {
+        "ready": len(required_failed) == 0,
+        "checks": results,
+        "required_failed": required_failed,
+        "warnings": warnings,
+    }
+
+
+def main() -> None:
+    summary = run_checks()
+    print(json.dumps(summary, indent=2))
+    sys.exit(0 if summary["ready"] else 1)
+
+
+if __name__ == "__main__":
+    main()

package/src/prompts/automation_design.md

@@ -0,0 +1,12 @@
+# Automation Design Prompt
+
+{{include:policies/guardrails.md}}
+{{include:policies/automation_generation_policy.md}}
+
+You are the IGEL Test Script Planning specialist.
+Your job is to design an automation plan from approved test cases and discovered assets.
+
+Rules:
+- Planning only. No code generation.
+- Select only from provided available asset lists.
+- Output MUST be valid JSON with keys: page_objects, fixtures, utilities.

package/src/prompts/policies/automation_generation_policy.md

@@ -0,0 +1,11 @@
+# Automation Generation Policy
+
+- Maximize reuse in this order:
+  1. Existing business components
+  2. Existing APIs
+  3. Existing fixtures
+  4. Existing utilities
+  5. Existing page objects
+- Never modify working framework assets without explicit approval.
+- Keep script placement under tests/<category> and respect existing layer boundaries.
+- Avoid hallucinated imports or utility calls.

package/src/prompts/policies/generation_instructions.md

@@ -0,0 +1,24 @@
+# Generation Instructions
+
+Purpose: enforce enterprise-grade detailedness and executability.
+
+Detailedness rules:
+- Assume executor has no IGEL or domain background.
+- Outputs must be self-explanatory and directly actionable.
+
+Test case rules:
+- Cover positive, negative, and edge scenarios.
+- Include objective, preconditions, dependencies, environment, data, explicit steps, expected result per step, cleanup, and post-conditions.
+
+Step quality rules:
+- One clear action per step.
+- One explicit expected result per step.
+- Avoid vague verbs like verify/check/open/navigate when they hide execution detail.
+
+Token and context rules:
+- Prefer context.md guidance before deep file traversal.
+- Minimize unnecessary repo scanning.
+
+Failure policy:
+- Do not hallucinate missing context.
+- Report context gaps and request clarification or SME review.

package/src/prompts/policies/guardrails.md

@@ -0,0 +1,6 @@
+# Guardrails
+
+- IGEL OS12 only. Do not reference OS11.
+- Do not invent APIs, fixtures, page objects, utilities, or behavior not grounded in supplied context.
+- Preserve business intent and traceability from requirement -> rule -> testcase -> script.
+- When constraints conflict, document conflict and require approval.

package/src/prompts/policies/traceability_policy.md

@@ -0,0 +1,9 @@
+# Traceability Policy
+
+Every generated artifact must maintain links to:
+- Requirement key
+- Scenario / business rule source
+- Test case identifier
+- Referenced framework assets
+
+Output should clearly indicate source assumptions, risks, and review recommendation where confidence is low.

package/src/prompts/prompt_loader.py

@@ -0,0 +1,56 @@
+"""Prompt loader with lightweight include support.
+
+Supported include syntax inside prompt files:
+  {{include:relative/path/from/src/prompts.md}}
+"""
+from __future__ import annotations
+
+import logging
+import re
+from functools import lru_cache
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_PROMPTS_ROOT = Path(__file__).resolve().parent
+_INCLUDE_RE = re.compile(r"\{\{include:([^}]+)\}\}")
+
+
+def _resolve_with_includes(path: Path, seen: set[Path], depth: int = 0) -> str:
+    if depth > 8:
+        raise ValueError(f"Prompt include depth too large for {path}")
+
+    full = path.resolve()
+    if full in seen:
+        raise ValueError(f"Circular prompt include detected at {full}")
+    seen.add(full)
+
+    text = full.read_text(encoding="utf-8")
+
+    def _replace(match: re.Match) -> str:
+        rel = match.group(1).strip()
+        include_path = (_PROMPTS_ROOT / rel).resolve()
+        if not include_path.exists() or not include_path.is_file():
+            raise FileNotFoundError(f"Prompt include not found: {rel}")
+        return _resolve_with_includes(include_path, seen=set(seen), depth=depth + 1)
+
+    return _INCLUDE_RE.sub(_replace, text)
+
+
+@lru_cache(maxsize=64)
+def load_prompt(rel_path: str, fallback: str = "") -> str:
+    """Load a prompt from src/prompts with include expansion.
+
+    Args:
+        rel_path: Path relative to src/prompts (e.g. "test_case_generator.md").
+        fallback: Returned when file cannot be loaded.
+    """
+    try:
+        path = (_PROMPTS_ROOT / rel_path).resolve()
+        if not path.exists() or not path.is_file():
+            logger.warning("Prompt file missing: %s", rel_path)
+            return fallback
+        return _resolve_with_includes(path, seen=set())
+    except Exception as exc:
+        logger.warning("Failed to load prompt '%s': %s", rel_path, exc)
+        return fallback

package/src/prompts/requirement_understanding.md

@@ -0,0 +1,17 @@
+# Requirement Understanding Prompt
+
+{{include:policies/guardrails.md}}
+{{include:policies/generation_instructions.md}}
+
+You are the IGEL Test Case Planning specialist.
+Analyze a Jira/Xray requirement and extract normalized semantic understanding.
+
+Output MUST be valid JSON:
+{
+	"actors": ["string"],
+	"entities": ["string"],
+	"business_rules": ["string"],
+	"validations": ["string"],
+	"constraints": ["string"],
+	"dependencies": ["string"]
+}

package/src/prompts/scenario_generator.md

@@ -0,0 +1,17 @@
+# Scenario Generator Prompt
+
+{{include:policies/guardrails.md}}
+{{include:policies/generation_instructions.md}}
+
+Generate exhaustive business rules for a single scenario step.
+Include positive, negative, and edge rule candidates.
+
+Output MUST be valid JSON:
+{
+	"business_rules": [
+		{
+			"rule_type": "positive" | "negative" | "edge",
+			"description": "string"
+		}
+	]
+}

package/src/prompts/script_generator.md

@@ -0,0 +1,13 @@
+# Script Generator Prompt
+
+{{include:policies/guardrails.md}}
+{{include:policies/automation_generation_policy.md}}
+{{include:policies/generation_instructions.md}}
+{{include:policies/traceability_policy.md}}
+
+You generate enterprise-grade pytest automation scripts from approved test cases and automation plan.
+
+Hard constraints:
+- Use ONLY fixtures/page objects/utilities provided in the automation plan.
+- Do not hallucinate imports or helpers.
+- Produce valid Python 3 code.

package/src/prompts/test_case_generator.md

@@ -0,0 +1,23 @@
+# Test Case Generator Prompt
+
+{{include:policies/guardrails.md}}
+{{include:policies/generation_instructions.md}}
+{{include:policies/traceability_policy.md}}
+
+Generate a detailed manual test case for the supplied business rule and scenario context.
+Do not invent product behavior outside provided context.
+
+Output MUST be valid JSON:
+{
+	"title": "string",
+	"priority": "critical" | "high" | "medium" | "low",
+	"preconditions": ["string"],
+	"steps": [
+		{
+			"step_number": 1,
+			"action": "string",
+			"data": "string",
+			"expected_result": "string"
+		}
+	]
+}

package/{knowledge_base → src}/retrieval/context_fusion.py

@@ -5,15 +5,15 @@ from __future__ import annotations
 import logging
 from typing import TYPE_CHECKING
 
-from knowledge_base.config import cfg
-from knowledge_base.core.precedence import (
+from src.config import cfg
+from src.core.precedence import (
     apply_precedence_filter,
     detect_precedence_conflicts,
     infer_source_type,
 )
 
 if TYPE_CHECKING:
-    from knowledge_base.retrieval.retriever import SearchResult
+    from src.retrieval.retriever import SearchResult
 
 logger = logging.getLogger(__name__)
 

package/{knowledge_base → src}/retrieval/knowledge_graph.py

@@ -6,8 +6,8 @@ import logging
 import re
 from dataclasses import dataclass
 
-from knowledge_base.db.client import get_conn
-from knowledge_base.db.kg_introspect import get_kg_layout
+from src.db.client import get_conn
+from src.db.kg_introspect import get_kg_layout
 
 logger = logging.getLogger(__name__)
 
@@ -146,10 +146,10 @@ def retrieve_kg_context(
     Expand around nodes matching query terms (+ optional anchor parent chunks).
     Routes to Neo4j when KG_BACKEND=neo4j, else PostgreSQL.
     """
-    from knowledge_base.config import cfg
+    from src.config import cfg
 
     if cfg.KG_BACKEND == "neo4j":
-        from knowledge_base.graph_db.kg_traversal import retrieve_kg_context_neo4j
+        from src.graph_db.kg_traversal import retrieve_kg_context_neo4j
 
         return retrieve_kg_context_neo4j(
             query,

package/{knowledge_base → src}/retrieval/reranker.py

@@ -22,7 +22,7 @@ import os
 from dataclasses import dataclass, field
 from time import perf_counter
 
-from knowledge_base.config import cfg
+from src.config import cfg
 
 logger = logging.getLogger(__name__)
 
@@ -93,7 +93,7 @@ def _load_cross_encoder(model_name: str):
     except ImportError as e:
         raise ImportError(
             "sentence-transformers not available. Run: "
-            "bash knowledge_base/deploy/install_reranker.sh"
+            "bash src/deploy/install_reranker.sh"
         ) from e
 
     logger.info(

package/{knowledge_base → src}/retrieval/retriever.py

@@ -18,9 +18,9 @@ from time import perf_counter
 
 import psycopg2
 
-from knowledge_base.db.client import get_conn, init_pool
-from knowledge_base.embeddings.azure_embedder import embed_single, format_for_pgvector
-from knowledge_base.retrieval.knowledge_graph import (
+from src.db.client import get_conn, init_pool
+from src.embeddings.azure_embedder import embed_single, format_for_pgvector
+from src.retrieval.knowledge_graph import (
     KGContext,
     classify_query_style,
     retrieve_kg_context,
@@ -99,7 +99,7 @@ def retrieve(
             if "vector" in str(e).lower():
                 logger.error(
                     "pgvector extension not available. Run: CREATE EXTENSION IF NOT EXISTS vector; "
-                    "then re-run knowledge_base.ingestion.ingest"
+                    "then re-run src.ingestion.ingest"
                 )
             else:
                 logger.error("Vector search failed: %s", e)
@@ -184,7 +184,7 @@ def _embed_query(query: str) -> list[float]:
         return embed_single(query)
     except Exception as e:
         logger.warning("Embedding failed for query, falling back to FTS only: %s", e)
-        from knowledge_base.config import cfg
+        from src.config import cfg
         return [0.0] * cfg.EMBEDDING_DIM
 
 
@@ -346,10 +346,10 @@ def retrieve_v3(
     """
     Hybrid retrieval with RRF, optional cross-encoder reranking, KG context, fusion, debug traces.
     """
-    from knowledge_base.config import cfg
-    from knowledge_base.retrieval.context_fusion import fuse_retrieval_context
-    from knowledge_base.retrieval.reranker import RerankItem, rerank_passages
-    from knowledge_base.retrieval.retrieval_debug import ChunkScoreTrace, RetrievalDebugReport
+    from src.config import cfg
+    from src.retrieval.context_fusion import fuse_retrieval_context
+    from src.retrieval.reranker import RerankItem, rerank_passages
+    from src.retrieval.retrieval_debug import ChunkScoreTrace, RetrievalDebugReport
 
     timings: dict[str, float] = {}
     debug = RetrievalDebugReport(
@@ -420,7 +420,7 @@ def retrieve_v3(
         t_web = perf_counter()
         if getattr(cfg, "WEB_RETRIEVAL_ENABLED", False):
             try:
-                from knowledge_base.retrieval.web_retriever import retrieve_web_context, format_web_block
+                from src.retrieval.web_retriever import retrieve_web_context, format_web_block
 
                 web_hits = retrieve_web_context(
                     query,
@@ -606,7 +606,7 @@ def retrieve_v3(
             local_top_score = min(1.0, (0.6 * sorted_scores[0]) + (0.4 * margin))
     if getattr(cfg, "WEB_RETRIEVAL_ENABLED", False):
         try:
-            from knowledge_base.retrieval.web_retriever import retrieve_web_context, format_web_block
+            from src.retrieval.web_retriever import retrieve_web_context, format_web_block
 
             web_hits = retrieve_web_context(
                 query,

package/{knowledge_base → src}/retrieval/unified_query.py

@@ -9,17 +9,17 @@ import random
 from dataclasses import dataclass, field
 from time import perf_counter
 
-from knowledge_base.config import cfg
-from knowledge_base.connectors.query_bridge import ExternalContext, query_mcp_sources
-from knowledge_base.retrieval.retriever import RetrievalBundle, retrieve_v3
+from src.config import cfg
+from src.connectors.query_bridge import ExternalContext, query_mcp_sources
+from src.retrieval.retriever import RetrievalBundle, retrieve_v3
 
 logger = logging.getLogger(__name__)
 
 
 def _empty_bundle(query: str, *, reason: str) -> RetrievalBundle:
     """Build a safe empty fallback bundle for timeout/error paths."""
-    from knowledge_base.retrieval.knowledge_graph import KGContext
-    from knowledge_base.retrieval.retrieval_debug import RetrievalDebugReport
+    from src.retrieval.knowledge_graph import KGContext
+    from src.retrieval.retrieval_debug import RetrievalDebugReport
 
     return RetrievalBundle(
         results=[],
@@ -95,8 +95,8 @@ def _maybe_persist_trace(query: str, result: "UnifiedQueryResult") -> None:
     if rate <= 0 or random.random() > rate:
         return
     try:
-        from knowledge_base.core.request_context import get_request_id
-        from knowledge_base.db.client import get_conn
+        from src.core.request_context import get_request_id
+        from src.db.client import get_conn
         import json
         with get_conn() as conn:
             with conn.cursor() as cur:

package/{knowledge_base → src}/retrieval/web_retriever.py

@@ -9,11 +9,11 @@ from dataclasses import dataclass, field
 from time import perf_counter
 from urllib.parse import urlparse
 
-from knowledge_base.config import cfg
-from knowledge_base.connectors.search_backends import get_configured_backends
-from knowledge_base.connectors.web_fetcher import fetch_url, is_safe_fetch_target
-from knowledge_base.parsers.html_parser import parse_html_page
-from knowledge_base.retrieval.reranker import rerank_web_pages
+from src.config import cfg
+from src.connectors.search_backends import get_configured_backends
+from src.connectors.web_fetcher import fetch_url, is_safe_fetch_target
+from src.parsers.html_parser import parse_html_page
+from src.retrieval.reranker import rerank_web_pages
 
 logger = logging.getLogger(__name__)
 
@@ -136,7 +136,7 @@ async def _fetch_page_content_crawl4ai(url: str, query: str) -> tuple[str, str]:
 
 
 def _fetch_page_content_playwright(url: str) -> tuple[str, str]:
-    from knowledge_base.connectors.playwright_fetcher import fetch_page_playwright, playwright_available
+    from src.connectors.playwright_fetcher import fetch_page_playwright, playwright_available
 
     if not playwright_available():
         raise ImportError("playwright not installed")

package/{knowledge_base → src}/validators/step_fidelity.py

@@ -7,7 +7,7 @@ import re
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    from knowledge_base.generation.manual_tc_parser import ParsedStep
+    from src.generation.manual_tc_parser import ParsedStep
 
 
 _VAGUE_PHRASES = (

package/{knowledge_base → src}/vision/image_describer.py

@@ -16,7 +16,7 @@ from dataclasses import dataclass, field
 from openai import AzureOpenAI
 from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
 
-from knowledge_base.config import cfg
+from src.config import cfg
 
 logger = logging.getLogger(__name__)