igel-qe-core 1.0.3 → 1.0.7

package/src/project/context_enricher.py

@@ -0,0 +1,333 @@
+"""Framework context enrichment with AST + dependency graph persistence.
+
+This module enriches scaffolded context.md files by:
+1) Inspecting folder-local code/doc assets
+2) Building a lightweight dependency graph from Python AST imports
+3) Writing human and LLM-friendly markdown content to context.md
+4) Persisting/updating context metadata in PostgreSQL
+"""
+from __future__ import annotations
+
+import ast
+import hashlib
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from src.db.client import get_transaction
+
+
+EXCLUDED_DIRS = {
+    "node_modules",
+    ".git",
+    ".github",
+    ".vscode",
+    ".venv",
+    "__pycache__",
+    "dist",
+    "build",
+    ".idea",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".ai",
+}
+
+
+@dataclass
+class ContextEnrichResult:
+    path: str
+    folder: str
+    files_scanned: int
+    python_files: int
+    dependency_edges: int
+    key_assets: list[str]
+
+
+def _norm_rel(repo_root: Path, path: Path) -> str:
+    return str(path.resolve().relative_to(repo_root.resolve()).as_posix())
+
+
+def _folder_label(repo_root: Path, folder: Path) -> str:
+    rel = _norm_rel(repo_root, folder)
+    return "." if rel == "." else rel
+
+
+def _iter_context_files(repo_root: Path, context_paths: list[str] | None) -> list[Path]:
+    if context_paths:
+        files: list[Path] = []
+        for p in context_paths:
+            c = Path(p)
+            full = c if c.is_absolute() else (repo_root / c)
+            if full.exists() and full.is_file() and full.name == "context.md":
+                files.append(full.resolve())
+        return sorted(set(files))
+
+    found: list[Path] = []
+    for p in repo_root.rglob("context.md"):
+        rel_parts = p.relative_to(repo_root).parts
+        if any(part in EXCLUDED_DIRS or part.startswith(".") for part in rel_parts[:-1]):
+            continue
+        found.append(p.resolve())
+    return sorted(set(found))
+
+
+def _iter_folder_files(folder: Path) -> list[Path]:
+    files: list[Path] = []
+    for child in folder.iterdir():
+        if child.is_file() and child.name != "context.md":
+            files.append(child)
+    return sorted(files)
+
+
+def _extract_imports(py_file: Path) -> set[str]:
+    try:
+        text = py_file.read_text(encoding="utf-8", errors="ignore")
+        tree = ast.parse(text)
+    except Exception:
+        return set()
+
+    imports: set[str] = set()
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            for alias in node.names:
+                imports.add(alias.name)
+        elif isinstance(node, ast.ImportFrom):
+            if node.module:
+                imports.add(node.module)
+    return imports
+
+
+def _extract_assets(py_file: Path) -> list[str]:
+    assets: list[str] = []
+    try:
+        text = py_file.read_text(encoding="utf-8", errors="ignore")
+        tree = ast.parse(text)
+    except Exception:
+        return assets
+
+    for node in ast.walk(tree):
+        if isinstance(node, ast.ClassDef):
+            assets.append(f"class {node.name}")
+        elif isinstance(node, ast.FunctionDef):
+            assets.append(f"function {node.name}")
+    return assets
+
+
+def _local_module_map(repo_root: Path, files: list[Path]) -> dict[str, str]:
+    module_map: dict[str, str] = {}
+    for f in files:
+        if f.suffix != ".py":
+            continue
+        rel = _norm_rel(repo_root, f)
+        mod = rel[:-3].replace("/", ".")
+        module_map[mod] = rel
+    return module_map
+
+
+def _build_dependency_graph(repo_root: Path, files: list[Path]) -> tuple[list[dict], int]:
+    module_map = _local_module_map(repo_root, files)
+    edges: list[dict] = []
+
+    for f in files:
+        if f.suffix != ".py":
+            continue
+        src = _norm_rel(repo_root, f)
+        for imp in sorted(_extract_imports(f)):
+            target = module_map.get(imp)
+            if target is None:
+                # Match nested imports: package.mod.sub -> package.mod
+                parts = imp.split(".")
+                while len(parts) > 1 and target is None:
+                    parts = parts[:-1]
+                    target = module_map.get(".".join(parts))
+            edges.append(
+                {
+                    "source": src,
+                    "import": imp,
+                    "target": target,
+                    "local": bool(target),
+                }
+            )
+
+    return edges, sum(1 for e in edges if e["local"])
+
+
+def _render_context_markdown(
+    *,
+    repo_root: Path,
+    folder: Path,
+    files: list[Path],
+    python_files: list[Path],
+    assets: list[str],
+    dependency_edges: list[dict],
+) -> str:
+    folder_label = _folder_label(repo_root, folder)
+    rel_files = [_norm_rel(repo_root, f) for f in files]
+
+    key_assets = assets[:12]
+    file_preview = rel_files[:15]
+    local_edges = [e for e in dependency_edges if e["local"]][:15]
+
+    lines: list[str] = []
+    lines.append(f"# Context - {folder_label}")
+    lines.append("")
+    lines.append("## Purpose")
+    lines.append(
+        f"This folder owns automation assets for `{folder_label}` and is used by IGEL QE orchestration for both human review and LLM grounding."
+    )
+    lines.append("")
+    lines.append("## Folder Snapshot")
+    lines.append(f"- Total files (excluding context): {len(files)}")
+    lines.append(f"- Python files parsed with AST: {len(python_files)}")
+    lines.append(f"- Candidate key assets discovered: {len(assets)}")
+    lines.append("")
+    lines.append("## Key Assets")
+    if key_assets:
+        for item in key_assets:
+            lines.append(f"- {item}")
+    else:
+        lines.append("- No Python class/function assets detected in this folder yet.")
+    lines.append("")
+    lines.append("## Dependency Graph")
+    if local_edges:
+        for edge in local_edges:
+            lines.append(f"- {edge['source']} imports {edge['target']} (via `{edge['import']}`)")
+    else:
+        lines.append("- No local Python import edges detected from AST analysis.")
+    lines.append("")
+    lines.append("## File Inventory")
+    if file_preview:
+        for p in file_preview:
+            lines.append(f"- {p}")
+    else:
+        lines.append("- Folder currently has no files besides context.md.")
+    lines.append("")
+    lines.append("## Risks and Dependencies")
+    lines.append("- Keep import boundaries stable; changing module names can break downstream scripts.")
+    lines.append("- Update this context after adding new fixtures/page objects/utilities.")
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def _table_exists(cur, table: str) -> bool:
+    cur.execute(
+        "SELECT 1 FROM information_schema.tables WHERE table_name = %s",
+        (table,),
+    )
+    return cur.fetchone() is not None
+
+
+def _upsert_context_registry(
+    *,
+    repo_root: Path,
+    context_file: Path,
+    markdown: str,
+    files: list[Path],
+    python_files: list[Path],
+    assets: list[str],
+    dependency_edges: list[dict],
+) -> None:
+    with get_transaction() as conn:
+        with conn.cursor() as cur:
+            if not _table_exists(cur, "igel_context_registry"):
+                return
+
+            context_path = _norm_rel(repo_root, context_file)
+            folder_path = _norm_rel(repo_root, context_file.parent)
+            content_hash = hashlib.sha256(markdown.encode("utf-8")).hexdigest()
+
+            metadata = {
+                "files_scanned": len(files),
+                "python_files": len(python_files),
+                "dependency_edges": len(dependency_edges),
+                "local_dependency_edges": sum(1 for e in dependency_edges if e.get("local")),
+            }
+
+            cur.execute(
+                """
+                INSERT INTO igel_context_registry (
+                    context_path,
+                    folder_path,
+                    content,
+                    key_assets,
+                    dependency_graph,
+                    metadata,
+                    source_hash,
+                    updated_at
+                ) VALUES (%s, %s, %s, %s::jsonb, %s::jsonb, %s::jsonb, %s, NOW())
+                ON CONFLICT (context_path)
+                DO UPDATE SET
+                    folder_path = EXCLUDED.folder_path,
+                    content = EXCLUDED.content,
+                    key_assets = EXCLUDED.key_assets,
+                    dependency_graph = EXCLUDED.dependency_graph,
+                    metadata = EXCLUDED.metadata,
+                    source_hash = EXCLUDED.source_hash,
+                    updated_at = NOW()
+                """,
+                (
+                    context_path,
+                    folder_path,
+                    markdown,
+                    json.dumps(assets[:30]),
+                    json.dumps(dependency_edges[:200]),
+                    json.dumps(metadata),
+                    content_hash,
+                ),
+            )
+
+
+def enrich_context_files(
+    repo_root: Path,
+    *,
+    context_paths: list[str] | None = None,
+    rewrite_files: bool = True,
+) -> list[ContextEnrichResult]:
+    """Enrich context.md files and persist context intelligence in Postgres."""
+    repo_root = repo_root.resolve()
+    results: list[ContextEnrichResult] = []
+
+    for context_file in _iter_context_files(repo_root, context_paths):
+        folder = context_file.parent
+        files = _iter_folder_files(folder)
+        python_files = [f for f in files if f.suffix == ".py"]
+        assets: list[str] = []
+        for py_file in python_files:
+            assets.extend(_extract_assets(py_file))
+
+        dependency_edges, local_edges_count = _build_dependency_graph(repo_root, python_files)
+
+        markdown = _render_context_markdown(
+            repo_root=repo_root,
+            folder=folder,
+            files=files,
+            python_files=python_files,
+            assets=assets,
+            dependency_edges=dependency_edges,
+        )
+        if rewrite_files:
+            context_file.write_text(markdown + "\n", encoding="utf-8")
+
+        _upsert_context_registry(
+            repo_root=repo_root,
+            context_file=context_file,
+            markdown=markdown,
+            files=files,
+            python_files=python_files,
+            assets=assets,
+            dependency_edges=dependency_edges,
+        )
+
+        results.append(
+            ContextEnrichResult(
+                path=_norm_rel(repo_root, context_file),
+                folder=_norm_rel(repo_root, folder),
+                files_scanned=len(files),
+                python_files=len(python_files),
+                dependency_edges=local_edges_count,
+                key_assets=assets[:10],
+            )
+        )
+
+    return results

package/src/project/context_projection_builder.py

@@ -0,0 +1,167 @@
+"""Build .ai/context.md as a registry-backed projection.
+
+V2 intent: context.md is a human-readable cache generated from registries,
+not a source of truth.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+from src.db.client import get_conn
+
+
+@dataclass
+class ProjectionStats:
+    requirements: int = 0
+    scenarios: int = 0
+    business_rules: int = 0
+    testcases: int = 0
+    scripts: int = 0
+    assets: int = 0
+
+
+def _table_exists(cur, table: str) -> bool:
+    cur.execute(
+        "SELECT 1 FROM information_schema.tables WHERE table_name=%s",
+        (table,),
+    )
+    return cur.fetchone() is not None
+
+
+def _count(cur, table: str) -> int:
+    cur.execute(f"SELECT COUNT(*) FROM {table}")
+    row = cur.fetchone()
+    return int(row[0]) if row else 0
+
+
+class ContextProjectionBuilder:
+    def __init__(self, repo_root: Path):
+        self.repo_root = repo_root
+
+    def build(self) -> dict:
+        stats = ProjectionStats()
+        top_assets: list[tuple[str, int]] = []
+        top_rules: list[str] = []
+        recent_runs: list[tuple[str, str, float | None]] = []
+
+        with get_conn() as conn:
+            with conn.cursor() as cur:
+                if _table_exists(cur, "igel_requirements"):
+                    stats.requirements = _count(cur, "igel_requirements")
+                if _table_exists(cur, "igel_scenarios"):
+                    stats.scenarios = _count(cur, "igel_scenarios")
+                if _table_exists(cur, "igel_generated_testcases"):
+                    stats.testcases = _count(cur, "igel_generated_testcases")
+                if _table_exists(cur, "igel_automation_scripts"):
+                    stats.scripts = _count(cur, "igel_automation_scripts")
+                if _table_exists(cur, "igel_asset_registry"):
+                    stats.assets = _count(cur, "igel_asset_registry")
+                    cur.execute(
+                        """
+                        SELECT asset_type, COUNT(*) AS c
+                        FROM igel_asset_registry
+                        GROUP BY asset_type
+                        ORDER BY c DESC
+                        LIMIT 8
+                        """
+                    )
+                    top_assets = [(str(r[0]), int(r[1])) for r in cur.fetchall()]
+
+                if _table_exists(cur, "igel_business_rule_registry"):
+                    stats.business_rules = _count(cur, "igel_business_rule_registry")
+                    cur.execute(
+                        """
+                        SELECT COALESCE(rule_key, id::text) AS key, rule_text
+                        FROM igel_business_rule_registry
+                        ORDER BY updated_at DESC NULLS LAST, created_at DESC
+                        LIMIT 6
+                        """
+                    )
+                    top_rules = [f"{r[0]}: {r[1][:140]}" for r in cur.fetchall()]
+                elif _table_exists(cur, "igel_business_rules"):
+                    stats.business_rules = _count(cur, "igel_business_rules")
+                    cur.execute(
+                        "SELECT description FROM igel_business_rules ORDER BY created_at DESC LIMIT 6"
+                    )
+                    top_rules = [str(r[0])[:160] for r in cur.fetchall()]
+
+                if _table_exists(cur, "igel_generation_history"):
+                    cur.execute(
+                        """
+                        SELECT COALESCE(jira_key, '-'), action, quality_score
+                        FROM igel_generation_history
+                        ORDER BY created_at DESC
+                        LIMIT 10
+                        """
+                    )
+                    recent_runs = [
+                        (str(r[0]), str(r[1]), float(r[2]) if r[2] is not None else None)
+                        for r in cur.fetchall()
+                    ]
+
+        content = self._render(stats, top_assets, top_rules, recent_runs)
+        context_path = self.repo_root / ".ai" / "context.md"
+        context_path.parent.mkdir(parents=True, exist_ok=True)
+        context_path.write_text(content, encoding="utf-8")
+
+        return {
+            "context_path": str(context_path),
+            "stats": {
+                "requirements": stats.requirements,
+                "scenarios": stats.scenarios,
+                "business_rules": stats.business_rules,
+                "testcases": stats.testcases,
+                "scripts": stats.scripts,
+                "assets": stats.assets,
+            },
+        }
+
+    def _render(
+        self,
+        stats: ProjectionStats,
+        top_assets: list[tuple[str, int]],
+        top_rules: list[str],
+        recent_runs: list[tuple[str, str, float | None]],
+    ) -> str:
+        lines: list[str] = []
+        lines.append("# Project Context (Generated Projection)")
+        lines.append("")
+        lines.append("This file is generated from registries and execution memory.")
+        lines.append("Do not use it as source of truth for workflow decisions.")
+        lines.append("")
+        lines.append("## Registry Snapshot")
+        lines.append(f"- Requirements: {stats.requirements}")
+        lines.append(f"- Scenarios: {stats.scenarios}")
+        lines.append(f"- Business Rules: {stats.business_rules}")
+        lines.append(f"- Test Cases: {stats.testcases}")
+        lines.append(f"- Scripts: {stats.scripts}")
+        lines.append(f"- Assets: {stats.assets}")
+        lines.append("")
+
+        lines.append("## Asset Mix")
+        if top_assets:
+            for asset_type, count in top_assets:
+                lines.append(f"- {asset_type}: {count}")
+        else:
+            lines.append("- No asset data available")
+        lines.append("")
+
+        lines.append("## Recent Business Rules")
+        if top_rules:
+            for rule in top_rules:
+                lines.append(f"- {rule}")
+        else:
+            lines.append("- No business rules available")
+        lines.append("")
+
+        lines.append("## Recent Generation Runs")
+        if recent_runs:
+            for jira_key, action, quality in recent_runs:
+                q = "n/a" if quality is None else f"{quality:.3f}"
+                lines.append(f"- {jira_key} | {action} | quality={q}")
+        else:
+            lines.append("- No generation history available")
+        lines.append("")
+
+        return "\n".join(lines)

package/src/project/context_requirement_sync.py

@@ -0,0 +1,205 @@
+"""Requirement context enrichment + registry synchronization.
+
+This module implements a single orchestrated flow that:
+1) Persists requirement-scoped context markdown in .ai/context/requirements/<JIRA>/context.md
+2) Finds affected files for the requirement (explicit + registry + repo scan)
+3) Updates PostgreSQL requirement and mapping tables in one transaction
+4) Regenerates .ai/context.md projection cache from registries
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+from src.db.client import get_transaction
+from src.project.context_projection_builder import ContextProjectionBuilder
+
+
+@dataclass
+class RequirementSyncResult:
+    jira_key: str
+    requirement_id: str | None
+    affected_files: list[str]
+    mapped_assets: int
+    requirement_context_path: str
+    projection: dict
+
+
+def _norm_key(jira_key: str) -> str:
+    return jira_key.strip().upper()
+
+
+def _table_exists(cur, table: str) -> bool:
+    cur.execute(
+        "SELECT 1 FROM information_schema.tables WHERE table_name = %s",
+        (table,),
+    )
+    return cur.fetchone() is not None
+
+
+def _write_requirement_context(repo_root: Path, jira_key: str, context_text: str) -> Path:
+    key = _norm_key(jira_key)
+    path = repo_root / ".ai" / "context" / "requirements" / key / "context.md"
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(context_text.strip() + "\n", encoding="utf-8")
+    return path
+
+
+def _repo_scan_for_requirement_files(repo_root: Path, jira_key: str, *, limit: int = 100) -> list[str]:
+    key = _norm_key(jira_key)
+    matches: list[str] = []
+
+    roots = [
+        repo_root / "tests",
+        repo_root / "test",
+        repo_root / "docs",
+        repo_root / ".ai",
+    ]
+    allowed_ext = {".py", ".md", ".yaml", ".yml", ".json", ".feature", ".txt"}
+
+    for root in roots:
+        if not root.exists() or not root.is_dir():
+            continue
+        for fp in root.rglob("*"):
+            if len(matches) >= limit:
+                return sorted(set(matches))
+            if not fp.is_file() or fp.suffix.lower() not in allowed_ext:
+                continue
+
+            rel = str(fp.relative_to(repo_root))
+            if key in rel.upper():
+                matches.append(rel)
+                continue
+
+            try:
+                text = fp.read_text(encoding="utf-8", errors="ignore")
+            except Exception:
+                continue
+
+            if key in text.upper():
+                matches.append(rel)
+
+    return sorted(set(matches))
+
+
+def _registry_files_for_requirement(cur, jira_key: str) -> list[str]:
+    if not (_table_exists(cur, "igel_requirements") and _table_exists(cur, "igel_asset_mappings") and _table_exists(cur, "igel_asset_registry")):
+        return []
+
+    cur.execute(
+        """
+        SELECT DISTINCT a.file_path
+        FROM igel_asset_registry a
+        JOIN igel_asset_mappings m ON m.asset_id = a.id
+        JOIN igel_requirements r ON r.id = m.requirement_id
+        WHERE r.jira_key = %s
+        """,
+        (_norm_key(jira_key),),
+    )
+    rows = cur.fetchall()
+    return sorted({str(r[0]) for r in rows if r and r[0]})
+
+
+def sync_requirement_context(
+    repo_root: Path,
+    jira_key: str,
+    context_text: str,
+    *,
+    changed_files: list[str] | None = None,
+) -> RequirementSyncResult:
+    key = _norm_key(jira_key)
+
+    requirement_context_path = _write_requirement_context(repo_root, key, context_text)
+
+    changed_files = [str(Path(p).as_posix()) for p in (changed_files or []) if str(p).strip()]
+
+    with get_transaction() as conn:
+        with conn.cursor() as cur:
+            requirement_id: str | None = None
+            mapped_assets = 0
+
+            if _table_exists(cur, "igel_requirements"):
+                # Upsert requirement row and keep latest context text in description.
+                cur.execute(
+                    """
+                    INSERT INTO igel_requirements (jira_key, summary, description)
+                    VALUES (%s, %s, %s)
+                    ON CONFLICT (jira_key)
+                    DO UPDATE SET
+                        summary = EXCLUDED.summary,
+                        description = EXCLUDED.description
+                    RETURNING id::text
+                    """,
+                    (key, f"Context enrichment for {key}", context_text),
+                )
+                row = cur.fetchone()
+                requirement_id = str(row[0]) if row else None
+
+            db_files = _registry_files_for_requirement(cur, key)
+            repo_files = _repo_scan_for_requirement_files(repo_root, key)
+
+            all_files = sorted({*changed_files, *db_files, *repo_files})
+
+            if (
+                requirement_id
+                and all_files
+                and _table_exists(cur, "igel_asset_registry")
+                and _table_exists(cur, "igel_asset_mappings")
+            ):
+                # Find assets in affected files and ensure requirement mappings exist.
+                for rel_path in all_files:
+                    cur.execute(
+                        """
+                        SELECT id::text
+                        FROM igel_asset_registry
+                        WHERE file_path = %s
+                           OR file_path LIKE %s
+                        """,
+                        (rel_path, f"%/{rel_path}"),
+                    )
+                    asset_rows = cur.fetchall()
+                    for asset_row in asset_rows:
+                        asset_id = str(asset_row[0])
+                        cur.execute(
+                            """
+                            INSERT INTO igel_asset_mappings (requirement_id, asset_id, mapping_type)
+                            SELECT %s::uuid, %s::uuid, %s
+                            WHERE NOT EXISTS (
+                                SELECT 1 FROM igel_asset_mappings
+                                WHERE requirement_id = %s::uuid
+                                  AND asset_id = %s::uuid
+                                  AND mapping_type = %s
+                            )
+                            """,
+                            (
+                                requirement_id,
+                                asset_id,
+                                "required_for_test",
+                                requirement_id,
+                                asset_id,
+                                "required_for_test",
+                            ),
+                        )
+                        mapped_assets += 1
+
+            # Persist discovered requirement dependencies (file references) when table exists.
+            if requirement_id and _table_exists(cur, "igel_requirement_understanding"):
+                cur.execute(
+                    """
+                    INSERT INTO igel_requirement_understanding (requirement_id, dependencies)
+                    VALUES (%s::uuid, %s::jsonb)
+                    """,
+                    (requirement_id, json.dumps(all_files)),
+                )
+
+    projection = ContextProjectionBuilder(repo_root).build()
+
+    return RequirementSyncResult(
+        jira_key=key,
+        requirement_id=requirement_id,
+        affected_files=all_files,
+        mapped_assets=mapped_assets,
+        requirement_context_path=str(requirement_context_path),
+        projection=projection,
+    )