@jaguilar87/gaia-ops 2.2.0 → 2.2.2

package/tests/tools/test_agent_router.py

@@ -0,0 +1,266 @@
+"""
+Test suite for agent_router.py
+Tests semantic routing, intent classification, and capability validation
+"""
+
+import pytest
+import sys
+from pathlib import Path
+
+# Add tools directory to path
+TOOLS_PATH = Path(__file__).resolve().parents[2] / "tools"
+sys.path.insert(0, str(TOOLS_PATH))
+
+from agent_router import IntentClassifier, CapabilityValidator, AgentRouter
+
+
+class TestIntentClassifier:
+    """Test suite for semantic intent classification"""
+
+    @pytest.fixture
+    def classifier(self):
+        """Initialize intent classifier"""
+        return IntentClassifier()
+
+    def test_infrastructure_creation_intent(self, classifier):
+        """Should classify 'create cluster' as infrastructure_creation"""
+        request = "create a new gke cluster"
+        intent, confidence = classifier.classify(request)
+
+        assert intent == "infrastructure_creation", \
+            f"Expected infrastructure_creation, got {intent}"
+        assert confidence > 0.3, f"Confidence {confidence} should be > 0.3"
+        assert 0 <= confidence <= 1, "Confidence should be normalized to 0-1"
+
+    def test_infrastructure_diagnosis_intent(self, classifier):
+        """Should classify 'diagnose connectivity' as infrastructure_diagnosis"""
+        request = "diagnose cluster connectivity issues"
+        intent, confidence = classifier.classify(request)
+
+        assert intent == "infrastructure_diagnosis", \
+            f"Expected infrastructure_diagnosis, got {intent}"
+        assert confidence > 0.3, f"Confidence {confidence} should be > 0.3"
+
+    def test_kubernetes_operations_intent(self, classifier):
+        """Should classify 'check pod status' as kubernetes_operations"""
+        request = "check pod status in tcm-non-prod namespace"
+        intent, confidence = classifier.classify(request)
+
+        assert intent == "kubernetes_operations", \
+            f"Expected kubernetes_operations, got {intent}"
+        assert confidence > 0.3, f"Confidence {confidence} should be > 0.3"
+
+    def test_application_development_intent(self, classifier):
+        """Should classify 'build docker image' as application_development"""
+        request = "build docker image and run tests"
+        intent, confidence = classifier.classify(request)
+
+        assert intent == "application_development", \
+            f"Expected application_development, got {intent}"
+        assert confidence > 0.3, f"Confidence {confidence} should be > 0.3"
+
+    def test_infrastructure_validation_intent(self, classifier):
+        """Should classify 'validate terraform' as infrastructure_validation"""
+        request = "validate terraform configuration"
+        intent, confidence = classifier.classify(request)
+
+        assert intent == "infrastructure_validation", \
+            f"Expected infrastructure_validation, got {intent}"
+        assert confidence > 0.3, f"Confidence {confidence} should be > 0.3"
+
+    def test_ambiguous_request_low_confidence(self, classifier):
+        """Ambiguous requests should return None intent or low confidence"""
+        request = "what should i do?"
+        intent, confidence = classifier.classify(request)
+
+        if intent is None:
+            assert confidence == 0.0
+        else:
+            assert confidence < 0.3, \
+                "Ambiguous request should have low confidence"
+
+    def test_classification_consistency(self, classifier):
+        """Same request should always classify to same intent"""
+        request = "create a new vpc network"
+
+        results = []
+        for _ in range(5):
+            intent, _ = classifier.classify(request)
+            results.append(intent)
+
+        assert len(set(results)) == 1, \
+            "Classification should be deterministic and consistent"
+
+
+class TestCapabilityValidator:
+    """Test suite for agent capability validation"""
+
+    @pytest.fixture
+    def validator(self):
+        """Initialize capability validator"""
+        return CapabilityValidator()
+
+    def test_terraform_can_create_infrastructure(self, validator):
+        """terraform-architect should handle infrastructure_creation"""
+        is_valid = validator.validate("terraform-architect", "infrastructure_creation")
+        assert is_valid is True, \
+            "terraform-architect should handle infrastructure_creation"
+
+    def test_terraform_cannot_do_kubernetes(self, validator):
+        """terraform-architect should not handle kubernetes_operations"""
+        is_valid = validator.validate("terraform-architect", "kubernetes_operations")
+        assert is_valid is False, \
+            "terraform-architect cannot handle kubernetes_operations"
+
+    def test_unknown_agent_returns_false(self, validator):
+        """Unknown agents should return False"""
+        is_valid = validator.validate("unknown-agent", "infrastructure_creation")
+        assert is_valid is False, "Unknown agents should be invalid"
+
+    def test_find_fallback_agent_for_diagnosis(self, validator):
+        """Should find valid fallback for infrastructure_diagnosis"""
+        fallback = validator.find_fallback_agent("infrastructure_diagnosis")
+
+        assert fallback is not None, "Should find a fallback agent"
+        assert validator.validate(fallback, "infrastructure_diagnosis"), \
+            "Fallback agent should be capable"
+
+    def test_fallback_excludes_agent(self, validator):
+        """Fallback should exclude specified agent"""
+        primary = "terraform-architect"
+        fallback = validator.find_fallback_agent(
+            "infrastructure_validation", 
+            exclude=primary
+        )
+
+        assert fallback != primary, \
+            "Fallback should not be the same as excluded agent"
+
+    def test_capability_matrix_consistency(self, validator):
+        """Capability matrix should be well-defined"""
+        agents = list(validator.agent_capabilities.keys())
+        assert len(agents) >= 4, "Should have at least 4 agents"
+
+        for agent, capabilities in validator.agent_capabilities.items():
+            assert "can_do" in capabilities, \
+                f"{agent} should have 'can_do' list"
+            assert "cannot_do" in capabilities, \
+                f"{agent} should have 'cannot_do' list"
+            
+            # No intent should be in both lists
+            conflict = set(capabilities["can_do"]) & set(capabilities["cannot_do"])
+            assert len(conflict) == 0, \
+                f"{agent} has conflicting capabilities: {conflict}"
+
+
+class TestAgentRouter:
+    """Integration tests for AgentRouter"""
+
+    @pytest.fixture
+    def router(self):
+        """Initialize agent router"""
+        return AgentRouter()
+
+    def test_router_has_semantic_routing(self, router):
+        """Router should have semantic routing capability"""
+        assert hasattr(router, '_route_semantic'), \
+            "Router should have _route_semantic method"
+        assert hasattr(router, 'intent_classifier'), \
+            "Router should have intent_classifier"
+        assert hasattr(router, 'capability_validator'), \
+            "Router should have capability_validator"
+
+    def test_semantic_routing_returns_proper_format(self, router):
+        """_route_semantic should return (agent, confidence, reason)"""
+        agent, confidence, reason = router._route_semantic("create a cluster")
+
+        assert isinstance(agent, str), "Agent should be a string"
+        assert isinstance(confidence, float), "Confidence should be a float"
+        assert isinstance(reason, str), "Reason should be a string"
+        assert 0 <= confidence <= 1, "Confidence should be normalized 0-1"
+
+    def test_semantic_routing_selects_valid_agent(self, router):
+        """_route_semantic should only select valid agents"""
+        test_requests = [
+            "create vpc",
+            "diagnose connectivity",
+            "check pod logs",
+            "build docker image",
+            "validate terraform"
+        ]
+
+        valid_agents = [
+            "terraform-architect", 
+            "gitops-operator", 
+            "gcp-troubleshooter", 
+            "devops-developer"
+        ]
+
+        for request in test_requests:
+            agent, _, _ = router._route_semantic(request)
+            assert agent in valid_agents, \
+                f"Got invalid agent {agent} for request: {request}"
+
+
+class TestRoutingAccuracy:
+    """Accuracy tests for semantic routing"""
+
+    @pytest.fixture
+    def router(self):
+        """Initialize agent router"""
+        return AgentRouter()
+
+    def test_semantic_routing_accuracy_golden_set(self, router):
+        """Test accuracy on golden set of requests"""
+        golden_set = [
+            # infrastructure_creation -> terraform-architect
+            ("create a new gke cluster", "terraform-architect"),
+            ("provision vpc for prod", "terraform-architect"),
+            ("deploy infrastructure changes", "terraform-architect"),
+            
+            # infrastructure_diagnosis -> gcp-troubleshooter
+            ("diagnose connectivity issues", "gcp-troubleshooter"),
+            ("troubleshoot cluster crash", "gcp-troubleshooter"),
+            
+            # kubernetes_operations -> gitops-operator
+            ("check pod status in default", "gitops-operator"),
+            ("verify flux reconciliation", "gitops-operator"),
+            
+            # application_development -> devops-developer
+            ("build docker image", "devops-developer"),
+            ("run unit tests", "devops-developer"),
+            
+            # infrastructure_validation -> terraform-architect
+            ("validate terraform config", "terraform-architect"),
+        ]
+
+        correct = 0
+        failures = []
+        
+        for request, expected_agent in golden_set:
+            agent, _, _ = router._route_semantic(request)
+            if agent == expected_agent:
+                correct += 1
+            else:
+                failures.append({
+                    "request": request,
+                    "expected": expected_agent,
+                    "got": agent
+                })
+
+        accuracy = correct / len(golden_set)
+        
+        if failures:
+            print("\nRouting failures:")
+            for failure in failures:
+                print(f"  '{failure['request']}'")
+                print(f"    Expected: {failure['expected']}, Got: {failure['got']}")
+        
+        assert accuracy >= 0.75, \
+            f"Semantic routing accuracy should be >= 75%, got {accuracy*100:.1f}%"
+
+        print(f"\nSemantic Routing Accuracy: {accuracy*100:.1f}% ({correct}/{len(golden_set)})")
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v", "--tb=short"])

package/tests/tools/test_clarify_engine.py

@@ -0,0 +1,413 @@
+"""
+Unit tests for clarify_engine.py
+"""
+
+import pytest
+import json
+import sys
+import os
+
+# Add tools to path (gaia-ops/tools)
+tools_path = os.path.join(os.path.dirname(__file__), '..', '..', 'tools')
+sys.path.insert(0, tools_path)
+
+from clarify_engine import ClarificationEngine, request_clarification, process_clarification
+
+
+@pytest.fixture
+def mock_project_context():
+    """Mock project-context.json with test data."""
+    return {
+        "sections": {
+            "application_services": [
+                {
+                    "name": "tcm-api",
+                    "tech_stack": "NestJS",
+                    "namespace": "tcm-non-prod",
+                    "port": 3001,
+                    "status": "running"
+                },
+                {
+                    "name": "tcm-web",
+                    "tech_stack": "React SPA",
+                    "namespace": "tcm-non-prod",
+                    "port": 3000,
+                    "status": "running"
+                },
+                {
+                    "name": "pg-api",
+                    "tech_stack": "Spring Boot",
+                    "namespace": "pg-non-prod",
+                    "port": 8086,
+                    "status": "running"
+                }
+            ],
+            "cluster_details": {
+                "primary_namespaces": ["tcm-non-prod", "pg-non-prod"]
+            },
+            "project_details": {
+                "environment": "non-prod"
+            },
+            "terraform_infrastructure": {
+                "modules": {
+                    "tcm-redis": {
+                        "resources": "Memorystore Redis",
+                        "status": "running",
+                        "tier": "BASIC"
+                    },
+                    "pg-redis": {
+                        "resources": "Memorystore Redis",
+                        "status": "running",
+                        "tier": "STANDARD_HA"
+                    }
+                }
+            }
+        }
+    }
+
+
+@pytest.fixture
+def engine_with_mock_context(mock_project_context, tmp_path):
+    """Create engine with mock project context."""
+    # Create temporary project-context.json
+    context_file = tmp_path / "project-context.json"
+    with open(context_file, "w") as f:
+        json.dump(mock_project_context, f)
+
+    engine = ClarificationEngine(project_context_path=str(context_file))
+    return engine
+
+
+def test_detect_service_ambiguity(engine_with_mock_context):
+    """Test detection of ambiguous service references."""
+    result = engine_with_mock_context.detect_ambiguity("Check the API")
+
+    assert result["needs_clarification"] == True
+    assert result["ambiguity_score"] > 30
+    assert len(result["ambiguity_points"]) > 0
+    assert "service" in result["ambiguity_points"][0]["pattern"]
+
+
+def test_detect_namespace_ambiguity(engine_with_mock_context):
+    """Test detection of ambiguous namespace references."""
+    result = engine_with_mock_context.detect_ambiguity("Deploy to cluster")
+
+    assert result["needs_clarification"] == True
+    assert any("namespace" in a["pattern"] for a in result["ambiguity_points"])
+
+
+def test_detect_environment_warning(engine_with_mock_context):
+    """Test detection of environment mismatch."""
+    result = engine_with_mock_context.detect_ambiguity("Deploy to production")
+
+    assert result["needs_clarification"] == True
+    assert any("environment" in a["pattern"] for a in result["ambiguity_points"])
+    # Environment ambiguity has highest weight (90)
+    if result["ambiguity_points"]:
+        assert result["ambiguity_score"] >= 70  # High weight
+
+
+def test_detect_resource_ambiguity(engine_with_mock_context):
+    """Test detection of ambiguous Redis resources."""
+    result = engine_with_mock_context.detect_ambiguity("Check the Redis")
+
+    assert result["needs_clarification"] == True
+    assert any("resource" in a["pattern"] for a in result["ambiguity_points"])
+
+
+def test_no_ambiguity_specific_prompt(engine_with_mock_context):
+    """Test that specific prompts don't trigger clarification."""
+    result = engine_with_mock_context.detect_ambiguity(
+        "Check tcm-api service in tcm-non-prod namespace"
+    )
+
+    assert result["needs_clarification"] == False
+    assert result["ambiguity_score"] <= 30
+
+
+def test_generate_questions(engine_with_mock_context):
+    """Test question generation with rich options."""
+    ambiguity_analysis = {
+        "needs_clarification": True,
+        "ambiguity_score": 80,
+        "ambiguity_points": [
+            {
+                "pattern": "service_ambiguity",
+                "detected_keyword": "the api",
+                "ambiguity_reason": "Multiple services",
+                "available_options": ["tcm-api", "pg-api"],
+                "services_metadata": {
+                    "tcm-api": {
+                        "tech_stack": "NestJS",
+                        "namespace": "tcm-non-prod",
+                        "port": 3001,
+                        "status": "running"
+                    },
+                    "pg-api": {
+                        "tech_stack": "Spring Boot",
+                        "namespace": "pg-non-prod",
+                        "port": 8086,
+                        "status": "running"
+                    }
+                },
+                "suggested_question": "Which API?",
+                "weight": 80,
+                "allow_multiple": False
+            }
+        ],
+        "suggested_questions": ["Which API?"]
+    }
+
+    result = engine_with_mock_context.generate_questions(ambiguity_analysis)
+
+    assert "summary" in result
+    assert "question_config" in result
+    assert len(result["question_config"]["questions"]) == 1
+
+    # Check question structure
+    question = result["question_config"]["questions"][0]
+    assert question["question"] == "Which API?"
+    assert question["multiSelect"] == False
+    assert len(question["options"]) == 2  # 2 options (tcm-api, pg-api)
+
+    # Check options have emoji and rich descriptions
+    for option in question["options"]:
+        assert "label" in option
+        assert "description" in option
+        # Service options should have some emoji (❓, 📦, 🎯, etc.)
+        # Just verify label is not empty and has some kind of prefix
+        assert len(option["label"]) > 0
+        assert ("Namespace:" in option["description"] or "N/A" in option["description"]
+                or "Tech" in option["description"])
+
+
+def test_generate_questions_with_catchall(engine_with_mock_context):
+    """Test question generation with 4th 'All' option."""
+    ambiguity_analysis = {
+        "needs_clarification": True,
+        "ambiguity_score": 80,
+        "ambiguity_points": [
+            {
+                "pattern": "service_ambiguity",
+                "detected_keyword": "services",
+                "ambiguity_reason": "Multiple services",
+                "available_options": ["tcm-api", "tcm-web", "pg-api", "pg-web"],  # 4+ options
+                "services_metadata": {},
+                "suggested_question": "Which services?",
+                "weight": 80,
+                "allow_multiple": False
+            }
+        ],
+        "suggested_questions": ["Which services?"]
+    }
+
+    result = engine_with_mock_context.generate_questions(ambiguity_analysis)
+
+    question = result["question_config"]["questions"][0]
+    # Should have 4 options: 3 specific + 1 "All"
+    assert len(question["options"]) == 4
+    # Last option should be "All"
+    assert "Todos" in question["options"][-1]["label"] or "🌐" in question["options"][-1]["label"]
+
+
+def test_enrich_prompt(engine_with_mock_context):
+    """Test prompt enrichment with user responses."""
+    original_prompt = "Check the API"
+    user_responses = {"question_1": "📦 tcm-api"}
+    clarification_context = {
+        "ambiguities": [
+            {
+                "pattern": "service_ambiguity",
+                "suggested_question": "Which API?",
+                "available_options": ["tcm-api", "pg-api"]
+            }
+        ]
+    }
+
+    enriched = engine_with_mock_context.enrich_prompt(
+        original_prompt,
+        user_responses,
+        clarification_context
+    )
+
+    assert "Check the API" in enriched
+    assert "tcm-api" in enriched
+    assert "[Clarification" in enriched
+
+
+def test_clean_answer(engine_with_mock_context):
+    """Test emoji removal from user answers."""
+    assert engine_with_mock_context._clean_answer("📦 tcm-api") == "tcm-api"
+    assert engine_with_mock_context._clean_answer("🎯 tcm-non-prod") == "tcm-non-prod"
+    assert engine_with_mock_context._clean_answer("plain text") == "plain text"
+
+
+def test_validate_answer_exact_match(engine_with_mock_context):
+    """Test exact answer validation."""
+    ambiguity = {
+        "available_options": ["tcm-api", "pg-api"]
+    }
+
+    assert engine_with_mock_context._validate_answer("tcm-api", ambiguity) == "tcm-api"
+
+
+def test_validate_answer_fuzzy_match(engine_with_mock_context):
+    """Test fuzzy matching of user answers."""
+    ambiguity = {
+        "available_options": ["tcm-api", "pg-api"]
+    }
+
+    # User types "tcm api" (with space)
+    result = engine_with_mock_context._validate_answer("tcm api", ambiguity)
+    assert "tcm-api" in result.lower() or result == "tcm api"  # Either matched or kept as-is
+
+
+def test_validate_answer_all_keyword(engine_with_mock_context):
+    """Test 'all' keyword detection."""
+    ambiguity = {
+        "available_options": ["tcm-api", "pg-api"]
+    }
+
+    result = engine_with_mock_context._validate_answer("todos", ambiguity)
+    assert "Todos" in result or "tcm-api" in result
+
+
+def test_convenience_function_request_clarification():
+    """Test convenience function with minimal setup."""
+    # This will use actual project-context.json if it exists
+    result = request_clarification("Check the API")
+
+    # Should return dict with expected keys
+    assert "needs_clarification" in result
+
+    if result["needs_clarification"]:
+        assert "summary" in result
+        assert "question_config" in result
+        assert "engine_instance" in result
+
+
+def test_command_context_filtering(engine_with_mock_context):
+    """Test that command context filters patterns."""
+    # Mock config to disable service_ambiguity for a specific command
+    engine_with_mock_context.config["command_rules"]["test_command"] = {
+        "enabled": True,
+        "patterns": ["namespace_ambiguity"]  # Only namespace, not service
+    }
+
+    result = engine_with_mock_context.detect_ambiguity(
+        "Check the API",
+        command_context={"command": "test_command"}
+    )
+
+    # Service ambiguity should be filtered out
+    if result["ambiguity_points"]:
+        assert all(a["pattern"] != "service_ambiguity" for a in result["ambiguity_points"])
+
+
+def test_disabled_command(engine_with_mock_context):
+    """Test that disabled commands skip clarification."""
+    engine_with_mock_context.config["command_rules"]["disabled_command"] = {
+        "enabled": False
+    }
+
+    result = engine_with_mock_context.detect_ambiguity(
+        "Check the API",
+        command_context={"command": "disabled_command"}
+    )
+
+    assert result["needs_clarification"] == False
+
+
+def test_multiple_ambiguities_sorted_by_weight(engine_with_mock_context):
+    """Test that multiple ambiguities are sorted by weight."""
+    result = engine_with_mock_context.detect_ambiguity(
+        "Deploy the API to cluster in production"
+    )
+
+    if len(result["ambiguity_points"]) > 1:
+        # Should be sorted by weight (descending)
+        weights = [a["weight"] for a in result["ambiguity_points"]]
+        assert weights == sorted(weights, reverse=True)
+
+
+def test_get_option_metadata_service(engine_with_mock_context):
+    """Test metadata generation for service options."""
+    ambiguity = {
+        "pattern": "service_ambiguity",
+        "services_metadata": {
+            "tcm-api": {
+                "tech_stack": "NestJS",
+                "namespace": "tcm-non-prod",
+                "port": 3001,
+                "status": "running"
+            }
+        }
+    }
+
+    metadata = engine_with_mock_context._get_option_metadata("tcm-api", ambiguity)
+
+    assert "NestJS" in metadata
+    assert "tcm-non-prod" in metadata
+    assert "3001" in metadata
+    assert "✅" in metadata  # Running status emoji
+
+
+def test_get_option_metadata_namespace(engine_with_mock_context):
+    """Test metadata generation for namespace options."""
+    ambiguity = {
+        "pattern": "namespace_ambiguity",
+        "namespace_metadata": {
+            "tcm-non-prod": {
+                "services": ["tcm-api", "tcm-web"],
+                "service_count": 2
+            }
+        }
+    }
+
+    metadata = engine_with_mock_context._get_option_metadata("tcm-non-prod", ambiguity)
+
+    assert "tcm-api" in metadata
+    assert "2 servicios" in metadata or "2 services" in metadata
+
+
+def test_spanish_keywords_detected(engine_with_mock_context):
+    """Test that Spanish keywords are detected."""
+    # Spanish: "Chequea el servicio"
+    result = engine_with_mock_context.detect_ambiguity("Chequea el servicio")
+
+    assert result["needs_clarification"] == True
+    assert any("service" in a["pattern"] for a in result["ambiguity_points"])
+
+
+def test_log_clarification(engine_with_mock_context, tmp_path):
+    """Test that clarification is logged properly."""
+    # Override log path to temp directory
+    log_file = tmp_path / "clarifications.jsonl"
+    engine_with_mock_context.clarification_log_path = str(log_file)
+    os.makedirs(tmp_path, exist_ok=True)
+
+    ambiguity_analysis = {
+        "ambiguity_score": 80,
+        "ambiguity_points": [
+            {"pattern": "service_ambiguity"}
+        ]
+    }
+
+    engine_with_mock_context.log_clarification(
+        "Original prompt",
+        "Enriched prompt",
+        ambiguity_analysis,
+        {"question_1": "tcm-api"}
+    )
+
+    # Check log file exists and has content
+    assert log_file.exists()
+    with open(log_file, "r") as f:
+        log_entry = json.loads(f.read())
+        assert log_entry["original_prompt"] == "Original prompt"
+        assert log_entry["enriched_prompt"] == "Enriched prompt"
+        assert log_entry["ambiguity_score"] == 80
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])