@ngocsangairvds/vsaf 3.2.14 → 3.2.15

package/tools/vds-scripts/audit_orchestrator/tests/unit/agents/test_skill_executor.py

@@ -1,316 +0,0 @@
-"""Tests for SkillExecutor (Phase 23 - FR-1.4).
-
-Tests the grepai_search() method with path parameter and deprecation warning.
-"""
-
-from __future__ import annotations
-
-import json
-import subprocess
-import warnings
-from unittest.mock import MagicMock, patch
-
-from vds_agent_core.skills.executor import SkillExecutor
-
-
-class TestGrepaiSearch:
-    """Tests for SkillExecutor._grepai_search() method."""
-
-    def test_grepai_search_without_path_issues_deprecation_warning(self) -> None:
-        """Test that calling grepai_search without path issues deprecation warning (FR-1.4)."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout="[]",
-                stderr="",
-            )
-
-            with warnings.catch_warnings(record=True) as w:
-                warnings.simplefilter("always")
-                result = SkillExecutor._grepai_search("test query")
-
-                # Should have issued a deprecation warning
-                assert len(w) == 1
-                assert issubclass(w[0].category, DeprecationWarning)
-                assert "without explicit path is deprecated" in str(w[0].message)
-                assert "SecurityPatternScanner" in str(w[0].message)
-
-            # Result should contain warning
-            assert "warning" in result
-            assert "deprecated" in result["warning"].lower()
-
-    def test_grepai_search_with_path_no_deprecation_warning(self) -> None:
-        """Test that calling grepai_search with path does not issue deprecation warning."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout="[]",
-                stderr="",
-            )
-
-            with warnings.catch_warnings(record=True) as w:
-                warnings.simplefilter("always")
-                result = SkillExecutor._grepai_search("test query", path="/some/repo")
-
-                # Should NOT have issued a deprecation warning
-                deprecation_warnings = [x for x in w if issubclass(x.category, DeprecationWarning)]
-                assert len(deprecation_warnings) == 0
-
-            # Result should not contain warning
-            assert "warning" not in result
-
-    def test_grepai_search_passes_path_to_cli(self) -> None:
-        """Test that path parameter is passed to grepai CLI command (FR-1.4)."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout="[]",
-                stderr="",
-            )
-
-            SkillExecutor._grepai_search("security patterns", path="/path/to/repo")
-
-            # Verify the command includes --path
-            mock_run.assert_called_once()
-            cmd = mock_run.call_args[0][0]
-            assert "--path" in cmd
-            path_index = cmd.index("--path")
-            assert cmd[path_index + 1] == "/path/to/repo"
-
-    def test_grepai_search_without_path_no_path_in_cli(self) -> None:
-        """Test that path is not in CLI command when not provided."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout="[]",
-                stderr="",
-            )
-
-            with warnings.catch_warnings():
-                warnings.simplefilter("ignore")
-                SkillExecutor._grepai_search("test query")
-
-            # Verify the command does NOT include --path
-            mock_run.assert_called_once()
-            cmd = mock_run.call_args[0][0]
-            assert "--path" not in cmd
-
-    def test_grepai_search_passes_limit_to_cli(self) -> None:
-        """Test that limit parameter is passed to grepai CLI command."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout="[]",
-                stderr="",
-            )
-
-            SkillExecutor._grepai_search("test query", path="/repo", limit=25)
-
-            cmd = mock_run.call_args[0][0]
-            assert "--limit" in cmd
-            limit_index = cmd.index("--limit")
-            assert cmd[limit_index + 1] == "25"
-
-    def test_grepai_search_returns_parsed_results(self) -> None:
-        """Test that grepai_search parses JSON results correctly."""
-        mock_results = [
-            {"file": "src/auth.py", "line": 10, "content": "def login():", "score": 0.95},
-            {"file": "src/security.py", "line": 20, "content": "def check_token():", "score": 0.85},
-        ]
-
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout=json.dumps(mock_results),
-                stderr="",
-            )
-
-            result = SkillExecutor._grepai_search("authentication", path="/repo")
-
-            assert result["success"] is True
-            assert result["results"] == mock_results
-            assert result["query"] == "authentication"
-            assert result["path"] == "/repo"
-
-    def test_grepai_search_handles_empty_results(self) -> None:
-        """Test that grepai_search handles empty results."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout="[]",
-                stderr="",
-            )
-
-            result = SkillExecutor._grepai_search("nonexistent pattern", path="/repo")
-
-            assert result["success"] is True
-            assert result["results"] == []
-
-    def test_grepai_search_handles_command_failure(self) -> None:
-        """Test that grepai_search handles command failure gracefully."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=1,
-                stdout="",
-                stderr="grepai: index not found",
-            )
-
-            result = SkillExecutor._grepai_search("test query", path="/repo")
-
-            assert result["success"] is False
-            assert "error" in result
-            assert "index not found" in result["error"]
-
-    def test_grepai_search_handles_timeout(self) -> None:
-        """Test that grepai_search handles timeout gracefully."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.side_effect = subprocess.TimeoutExpired(cmd="grepai", timeout=60)
-
-            result = SkillExecutor._grepai_search("slow query", path="/repo")
-
-            assert result["success"] is False
-            assert "error" in result
-            assert "timed out" in result["error"]
-
-    def test_grepai_search_handles_grepai_not_found(self) -> None:
-        """Test that grepai_search handles grepai not installed."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.side_effect = FileNotFoundError("grepai not found")
-
-            result = SkillExecutor._grepai_search("test query", path="/repo")
-
-            assert result["success"] is False
-            assert "error" in result
-            assert "not found" in result["error"]
-
-    def test_grepai_search_handles_json_parse_error(self) -> None:
-        """Test that grepai_search handles invalid JSON output."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout="not valid json {{{",
-                stderr="",
-            )
-
-            result = SkillExecutor._grepai_search("test query", path="/repo")
-
-            assert result["success"] is False
-            assert "error" in result
-            assert "parse" in result["error"].lower() or "raw_output" in result
-
-    def test_grepai_search_records_tool_usage(self) -> None:
-        """Test that grepai_search records tool usage."""
-        SkillExecutor.reset_usage()
-
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout="[]",
-                stderr="",
-            )
-
-            SkillExecutor._grepai_search("test query", path="/repo", limit=5)
-
-        usage = SkillExecutor.consume_usage()
-        assert len(usage) == 1
-        # Method was privatized (Phase 8 FR-8.7); recorded usage tool name
-        # tracks the underscored impl name.
-        assert usage[0]["tool"] == "_grepai_search"
-        assert usage[0]["query"] == "test query"
-        assert usage[0]["path"] == "/repo"
-        assert usage[0]["limit"] == 5
-        assert usage[0]["scoped"] is True
-
-    def test_grepai_search_records_unscoped_in_usage(self) -> None:
-        """Test that unscoped searches are marked in tool usage."""
-        SkillExecutor.reset_usage()
-
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout="[]",
-                stderr="",
-            )
-
-            with warnings.catch_warnings():
-                warnings.simplefilter("ignore")
-                SkillExecutor._grepai_search("test query")
-
-        usage = SkillExecutor.consume_usage()
-        assert len(usage) == 1
-        assert usage[0]["scoped"] is False
-
-    def test_grepai_search_uses_json_and_compact_flags(self) -> None:
-        """Test that grepai_search uses --json and --compact flags."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout="[]",
-                stderr="",
-            )
-
-            SkillExecutor._grepai_search("test query", path="/repo")
-
-            cmd = mock_run.call_args[0][0]
-            assert "--json" in cmd
-            assert "--compact" in cmd
-
-    def test_grepai_search_default_limit(self) -> None:
-        """Test that grepai_search uses default limit of 10."""
-        with patch("subprocess.run") as mock_run:
-            mock_run.return_value = MagicMock(
-                returncode=0,
-                stdout="[]",
-                stderr="",
-            )
-
-            result = SkillExecutor._grepai_search("test query", path="/repo")
-
-            assert result["limit"] == 10
-            cmd = mock_run.call_args[0][0]
-            limit_index = cmd.index("--limit")
-            assert cmd[limit_index + 1] == "10"
-
-
-class TestSkillExecutorUsageTracking:
-    """Tests for SkillExecutor usage tracking."""
-
-    def test_reset_usage_clears_context(self) -> None:
-        """Test that reset_usage clears the tool usage context."""
-        # Add some usage
-        SkillExecutor.record_tool_usage({"tool": "test", "success": True})
-        SkillExecutor.record_tool_usage({"tool": "test2", "success": True})
-
-        # Reset
-        SkillExecutor.reset_usage()
-
-        # Should be empty
-        usage = SkillExecutor.consume_usage()
-        assert usage == []
-
-    def test_consume_usage_returns_and_clears(self) -> None:
-        """Test that consume_usage returns usage and clears it."""
-        SkillExecutor.reset_usage()
-        SkillExecutor.record_tool_usage({"tool": "test", "success": True})
-
-        # First consume should return the usage
-        usage1 = SkillExecutor.consume_usage()
-        assert len(usage1) == 1
-
-        # Second consume should return empty
-        usage2 = SkillExecutor.consume_usage()
-        assert usage2 == []
-
-    def test_record_tool_usage_accumulates(self) -> None:
-        """Test that record_tool_usage accumulates entries."""
-        SkillExecutor.reset_usage()
-
-        SkillExecutor.record_tool_usage({"tool": "tool1", "success": True})
-        SkillExecutor.record_tool_usage({"tool": "tool2", "success": False})
-        SkillExecutor.record_tool_usage({"tool": "tool3", "success": True})
-
-        usage = SkillExecutor.consume_usage()
-        assert len(usage) == 3
-        assert usage[0]["tool"] == "tool1"
-        assert usage[1]["tool"] == "tool2"
-        assert usage[2]["tool"] == "tool3"

package/tools/vds-scripts/audit_orchestrator/tests/unit/agents/test_skill_fallback.py

@@ -1,299 +0,0 @@
-"""Tests for skill fallback robustness (FR-226, TSK-910.15–910.17)."""
-
-from __future__ import annotations
-
-from vds_agent_core.skills.policy import (
-    FallbackCandidate,
-    FallbackResult,
-    SkillPolicyEnforcer,
-    SkillRankingEntry,
-)
-
-
-def _make_ranking(name: str, score: float, *, executions: int = 5, successes: int = 4) -> SkillRankingEntry:
-    return SkillRankingEntry(
-        skill_name=name,
-        execution_count=executions,
-        success_count=successes,
-        effectiveness_score=score,
-        rationale="high_effectiveness" if score >= 0.8 else "moderate_effectiveness",
-    )
-
-
-# ---------------------------------------------------------------------------
-# TSK-910.15: Broad fallback skill expansion (FR-226)
-# ---------------------------------------------------------------------------
-
-
-class TestFallbackTriggersWhenPrimaryFails:
-    """Fallback should trigger when primary skill execution fails."""
-
-    def test_fallback_triggers_when_primary_fails(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        available = [
-            _make_ranking("primary-skill", 0.9),
-            _make_ranking("backup-a", 0.8),
-            _make_ranking("backup-b", 0.6),
-        ]
-        # All candidates fail
-        result = enforcer.attempt_fallback("primary-skill", available, execute_fn=lambda _: False)
-        assert result.triggered is True
-        assert result.success is False
-        assert result.primary_skill == "primary-skill"
-        assert result.candidate_count == 2
-
-    def test_fallback_does_not_trigger_when_primary_succeeds(self) -> None:
-        """When primary succeeds, attempt_fallback is not called at all.
-
-        This test verifies the FallbackResult default state represents
-        a no-fallback scenario.
-        """
-        result = FallbackResult()
-        assert result.triggered is False
-        assert result.success is False
-        assert result.candidate_count == 0
-        assert result.candidate_names == []
-
-
-class TestFallbackBounding:
-    """Fallback expansion must be bounded to max_fallback_candidates."""
-
-    def test_fallback_bounded_to_max_candidates(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        available = [
-            _make_ranking("primary", 0.95),
-            _make_ranking("alt-a", 0.9),
-            _make_ranking("alt-b", 0.85),
-            _make_ranking("alt-c", 0.8),
-            _make_ranking("alt-d", 0.7),
-        ]
-        result = enforcer.attempt_fallback("primary", available, execute_fn=lambda _: False, max_fallback_candidates=2)
-        assert result.candidate_count == 2
-
-    def test_fallback_bounded_to_one(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        available = [
-            _make_ranking("primary", 0.95),
-            _make_ranking("alt-a", 0.9),
-            _make_ranking("alt-b", 0.85),
-        ]
-        result = enforcer.attempt_fallback("primary", available, execute_fn=lambda _: False, max_fallback_candidates=1)
-        assert result.candidate_count == 1
-        assert result.candidates_tried[0].skill_name == "alt-a"
-
-    def test_fallback_max_zero_yields_empty(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        available = [
-            _make_ranking("primary", 0.95),
-            _make_ranking("alt-a", 0.9),
-        ]
-        result = enforcer.attempt_fallback("primary", available, execute_fn=lambda _: False, max_fallback_candidates=0)
-        assert result.triggered is True
-        assert result.candidate_count == 0
-        assert result.success is False
-
-
-class TestFallbackRanking:
-    """Fallback candidates ranked by objective_fit_score (effectiveness_score)."""
-
-    def test_fallback_ranked_by_objective_fit(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        available = [
-            _make_ranking("primary", 0.95),
-            _make_ranking("low-fit", 0.3),
-            _make_ranking("high-fit", 0.9),
-            _make_ranking("mid-fit", 0.6),
-        ]
-        result = enforcer.attempt_fallback("primary", available, execute_fn=lambda _: False, max_fallback_candidates=3)
-        names = result.candidate_names
-        assert names == ["high-fit", "mid-fit", "low-fit"]
-
-    def test_fallback_excludes_primary_skill(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        available = [
-            _make_ranking("primary", 0.95),
-            _make_ranking("alt-a", 0.8),
-        ]
-        result = enforcer.attempt_fallback("primary", available, execute_fn=lambda _: False)
-        assert "primary" not in result.candidate_names
-        assert result.candidate_names == ["alt-a"]
-
-    def test_fallback_with_empty_candidate_pool(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        # Only the primary skill exists — no fallback candidates
-        available = [_make_ranking("primary", 0.95)]
-        result = enforcer.attempt_fallback("primary", available, execute_fn=lambda _: False)
-        assert result.triggered is True
-        assert result.candidate_count == 0
-        assert result.success is False
-
-    def test_fallback_with_completely_empty_list(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        result = enforcer.attempt_fallback("primary", [], execute_fn=lambda _: False)
-        assert result.triggered is True
-        assert result.candidate_count == 0
-        assert result.success is False
-
-
-class TestFallbackExecution:
-    """Fallback execution behavior: early stop on success, all-fail path."""
-
-    def test_fallback_success_stops_expansion(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        available = [
-            _make_ranking("primary", 0.95),
-            _make_ranking("first-alt", 0.9),
-            _make_ranking("second-alt", 0.85),
-        ]
-        # First candidate succeeds — second should not be tried
-        result = enforcer.attempt_fallback(
-            "primary",
-            available,
-            execute_fn=lambda name: name == "first-alt",
-        )
-        assert result.success is True
-        assert result.candidate_count == 1
-        assert result.candidates_tried[0].skill_name == "first-alt"
-        assert result.candidates_tried[0].success is True
-
-    def test_fallback_all_candidates_fail(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        available = [
-            _make_ranking("primary", 0.95),
-            _make_ranking("alt-a", 0.8),
-            _make_ranking("alt-b", 0.7),
-        ]
-        result = enforcer.attempt_fallback("primary", available, execute_fn=lambda _: False)
-        assert result.success is False
-        assert result.candidate_count == 2
-        assert all(not c.success for c in result.candidates_tried)
-
-    def test_fallback_dry_run_without_execute_fn(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        available = [
-            _make_ranking("primary", 0.95),
-            _make_ranking("alt-a", 0.8),
-        ]
-        result = enforcer.attempt_fallback("primary", available, execute_fn=None)
-        assert result.triggered is True
-        assert result.success is False
-        assert result.candidate_count == 1
-        assert result.candidates_tried[0].success is False
-
-
-# ---------------------------------------------------------------------------
-# TSK-910.16: Fallback telemetry (FR-226)
-# ---------------------------------------------------------------------------
-
-
-class TestFallbackTelemetry:
-    """FR-226: Fallback telemetry fields emitted correctly."""
-
-    def test_fallback_telemetry_emitted_on_trigger(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        available = [
-            _make_ranking("primary", 0.95),
-            _make_ranking("alt-a", 0.8),
-            _make_ranking("alt-b", 0.7),
-        ]
-        fb = enforcer.attempt_fallback("primary", available, execute_fn=lambda _: False)
-        event = enforcer.build_telemetry_event(row_index=3, check_id="CL-010", fallback_result=fb)
-        assert event.fallback_triggered is True
-        assert event.fallback_candidate_count == 2
-        assert event.fallback_success is False
-        assert event.fallback_candidates == ["alt-a", "alt-b"]
-
-    def test_fallback_telemetry_not_emitted_on_success(self) -> None:
-        """When no fallback is needed, telemetry fields remain defaults."""
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        event = enforcer.build_telemetry_event(row_index=1)
-        assert event.fallback_triggered is False
-        assert event.fallback_candidate_count == 0
-        assert event.fallback_success is False
-        assert event.fallback_candidates == []
-
-    def test_fallback_telemetry_to_dict_includes_fallback(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        fb = FallbackResult(
-            triggered=True,
-            candidates_tried=[
-                FallbackCandidate("alt-a", 0.8, success=True),
-            ],
-            success=True,
-            primary_skill="primary",
-        )
-        event = enforcer.build_telemetry_event(fallback_result=fb)
-        d = event.to_dict()
-        assert d["fallback_triggered"] is True
-        assert d["fallback_candidate_count"] == 1
-        assert d["fallback_success"] is True
-        assert d["fallback_candidates"] == ["alt-a"]
-
-    def test_fallback_telemetry_to_dict_omits_when_not_triggered(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        event = enforcer.build_telemetry_event()
-        d = event.to_dict()
-        assert "fallback_triggered" not in d
-        assert "fallback_candidate_count" not in d
-        assert "fallback_success" not in d
-        assert "fallback_candidates" not in d
-
-    def test_fallback_candidate_count_in_telemetry(self) -> None:
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        available = [
-            _make_ranking("primary", 0.95),
-            _make_ranking("alt-a", 0.9),
-        ]
-        fb = enforcer.attempt_fallback(
-            "primary",
-            available,
-            execute_fn=lambda name: name == "alt-a",
-        )
-        event = enforcer.build_telemetry_event(fallback_result=fb)
-        assert event.fallback_candidate_count == 1
-        assert event.fallback_success is True
-
-    def test_fallback_telemetry_with_none_result(self) -> None:
-        """Passing fallback_result=None should not trigger fallback fields."""
-        enforcer = SkillPolicyEnforcer(mode="standard")
-        event = enforcer.build_telemetry_event(fallback_result=None)
-        assert event.fallback_triggered is False
-        d = event.to_dict()
-        assert "fallback_triggered" not in d
-
-
-# ---------------------------------------------------------------------------
-# FallbackResult / FallbackCandidate dataclass unit tests
-# ---------------------------------------------------------------------------
-
-
-class TestFallbackResultDataclass:
-    """Unit tests for FallbackResult and FallbackCandidate."""
-
-    def test_candidate_count_property(self) -> None:
-        result = FallbackResult(
-            triggered=True,
-            candidates_tried=[
-                FallbackCandidate("a", 0.9, success=False),
-                FallbackCandidate("b", 0.8, success=True),
-            ],
-        )
-        assert result.candidate_count == 2
-
-    def test_candidate_names_property(self) -> None:
-        result = FallbackResult(
-            triggered=True,
-            candidates_tried=[
-                FallbackCandidate("x-skill", 0.7, success=False),
-                FallbackCandidate("y-skill", 0.6, success=False),
-            ],
-        )
-        assert result.candidate_names == ["x-skill", "y-skill"]
-
-    def test_default_fallback_result(self) -> None:
-        result = FallbackResult()
-        assert result.triggered is False
-        assert result.success is False
-        assert result.primary_skill is None
-        assert result.candidate_count == 0
-        assert result.candidate_names == []