@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/audit_orchestrator/tests/test_agents/test_langgraph_workflow_efficiency_metrics.py

@@ -1,71 +0,0 @@
-"""Tests for compact efficiency metric extraction in langgraph workflow logs."""
-
-from __future__ import annotations
-
-from vds_audit_orchestrator.agents.langgraph_workflow import _extract_agent_efficiency_metrics
-
-
-def test_extract_agent_efficiency_metrics_prefers_agentic_summary() -> None:
-    metadata = {
-        "agentic_tool_efficiency": {
-            "tool_calls_started": 10,
-            "tool_calls_completed": 9,
-            "tool_calls_unique": 4,
-            "tool_calls_repeated": 5,
-            "tool_completion_rate": 0.91234,
-            "tool_repetition_rate": 0.55555,
-            "skill_calls_total": 3,
-            "skill_calls_effective": 2,
-            "skill_effective_rate": 0.66666,
-            "skill_discovery_ratio": 0.33333,
-            "tool_latency_avg_ms": 180,
-            "tool_latency_p95_ms": 420,
-            "ignored_verbose_payload": {"nested": "should-not-leak"},
-        },
-        "tool_usage": [
-            {"success": True},
-            {"success": False},
-            {"success": True},
-        ],
-    }
-
-    summary = _extract_agent_efficiency_metrics(metadata)
-
-    assert summary == {
-        "tool_calls_started": 10,
-        "tool_calls_completed": 9,
-        "tool_calls_unique": 4,
-        "tool_calls_repeated": 5,
-        "tool_completion_rate": 0.912,
-        "tool_repetition_rate": 0.556,
-        "skill_calls_total": 3,
-        "skill_calls_effective": 2,
-        "skill_effective_rate": 0.667,
-        "skill_discovery_ratio": 0.333,
-        "tool_latency_avg_ms": 180,
-        "tool_latency_p95_ms": 420,
-        "skill_executor_calls_total": 3,
-        "skill_executor_success_rate": 0.667,
-    }
-
-
-def test_extract_agent_efficiency_metrics_uses_fallback_and_ignores_non_numeric() -> None:
-    metadata = {
-        "tool_efficiency": {
-            "tool_calls_started": 4,
-            "tool_repetition_rate": 0.25,
-            "skill_effective_rate": True,  # bool must be ignored even though bool is an int subclass
-            "tool_calls_unique": "4",  # non-numeric types must be ignored
-            "extra_text": "ignore-me",
-        },
-        "tool_usage": [{"success": False}],
-    }
-
-    summary = _extract_agent_efficiency_metrics(metadata)
-
-    assert summary == {
-        "tool_calls_started": 4,
-        "tool_repetition_rate": 0.25,
-        "skill_executor_calls_total": 1,
-        "skill_executor_success_rate": 0.0,
-    }

package/tools/vds-scripts/audit_orchestrator/tests/test_agents/test_output_validators.py

@@ -1,317 +0,0 @@
-"""Tests for PydanticAI output validators across agent outputs."""
-
-from __future__ import annotations
-
-import json
-from collections.abc import Callable
-from typing import Any
-
-import pytest
-from pydantic_ai import Agent, ModelResponse, ModelRetry, RunContext, TextPart, models
-from pydantic_ai.models.function import FunctionModel
-from pydantic_ai.usage import RequestUsage
-from vds_audit_orchestrator.agents.pydantic_base import (
-    AuditDeps,
-    CodeFindings,
-    DocFindings,
-    SecurityFindings,
-    SynthesisResult,
-)
-
-PayloadFactory = Callable[[float], dict[str, Any]]
-
-
-def _code_payload(confidence: float) -> dict[str, Any]:
-    return {
-        "score": 7.0,
-        "issues": [],
-        "patterns": [],
-        "confidence": confidence,
-        "reasoning": "ok",
-        "files_analyzed": 1,
-        "tech_stack": [],
-    }
-
-
-def _docs_payload(confidence: float) -> dict[str, Any]:
-    return {
-        "readme_score": 7.0,
-        "confluence_score": 6.0,
-        "api_docs_score": 5.0,
-        "notes": [],
-        "confidence": confidence,
-        "missing_sections": [],
-        "strengths": [],
-        "recommendations": [],
-    }
-
-
-def _security_payload(confidence: float) -> dict[str, Any]:
-    return {
-        "vulnerabilities": [],
-        "posture_score": 7.0,
-        "recommendations": [],
-        "confidence": confidence,
-        "sonarqube_issues": 0,
-        "dependency_vulnerabilities": 0,
-        "security_controls": [],
-    }
-
-
-def _synthesis_payload(confidence: float) -> dict[str, Any]:
-    return {
-        "overall_score": 85.0,
-        "maturity_level": "Defined",
-        "gaps": [],
-        "aligned_practices": [],
-        "recommendations": [],
-        "confidence": confidence,
-        "executive_summary": "ok",
-        "component_scores": {},
-        "cross_cutting_concerns": [],
-    }
-
-
-def _create_test_agent(output_type: type[Any], model: FunctionModel) -> Agent[AuditDeps, Any]:
-    """Create a test agent with confidence validation.
-
-    This mirrors the validation logic in create_audit_agent but allows
-    injecting a FunctionModel for testing.
-    """
-    agent: Agent[AuditDeps, Any] = Agent(
-        model=model,
-        name="test-validator",
-        deps_type=AuditDeps,
-        output_type=output_type,
-        instructions="Return structured output",
-        retries=2,
-        output_retries=2,
-    )
-
-    @agent.output_validator
-    def _validate_confidence(ctx: RunContext[AuditDeps], output: Any) -> Any:
-        """Retry if confidence is below threshold."""
-        confidence = getattr(output, "confidence", None)
-        if confidence is not None and confidence < 0.4:
-            raise ModelRetry(f"Low confidence output ({confidence:.2f}); retry with more detailed analysis.")
-        return output
-
-    return agent
-
-
-@pytest.mark.asyncio
-@pytest.mark.parametrize(
-    "output_type,payload_factory",
-    [
-        (CodeFindings, _code_payload),
-        (DocFindings, _docs_payload),
-        (SecurityFindings, _security_payload),
-        (SynthesisResult, _synthesis_payload),
-    ],
-)
-async def test_output_validator_retries_low_confidence(output_type: type[Any], payload_factory: PayloadFactory) -> None:
-    models.ALLOW_MODEL_REQUESTS = False
-
-    outputs = [payload_factory(0.2), payload_factory(0.9)]
-    call_count = {"count": 0}
-
-    def model_fn(messages, info):
-        call_count["count"] += 1
-        payload = outputs.pop(0)
-        return ModelResponse(
-            parts=[TextPart(json.dumps(payload))],
-            usage=RequestUsage(input_tokens=12, output_tokens=34),
-        )
-
-    model = FunctionModel(model_fn)
-    agent = _create_test_agent(output_type, model)
-
-    result = await agent.run("Generate output", deps=AuditDeps())
-
-    assert result.output.confidence == 0.9
-    assert call_count["count"] == 2
-
-
-def test_code_findings_normalizes_common_alias_fields() -> None:
-    payload = {
-        "overall_score": 8.5,
-        "findings": [
-            {
-                "path": "src/service.py",
-                "line": "42",
-                "severity": "MAJOR",
-                "type": "security",
-                "message": "Potential SQL injection",
-                "fix": "Use parameterized queries",
-            }
-        ],
-        "analysis": "Code appears mostly healthy.",
-        "analyzed_files": "7",
-        "tech_stack": "python,fastapi",
-    }
-
-    output = CodeFindings.model_validate(payload)
-
-    assert output.score == 8.5
-    assert output.files_analyzed == 7
-    assert output.tech_stack == ["python", "fastapi"]
-    assert len(output.issues) == 1
-    issue = output.issues[0]
-    assert issue.file_path == "src/service.py"
-    assert issue.line_number == 42
-    assert issue.severity == "high"
-    assert issue.category == "security"
-    assert issue.recommendation == "Use parameterized queries"
-
-
-def test_code_findings_normalizes_text_payload_and_string_lists() -> None:
-    payload = {
-        "score": "6.5",
-        "issues": "Potential null check missing",
-        "patterns": "defensive coding",
-        "analysis": "Model returned a terse payload.",
-        "files_analyzed": "3",
-        "tech_stack": "java,spring",
-    }
-
-    output = CodeFindings.model_validate(payload)
-
-    assert output.score == 6.5
-    assert output.files_analyzed == 3
-    assert output.tech_stack == ["java", "spring"]
-    assert len(output.issues) == 1
-    assert output.issues[0].description == "Potential null check missing"
-    assert len(output.patterns) == 1
-    assert output.patterns[0].name == "defensive coding"
-
-
-def test_code_findings_normalizes_json_string_payload() -> None:
-    payload = json.dumps(
-        {
-            "overall_score": 8.0,
-            "findings": [
-                {
-                    "path": "src/service.java",
-                    "line": "19",
-                    "severity": "MAJOR",
-                    "type": "quality",
-                    "message": "Complex method detected",
-                }
-            ],
-            "summary": "Healthy overall.",
-        }
-    )
-
-    output = CodeFindings.model_validate(payload)
-
-    assert output.score == 8.0
-    assert len(output.issues) == 1
-    assert output.issues[0].file_path == "src/service.java"
-    assert output.issues[0].line_number == 19
-
-
-def test_code_findings_normalizes_confidence_scale_from_ten() -> None:
-    payload = {
-        "score": 7.5,
-        "confidence": 5,
-        "issues": [],
-        "patterns": [],
-    }
-
-    output = CodeFindings.model_validate(payload)
-
-    assert output.confidence == 0.5
-
-
-def test_doc_findings_normalizes_common_alias_fields() -> None:
-    payload = {
-        "score": 7.2,
-        "confluence_quality": 6.5,
-        "api_documentation": 5.8,
-        "notes": "README includes quick start",
-        "gaps": "Troubleshooting,Runbook",
-        "strengths": "Clear installation steps",
-        "actions": "Add API examples",
-    }
-
-    output = DocFindings.model_validate(payload)
-
-    assert output.readme_score == 7.2
-    assert output.confluence_score == 6.5
-    assert output.api_docs_score == 5.8
-    assert output.notes == ["README includes quick start"]
-    assert output.missing_sections == ["Troubleshooting", "Runbook"]
-    assert output.strengths == ["Clear installation steps"]
-    assert output.recommendations == ["Add API examples"]
-
-
-def test_doc_findings_normalizes_json_string_payload() -> None:
-    payload = json.dumps(
-        {
-            "readme_quality": 8.0,
-            "confluence": 4.0,
-            "api_score": 6.0,
-            "notes": ["docs mostly complete"],
-        }
-    )
-
-    output = DocFindings.model_validate(payload)
-
-    assert output.readme_score == 8.0
-    assert output.confluence_score == 4.0
-    assert output.api_docs_score == 6.0
-    assert output.notes == ["docs mostly complete"]
-
-
-def test_doc_findings_normalizes_confidence_scale_from_ten() -> None:
-    payload = {
-        "readme_score": 7.0,
-        "confidence": 8,
-    }
-
-    output = DocFindings.model_validate(payload)
-
-    assert output.confidence == 0.8
-
-
-def test_security_findings_normalizes_common_alias_fields() -> None:
-    payload = {
-        "score": 6.0,
-        "issues": [
-            {
-                "name": "Hardcoded API key",
-                "severity": "BLOCKER",
-                "type": "configuration",
-                "details": "API key found in source code",
-                "component": "src/config.py",
-                "recommendation": "Move key to environment variables",
-            }
-        ],
-        "actions": "Rotate credentials,Add secret scanning",
-        "controls": "JWT auth,input validation",
-    }
-
-    output = SecurityFindings.model_validate(payload)
-
-    assert output.posture_score == 6.0
-    assert output.recommendations == ["Rotate credentials", "Add secret scanning"]
-    assert output.security_controls == ["JWT auth", "input validation"]
-    assert len(output.vulnerabilities) == 1
-    vulnerability = output.vulnerabilities[0]
-    assert vulnerability.id.startswith("SEC-")
-    assert vulnerability.severity == "critical"
-    assert vulnerability.category == "configuration"
-    assert vulnerability.affected_component == "src/config.py"
-    assert vulnerability.remediation == "Move key to environment variables"
-
-
-def test_security_findings_normalizes_confidence_scale_from_ten() -> None:
-    payload = {
-        "posture_score": 6.0,
-        "confidence": 9,
-        "vulnerabilities": [],
-    }
-
-    output = SecurityFindings.model_validate(payload)
-
-    assert output.confidence == 0.9