@ngocsangairvds/vsaf 3.2.14 → 3.2.16

package/tools/vds-scripts/audit_orchestrator/tests/test_agents/test_pydantic_ai_models.py

@@ -1,1219 +0,0 @@
-"""Tests for PydanticAI TestModel and FunctionModel integration."""
-
-from __future__ import annotations
-
-import json
-from types import SimpleNamespace
-
-import pytest
-from pydantic import BaseModel
-from pydantic_ai import ModelResponse, TextPart, models
-from pydantic_ai.models.function import FunctionModel
-from pydantic_ai.models.test import TestModel
-from pydantic_ai.toolsets import FunctionToolset
-from pydantic_ai.usage import RequestUsage
-from vds_audit_orchestrator.agents.base import BaseAgent
-from vds_audit_orchestrator.agents.toolsets.skills_toolset import SkillPolicyDiagnostics, SkillPolicyDrop
-from vds_audit_orchestrator.config import LLMConfig
-from vds_audit_orchestrator.engine.provider_failure_classifier import RowFailoverContext
-from vds_audit_orchestrator.llm.cost_tracker import global_tracker
-from vds_audit_orchestrator.llm.provider import LLMProtocolType, LLMSettings
-
-
-class DummyOutput(BaseModel):
-    status: str
-
-
-class DummyAgent(BaseAgent):
-    async def analyze(self, evidence):
-        raise NotImplementedError("Dummy agent not used in tests")
-
-
-class _LoggerCapture:
-    def __init__(self) -> None:
-        self.records: list[tuple[str, dict[str, object]]] = []
-
-    def info(self, event: str, **kwargs: object) -> None:
-        self.records.append((event, kwargs))
-
-    def warning(self, event: str, **kwargs: object) -> None:
-        self.records.append((event, kwargs))
-
-    def debug(self, event: str, **kwargs: object) -> None:
-        self.records.append((event, kwargs))
-
-
-@pytest.mark.asyncio
-async def test_testmodel_returns_structured_output():
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings()
-    settings.enabled = True
-    settings.use_pydantic_ai = True
-
-    model = TestModel(custom_output_args={"status": "ok"})
-    agent = DummyAgent(settings=settings, model_override=model)
-
-    output = await agent._run_pydantic_agent(
-        system_prompt="Return status",
-        user_prompt="Provide status",
-        result_type=DummyOutput,
-        complexity="standard",
-    )
-
-    assert output.status == "ok"
-
-
-@pytest.mark.asyncio
-async def test_functionmodel_records_usage():
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings()
-    settings.enabled = True
-    settings.use_pydantic_ai = True
-    settings.agent_stream = False
-
-    usage = RequestUsage(input_tokens=12, output_tokens=34)
-
-    async def model_function(messages, info):
-        return ModelResponse(
-            parts=[TextPart(json.dumps({"status": "ok"}))],
-            usage=usage,
-        )
-
-    model = FunctionModel(model_function)
-    agent = DummyAgent(settings=settings, model_override=model)
-
-    before = len(global_tracker.records)
-    output = await agent._run_pydantic_agent(
-        system_prompt="Return status",
-        user_prompt="Provide status",
-        result_type=DummyOutput,
-        complexity="standard",
-        prompt_type="test_usage",
-    )
-    after = len(global_tracker.records)
-
-    assert output.status == "ok"
-    assert after == before + 1
-    record = global_tracker.records[-1]
-    assert record.input_tokens == 12
-    assert record.output_tokens == 34
-
-
-@pytest.mark.asyncio
-async def test_pydantic_agent_persists_telemetry_on_failure():
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings()
-    settings.enabled = True
-    settings.use_pydantic_ai = True
-    settings.agent_stream = False
-
-    async def model_function(messages, info):
-        raise RuntimeError("request_limit exhausted")
-
-    model = FunctionModel(model_function)
-    agent = DummyAgent(settings=settings, model_override=model)
-
-    with pytest.raises(RuntimeError, match="request_limit exhausted"):
-        await agent._run_pydantic_agent(
-            system_prompt="Return status",
-            user_prompt="Provide status",
-            result_type=DummyOutput,
-            complexity="standard",
-        )
-
-    telemetry = agent._last_agent_run_telemetry
-    assert telemetry.get("error") == "request_limit exhausted"
-    assert telemetry.get("protocol") == settings.protocol.value
-    assert telemetry.get("result_type") == "DummyOutput"
-    assert isinstance(telemetry.get("event_tool_names"), list)
-    assert isinstance(telemetry.get("operation_id"), str)
-    assert telemetry.get("operation_kind") == "pydantic_agent_run"
-    assert telemetry.get("operation_status") == "failed"
-    assert telemetry.get("operation_outcome") == "error"
-    assert isinstance(telemetry.get("operation_elapsed_sec"), float)
-
-
-@pytest.mark.asyncio
-async def test_pydantic_agent_operation_lifecycle_logs_success() -> None:
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings()
-    settings.enabled = True
-    settings.use_pydantic_ai = True
-    settings.agent_event_stream_enabled = False
-    settings.agent_stream = False
-
-    async def model_function(messages, info):
-        return ModelResponse(
-            parts=[TextPart(json.dumps({"status": "ok"}))],
-            usage=RequestUsage(input_tokens=3, output_tokens=2),
-        )
-
-    model = FunctionModel(model_function)
-    agent = DummyAgent(settings=settings, model_override=model)
-    capture = _LoggerCapture()
-    agent.logger = capture  # type: ignore[assignment]
-
-    output = await agent._run_pydantic_agent(
-        system_prompt="Return status",
-        user_prompt="Provide status",
-        result_type=DummyOutput,
-        complexity="standard",
-    )
-
-    assert output.status == "ok"
-    telemetry = agent._last_agent_run_telemetry
-    operation_id = telemetry.get("operation_id")
-    lifecycle_logs = [payload for event, payload in capture.records if event == "agent_operation_lifecycle"]
-    assert len(lifecycle_logs) >= 2
-    assert lifecycle_logs[0].get("operation_status") == "started"
-    assert lifecycle_logs[-1].get("operation_status") == "completed"
-    assert lifecycle_logs[-1].get("operation_outcome") == "success"
-    assert lifecycle_logs[-1].get("operation_id") == operation_id
-
-
-@pytest.mark.asyncio
-async def test_pydantic_agent_operation_lifecycle_logs_failure() -> None:
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings()
-    settings.enabled = True
-    settings.use_pydantic_ai = True
-    settings.agent_event_stream_enabled = False
-    settings.agent_stream = False
-
-    async def model_function(messages, info):
-        raise RuntimeError("operation failed")
-
-    model = FunctionModel(model_function)
-    agent = DummyAgent(settings=settings, model_override=model)
-    capture = _LoggerCapture()
-    agent.logger = capture  # type: ignore[assignment]
-
-    with pytest.raises(RuntimeError, match="operation failed"):
-        await agent._run_pydantic_agent(
-            system_prompt="Return status",
-            user_prompt="Provide status",
-            result_type=DummyOutput,
-            complexity="standard",
-        )
-
-    lifecycle_logs = [payload for event, payload in capture.records if event == "agent_operation_lifecycle"]
-    assert len(lifecycle_logs) >= 2
-    assert lifecycle_logs[0].get("operation_status") == "started"
-    assert lifecycle_logs[-1].get("operation_status") == "failed"
-    assert lifecycle_logs[-1].get("operation_outcome") == "error"
-    assert "operation failed" in str(lifecycle_logs[-1].get("error"))
-
-
-@pytest.mark.asyncio
-async def test_non_stream_run_backfills_telemetry_from_result(monkeypatch):
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings()
-    settings.enabled = True
-    settings.use_pydantic_ai = True
-    settings.agent_event_stream_enabled = False
-    settings.agent_stream = False
-
-    async def model_function(messages, info):
-        return ModelResponse(
-            parts=[TextPart(json.dumps({"status": "ok"}))],
-            usage=RequestUsage(input_tokens=1, output_tokens=1),
-        )
-
-    model = FunctionModel(model_function)
-    agent = DummyAgent(settings=settings, model_override=model)
-    calls = {"count": 0}
-    original = BaseAgent._populate_telemetry_from_result
-
-    def _wrapped_populate(self, result, run_telemetry):
-        calls["count"] += 1
-        run_telemetry["telemetry_backfill_seen"] = True
-        return original(self, result, run_telemetry)
-
-    monkeypatch.setattr(BaseAgent, "_populate_telemetry_from_result", _wrapped_populate)
-
-    output = await agent._run_pydantic_agent(
-        system_prompt="Return status",
-        user_prompt="Provide status",
-        result_type=DummyOutput,
-        complexity="standard",
-    )
-
-    telemetry = agent._last_agent_run_telemetry
-    assert output.status == "ok"
-    assert calls["count"] == 1
-    assert telemetry.get("telemetry_backfill_seen") is True
-
-
-@pytest.mark.asyncio
-async def test_pydantic_agent_emits_skill_policy_diagnostics_in_runtime_telemetry() -> None:
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings()
-    settings.enabled = True
-    settings.use_pydantic_ai = True
-    settings.agent_trace_payload_enabled = True
-    settings.agent_stream = False
-
-    async def model_function(messages, info):
-        return ModelResponse(
-            parts=[TextPart(json.dumps({"status": "ok"}))],
-            usage=RequestUsage(input_tokens=7, output_tokens=3),
-        )
-
-    model = FunctionModel(model_function)
-    agent = DummyAgent(settings=settings, model_override=model)
-    capture = _LoggerCapture()
-    agent.logger = capture  # type: ignore[assignment]
-
-    def read_skill_resource(resource_name: str) -> str:
-        return resource_name
-
-    toolset = FunctionToolset([read_skill_resource])
-    toolset._vds_skill_policy_diagnostics = SkillPolicyDiagnostics(
-        metadata_count=3,
-        policy_eligible_count=2,
-        runtime_loaded_count=1,
-        dropped=[
-            SkillPolicyDrop(skill_name="blocked-skill", reason="trust_tier_rejected", runtime_name="blocked-skill")
-        ],
-    )
-
-    output = await agent._run_pydantic_agent(
-        system_prompt="Return status",
-        user_prompt="Provide status",
-        result_type=DummyOutput,
-        complexity="standard",
-        toolsets=[toolset],
-    )
-
-    assert output.status == "ok"
-    telemetry = agent._last_agent_run_telemetry
-    diagnostics = telemetry.get("skill_policy_diagnostics")
-    assert isinstance(diagnostics, dict)
-    assert diagnostics.get("metadata_count") == 3
-    assert diagnostics.get("policy_eligible_count") == 2
-    assert diagnostics.get("runtime_loaded_count") == 1
-    drop_reasons = diagnostics.get("drop_reasons")
-    assert isinstance(drop_reasons, list)
-    assert drop_reasons[0]["skill_name"] == "blocked-skill"
-    assert drop_reasons[0]["reason"] == "trust_tier_rejected"
-
-    diagnostics_log = next(
-        (payload for event, payload in capture.records if event == "pydantic_agent_skill_policy_diagnostics"),
-        None,
-    )
-    assert isinstance(diagnostics_log, dict)
-    assert diagnostics_log.get("metadata_count") == 3
-    assert diagnostics_log.get("policy_eligible_count") == 2
-    assert diagnostics_log.get("runtime_loaded_count") == 1
-
-    trace_response = next(
-        (payload for event, payload in capture.records if event == "pydantic_agent_trace_response"),
-        None,
-    )
-    assert isinstance(trace_response, dict)
-    trace_payload = trace_response.get("payload")
-    assert isinstance(trace_payload, dict)
-    trace_diagnostics = trace_payload.get("skill_policy_diagnostics")
-    assert isinstance(trace_diagnostics, dict)
-    assert trace_diagnostics.get("metadata_count") == 3
-
-
-def test_retry_policy_uses_configured_values_for_non_anthropic_protocol() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.OPENAI,
-        agent_retries=1,
-        output_retries=0,
-    )
-    agent = DummyAgent(settings=settings)
-
-    retries, output_retries = agent._resolve_agent_retry_policy()
-
-    assert retries == 1
-    assert output_retries == 0
-
-
-def test_retry_policy_applies_anthropic_output_retry_floor() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.ANTHROPIC,
-        model_standard="claude-sonnet-4-5",
-        agent_retries=0,
-        output_retries=0,
-    )
-    agent = DummyAgent(settings=settings)
-
-    retries, output_retries = agent._resolve_agent_retry_policy()
-
-    assert retries == 0
-    assert output_retries == 4
-
-
-def test_retry_policy_does_not_force_floor_for_non_claude_anthropic_proxy() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.ANTHROPIC,
-        model_standard="z-ai/glm5",
-        agent_retries=0,
-        output_retries=1,
-    )
-    agent = DummyAgent(settings=settings)
-
-    retries, output_retries = agent._resolve_agent_retry_policy()
-
-    assert retries == 0
-    assert output_retries == 1
-
-
-def test_retry_policy_applies_proxy_retry_floor_for_non_official_anthropic_endpoint() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.ANTHROPIC,
-        base_url="http://localhost:8082",
-        model_standard="z-ai/glm5",
-        agent_retries=0,
-        output_retries=0,
-    )
-    agent = DummyAgent(settings=settings)
-
-    retries, output_retries = agent._resolve_agent_retry_policy()
-
-    assert retries == 3
-    assert output_retries == 2
-
-
-def test_retry_policy_does_not_apply_proxy_floor_for_official_anthropic_endpoint() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.ANTHROPIC,
-        base_url="https://api.anthropic.com",
-        model_standard="claude-sonnet-4-5",
-        agent_retries=1,
-        output_retries=1,
-    )
-    agent = DummyAgent(settings=settings)
-
-    retries, output_retries = agent._resolve_agent_retry_policy()
-
-    assert retries == 1
-    # official claude floor still applies
-    assert output_retries == 4
-
-
-def test_build_model_settings_suppresses_tool_choice_for_anthropic_proxy() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.ANTHROPIC,
-        base_url="http://localhost:8082",
-        model_standard="z-ai/glm5",
-    )
-    agent = DummyAgent(settings=settings)
-
-    model_settings = agent._build_model_settings()
-    assert model_settings.get("extra_body") == {"tool_choice": None}
-
-
-def test_build_model_settings_keeps_official_anthropic_endpoint_unchanged() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.ANTHROPIC,
-        base_url="https://api.anthropic.com",
-        model_standard="claude-sonnet-4-5",
-    )
-    agent = DummyAgent(settings=settings)
-
-    model_settings = agent._build_model_settings()
-    assert "extra_body" not in model_settings
-
-
-def test_build_model_settings_disables_parallel_tools_for_local_openai() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.OPENAI,
-        base_url="http://127.0.0.1:11434/v1",
-        model_standard="glm-4.7-flash",
-    )
-    agent = DummyAgent(settings=settings)
-
-    model_settings = agent._build_model_settings()
-    assert model_settings.get("parallel_tool_calls") is False
-
-
-def test_build_model_settings_suppresses_tool_choice_for_dashscope_openai() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.OPENAI,
-        base_url="https://coding-intl.dashscope.aliyuncs.com/v1",
-        model_standard="qwen3.5-plus",
-    )
-    agent = DummyAgent(settings=settings)
-
-    model_settings = agent._build_model_settings()
-    assert model_settings.get("extra_body") == {"tool_choice": None}
-
-
-def test_build_model_settings_keeps_parallel_tools_for_non_local_openai() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.OPENAI,
-        base_url="https://api.openai.com/v1",
-        model_standard="gpt-4o-mini",
-    )
-    agent = DummyAgent(settings=settings)
-
-    model_settings = agent._build_model_settings()
-    assert "parallel_tool_calls" not in model_settings
-    assert "openai_reasoning_effort" not in model_settings
-
-
-def test_build_model_settings_includes_openai_reasoning_effort_when_configured() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.OPENAI,
-        base_url="https://api.openai.com/v1",
-        model_standard="gpt-4o-mini",
-        reasoning_effort="high",
-    )
-    agent = DummyAgent(settings=settings)
-
-    model_settings = agent._build_model_settings()
-    assert model_settings.get("openai_reasoning_effort") == "high"
-
-
-def test_build_run_model_settings_disables_parallel_tools_for_local_openai() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.OPENAI,
-        base_url="http://127.0.0.1:11434/v1",
-        model_standard="glm-4.7-flash",
-    )
-    agent = DummyAgent(settings=settings)
-
-    run_settings = agent._build_run_model_settings(2048)
-    assert run_settings["parallel_tool_calls"] is False
-    assert run_settings["max_tokens"] == 2048
-
-
-def test_build_run_model_settings_suppresses_tool_choice_for_dashscope_openai() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.OPENAI,
-        base_url="https://coding-intl.dashscope.aliyuncs.com/v1",
-        model_standard="qwen3.5-plus",
-    )
-    agent = DummyAgent(settings=settings)
-
-    run_settings = agent._build_run_model_settings(2048)
-    assert run_settings["extra_body"] == {"tool_choice": None}
-    assert run_settings["max_tokens"] == 2048
-
-
-def test_build_run_model_settings_keeps_parallel_tools_for_non_local_openai() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.OPENAI,
-        base_url="https://api.openai.com/v1",
-        model_standard="gpt-4o-mini",
-    )
-    agent = DummyAgent(settings=settings)
-
-    run_settings = agent._build_run_model_settings(1024)
-    assert "parallel_tool_calls" not in run_settings
-    assert "openai_reasoning_effort" not in run_settings
-    assert run_settings["max_tokens"] == 1024
-
-
-def test_build_run_model_settings_includes_openai_reasoning_effort_when_configured() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.OPENAI,
-        base_url="https://api.openai.com/v1",
-        model_standard="gpt-4o-mini",
-        reasoning_effort="xhigh",
-    )
-    agent = DummyAgent(settings=settings)
-
-    run_settings = agent._build_run_model_settings(1024)
-    assert run_settings["openai_reasoning_effort"] == "xhigh"
-
-
-@pytest.mark.parametrize(
-    ("protocol", "base_url", "model_name", "reasoning_effort"),
-    [
-        (LLMProtocolType.OPENAI, "https://api.openai.com/v1", "gpt-4o-mini", "high"),
-        (LLMProtocolType.OPENAI_CODEX, "https://chatgpt.com/backend-api/codex", "gpt-5.3-codex", "minimal"),
-    ],
-)
-def test_reasoning_effort_propagates_across_openai_and_openai_codex_settings_paths(
-    protocol: LLMProtocolType,
-    base_url: str,
-    model_name: str,
-    reasoning_effort: str,
-) -> None:
-    """Integration-style cross-provider check for FR-349 injection behavior."""
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=protocol,
-        base_url=base_url,
-        model_standard=model_name,
-        reasoning_effort=reasoning_effort,
-    )
-    agent = DummyAgent(settings=settings)
-
-    model_settings = agent._build_model_settings()
-    run_settings = agent._build_run_model_settings(1024)
-
-    assert model_settings.get("openai_reasoning_effort") == reasoning_effort
-    assert run_settings.get("openai_reasoning_effort") == reasoning_effort
-
-
-def test_resolve_pydantic_usage_limits_expands_request_budget_for_tool_calls() -> None:
-    settings = LLMSettings(enabled=False)
-    agent = DummyAgent(settings=settings)
-
-    request_limit, tool_calls_limit = agent._resolve_pydantic_usage_limits(
-        max_turns=10,
-        max_tool_calls=12,
-    )
-
-    assert request_limit == 14
-    assert tool_calls_limit == 12
-
-
-def test_resolve_pydantic_usage_limits_keeps_higher_turn_budget() -> None:
-    settings = LLMSettings(enabled=False)
-    agent = DummyAgent(settings=settings)
-
-    request_limit, tool_calls_limit = agent._resolve_pydantic_usage_limits(
-        max_turns=30,
-        max_tool_calls=12,
-    )
-
-    assert request_limit == 30
-    assert tool_calls_limit == 12
-
-
-def test_resolve_pydantic_usage_limits_zero_tool_mode_disables_limits() -> None:
-    settings = LLMSettings(enabled=False)
-    agent = DummyAgent(settings=settings)
-
-    request_limit, tool_calls_limit = agent._resolve_pydantic_usage_limits(
-        max_turns=None,
-        max_tool_calls=0,
-    )
-
-    assert request_limit is None
-    assert tool_calls_limit == 0
-
-
-def test_resolve_pydantic_usage_limits_negative_tool_mode_disables_limits() -> None:
-    settings = LLMSettings(enabled=False)
-    agent = DummyAgent(settings=settings)
-
-    request_limit, tool_calls_limit = agent._resolve_pydantic_usage_limits(
-        max_turns=None,
-        max_tool_calls=-1,
-    )
-
-    assert request_limit is None
-    assert tool_calls_limit is None
-
-
-def test_resolve_pydantic_usage_limits_zero_tool_with_turn_budget_keeps_request_limit() -> None:
-    settings = LLMSettings(enabled=False)
-    agent = DummyAgent(settings=settings)
-
-    request_limit, tool_calls_limit = agent._resolve_pydantic_usage_limits(
-        max_turns=4,
-        max_tool_calls=0,
-    )
-
-    assert request_limit == 4
-    assert tool_calls_limit == 0
-
-
-def test_detects_tool_choice_protocol_mismatch_from_cause() -> None:
-    try:
-        try:
-            raise ValueError("Invalid value for `function`: `None` in `tool_choice`")
-        except ValueError as inner:
-            raise RuntimeError("Exceeded maximum retries (1) for output validation") from inner
-    except RuntimeError as exc:
-        assert DummyAgent._is_tool_choice_protocol_mismatch(exc)
-
-
-def test_does_not_flag_non_tool_choice_mismatch() -> None:
-    exc = RuntimeError("Exceeded maximum retries (1) for output validation")
-    assert not DummyAgent._is_tool_choice_protocol_mismatch(exc)
-
-
-def test_detects_rate_limit_error_from_message() -> None:
-    exc = RuntimeError("status_code: 429, body: {'error': {'type': 'rate_limit_error'}}")
-    assert DummyAgent._is_rate_limit_error(exc)
-
-
-def test_detects_rate_limit_error_from_cause_message() -> None:
-    try:
-        try:
-            raise ValueError("Rate limit exceeded")
-        except ValueError as inner:
-            raise RuntimeError("provider error") from inner
-    except RuntimeError as exc:
-        assert DummyAgent._is_rate_limit_error(exc)
-
-
-def test_detects_auth_error_from_message() -> None:
-    exc = RuntimeError("status_code: 401, body: {'error': {'type': 'authentication_error'}}")
-    assert DummyAgent._is_auth_error(exc)
-
-
-def test_detects_auth_error_from_cause_message() -> None:
-    try:
-        try:
-            raise ValueError("401 Unauthorized: invalid_api_key")
-        except ValueError as inner:
-            raise RuntimeError("provider error") from inner
-    except RuntimeError as exc:
-        assert DummyAgent._is_auth_error(exc)
-
-
-def test_detects_tool_call_argument_error_from_missing_required_field() -> None:
-    try:
-        try:
-            raise ValueError(
-                "1 validation error for search_evidence\nquery\n  Field required [type=missing, input_value={}, input_type=dict]"
-            )
-        except ValueError as inner:
-            raise RuntimeError("Tool 'search_evidence' exceeded max retries count of 1") from inner
-    except RuntimeError as exc:
-        assert DummyAgent._is_tool_call_argument_error(exc)
-
-
-def test_detects_tool_call_argument_error_from_malformed_json() -> None:
-    exc = RuntimeError("EOF while parsing an object at line 1 column 63")
-    assert DummyAgent._is_tool_call_argument_error(exc)
-
-
-def test_detects_retryable_provider_error_from_503() -> None:
-    exc = RuntimeError("status_code: 503, body: {'error': {'type': 'service_unavailable'}}")
-    assert DummyAgent._is_rate_limit_error(exc)
-
-
-def test_does_not_flag_non_tool_call_argument_error() -> None:
-    exc = RuntimeError("request_limit exhausted")
-    assert not DummyAgent._is_tool_call_argument_error(exc)
-
-
-def test_normalize_agent_output_parses_anthropic_sse_payload() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.ANTHROPIC,
-        stream=True,
-    )
-    agent = DummyAgent(settings=settings)
-    sse_payload = (
-        "event: message_start\n"
-        'data: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[]}}\n\n'
-        "event: content_block_delta\n"
-        'data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"{\\"status\\":\\"ok\\"}"}}\n\n'
-        "event: message_stop\n"
-        'data: {"type":"message_stop"}\n\n'
-    )
-
-    output = agent._normalize_agent_output(sse_payload, DummyOutput)
-
-    assert isinstance(output, DummyOutput)
-    assert output.status == "ok"
-
-
-def test_normalize_agent_output_rejects_invalid_anthropic_sse_payload() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.ANTHROPIC,
-        stream=True,
-    )
-    agent = DummyAgent(settings=settings)
-    invalid_sse_payload = "event: content_block_delta\ndata: not-json\n\n"
-
-    with pytest.raises(RuntimeError, match="Invalid SSE JSON payload"):
-        agent._normalize_agent_output(invalid_sse_payload, DummyOutput)
-
-
-def test_normalize_strict_anthropic_sse_payload_accepts_done_terminal_marker() -> None:
-    payload = (
-        "event: content_block_delta\n"
-        'data: {"type":"content_block_delta","delta":{"type":"text_delta","text":"hello"}}\n\n'
-        "[DONE]\n"
-    )
-    normalized = DummyAgent._normalize_strict_anthropic_sse_payload(payload)
-    assert normalized == "hello"
-
-
-@pytest.mark.asyncio
-async def test_call_llm_uses_stream_flag_and_normalizes_anthropic_stream_events() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=False,
-        protocol=LLMProtocolType.ANTHROPIC,
-        model_standard="claude-sonnet-4-5",
-        stream=True,
-    )
-    agent = DummyAgent(settings=settings)
-
-    class _Messages:
-        def __init__(self) -> None:
-            self.kwargs: dict[str, object] | None = None
-
-        def create(self, **kwargs):
-            self.kwargs = kwargs
-            return [
-                {"type": "content_block_delta", "delta": {"type": "text_delta", "text": "hello "}},
-                {"type": "content_block_delta", "delta": {"type": "text_delta", "text": "world"}},
-                {"type": "message_stop"},
-            ]
-
-    class _Client:
-        def __init__(self) -> None:
-            self.messages = _Messages()
-
-    client = _Client()
-    agent.client = client
-
-    output = await agent._call_llm("sys", "user")
-
-    assert output == "hello world"
-    assert client.messages.kwargs is not None
-    assert client.messages.kwargs.get("stream") is True
-
-
-@pytest.mark.asyncio
-async def test_call_llm_logs_marker_for_non_stream_sse_adapter_path() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=False,
-        protocol=LLMProtocolType.ANTHROPIC,
-        model_standard="claude-sonnet-4-5",
-        stream=False,
-    )
-    agent = DummyAgent(settings=settings)
-    capture = _LoggerCapture()
-    agent.logger = capture  # type: ignore[assignment]
-
-    class _Messages:
-        def __init__(self) -> None:
-            self.kwargs: dict[str, object] | None = None
-
-        def create(self, **kwargs):
-            self.kwargs = kwargs
-            return [
-                {"type": "content_block_delta", "delta": {"type": "text_delta", "text": "hello "}},
-                {"type": "content_block_delta", "delta": {"type": "text_delta", "text": "world"}},
-                {"type": "message_stop"},
-            ]
-
-    class _Client:
-        def __init__(self) -> None:
-            self.messages = _Messages()
-
-    client = _Client()
-    agent.client = client
-
-    output = await agent._call_llm("sys", "user")
-
-    assert output == "hello world"
-    assert client.messages.kwargs is not None
-    assert "stream" not in client.messages.kwargs
-    marker_payload = next(
-        (payload for event, payload in capture.records if event == "anthropic_non_stream_sse_adapter_used"),
-        {},
-    )
-    assert marker_payload.get("configured_stream") is False
-    assert marker_payload.get("adapter_path") == "anthropic_sse_stream_adapter"
-
-
-def test_normalize_agent_output_logs_marker_for_strict_no_fallback_parse_failure() -> None:
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.ANTHROPIC,
-        stream=False,
-    )
-    agent = DummyAgent(settings=settings)
-    capture = _LoggerCapture()
-    agent.logger = capture  # type: ignore[assignment]
-    invalid_sse_payload = "event: content_block_delta\ndata: not-json\n\n"
-
-    with pytest.raises(RuntimeError, match="Invalid SSE JSON payload"):
-        agent._normalize_agent_output(invalid_sse_payload, DummyOutput)
-
-    adapter_marker_payload = next(
-        (payload for event, payload in capture.records if event == "anthropic_non_stream_sse_adapter_used"),
-        {},
-    )
-    failure_marker_payload = next(
-        (payload for event, payload in capture.records if event == "anthropic_non_stream_sse_parse_failed_no_fallback"),
-        {},
-    )
-    assert adapter_marker_payload.get("configured_stream") is False
-    assert failure_marker_payload.get("fallback_allowed") is False
-    assert failure_marker_payload.get("adapter_path") == "anthropic_sse_stream_adapter"
-    assert failure_marker_payload.get("source") == "normalize_agent_output:model"
-
-
-@pytest.mark.asyncio
-async def test_pydantic_agent_retries_rate_limit_until_success() -> None:
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        agent_event_stream_enabled=False,
-        agent_stream=False,
-        agent_rate_limit_retry_attempts=2,
-        agent_rate_limit_retry_backoff_seconds=0.1,
-    )
-    attempts = {"count": 0}
-
-    async def model_function(messages, info):
-        attempts["count"] += 1
-        if attempts["count"] <= 2:
-            raise RuntimeError("status_code: 429, body: {'error': {'type': 'rate_limit_error'}}")
-        return ModelResponse(
-            parts=[TextPart(json.dumps({"status": "ok"}))],
-            usage=RequestUsage(input_tokens=1, output_tokens=1),
-        )
-
-    model = FunctionModel(model_function)
-    agent = DummyAgent(settings=settings, model_override=model)
-
-    output = await agent._run_pydantic_agent(
-        system_prompt="Return status",
-        user_prompt="Provide status",
-        result_type=DummyOutput,
-        complexity="standard",
-    )
-
-    assert output.status == "ok"
-    assert attempts["count"] == 3
-
-
-@pytest.mark.asyncio
-async def test_pydantic_agent_rate_limit_retry_exhausted() -> None:
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        agent_event_stream_enabled=False,
-        agent_stream=False,
-        agent_rate_limit_retry_attempts=1,
-        agent_rate_limit_retry_backoff_seconds=0.1,
-    )
-    attempts = {"count": 0}
-
-    async def model_function(messages, info):
-        attempts["count"] += 1
-        raise RuntimeError("status_code: 429, body: {'error': {'type': 'rate_limit_error'}}")
-
-    model = FunctionModel(model_function)
-    agent = DummyAgent(settings=settings, model_override=model)
-
-    with pytest.raises(RuntimeError, match="rate_limit_error"):
-        await agent._run_pydantic_agent(
-            system_prompt="Return status",
-            user_prompt="Provide status",
-            result_type=DummyOutput,
-            complexity="standard",
-        )
-
-    assert attempts["count"] == 2
-
-
-@pytest.mark.asyncio
-async def test_pydantic_agent_retries_transient_provider_error_until_success() -> None:
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        agent_event_stream_enabled=False,
-        agent_stream=False,
-        agent_rate_limit_retry_attempts=1,
-        agent_rate_limit_retry_backoff_seconds=0.1,
-    )
-    attempts = {"count": 0}
-
-    async def model_function(messages, info):
-        attempts["count"] += 1
-        if attempts["count"] == 1:
-            raise RuntimeError("status_code: 503, body: {'error': {'type': 'service_unavailable'}}")
-        return ModelResponse(
-            parts=[TextPart(json.dumps({"status": "ok"}))],
-            usage=RequestUsage(input_tokens=1, output_tokens=1),
-        )
-
-    model = FunctionModel(model_function)
-    agent = DummyAgent(settings=settings, model_override=model)
-
-    output = await agent._run_pydantic_agent(
-        system_prompt="Return status",
-        user_prompt="Provide status",
-        result_type=DummyOutput,
-        complexity="standard",
-    )
-
-    assert output.status == "ok"
-    assert attempts["count"] == 2
-
-
-@pytest.mark.asyncio
-async def test_pydantic_agent_fails_over_to_fallback_profile_after_transient_retry_exhaustion(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMConfig(  # type: ignore[arg-type]
-        enabled=True,
-        use_pydantic_ai=True,
-        agent_event_stream_enabled=False,
-        agent_stream=False,
-        agent_rate_limit_retry_attempts=0,
-        row_failover_profiles=["fallback-openai"],
-        row_failover_max_provider_hops=1,
-        model_standard="primary-model",
-    )
-    monkeypatch.setenv("VDS_AUDIT_ACTIVE_PROFILE", "primary-openai")
-    monkeypatch.setenv("VDS_AUDIT_LLM__ROW_FAILOVER_PROFILES", "")
-
-    primary_attempts = {"count": 0}
-    fallback_attempts = {"count": 0}
-
-    async def primary_model(messages, info):
-        primary_attempts["count"] += 1
-        raise RuntimeError("status_code: 503, body: {'error': {'type': 'service_unavailable'}}")
-
-    async def fallback_model(messages, info):
-        fallback_attempts["count"] += 1
-        return ModelResponse(
-            parts=[TextPart(json.dumps({"status": "ok"}))],
-            usage=RequestUsage(input_tokens=1, output_tokens=1),
-        )
-
-    primary = FunctionModel(primary_model)
-    fallback = FunctionModel(fallback_model)
-    agent = DummyAgent(settings=settings, model_override=primary)
-
-    monkeypatch.setattr(
-        "vds_audit_orchestrator.agents.base.inherit_runtime_llm_policy",
-        lambda profile_name, *, source_llm=None: LLMSettings(
-            enabled=True,
-            use_pydantic_ai=True,
-            agent_event_stream_enabled=False,
-            agent_stream=False,
-            model_standard=f"{profile_name}-model",
-        ),
-    )
-
-    def _build_model_for_profile(model_name: str):
-        if model_name == "fallback-openai-model":
-            return fallback
-        return primary
-
-    monkeypatch.setattr(agent, "_build_pydantic_ai_model", _build_model_for_profile)
-
-    output = await agent._run_pydantic_agent(
-        system_prompt="Return status",
-        user_prompt="Provide status",
-        result_type=DummyOutput,
-        complexity="standard",
-        prompt_type="row_eval",
-    )
-
-    assert output.status == "ok"
-    assert primary_attempts["count"] == 1
-    assert fallback_attempts["count"] == 1
-    assert agent._last_agent_run_telemetry["provider_failover_attempted"] is True
-    assert agent._last_agent_run_telemetry["provider_failover_final_provider"] == "fallback-openai"
-    assert agent._last_agent_run_telemetry["provider_failover_chain"] == [
-        "primary-openai",
-        "fallback-openai",
-    ]
-
-
-@pytest.mark.asyncio
-async def test_pydantic_agent_marks_failover_exhausted_when_no_fallback_remains(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMConfig(  # type: ignore[arg-type]
-        enabled=True,
-        use_pydantic_ai=True,
-        agent_event_stream_enabled=False,
-        agent_stream=False,
-        agent_rate_limit_retry_attempts=0,
-        row_failover_profiles=["fallback-openai", "dead-openai"],
-        row_failover_max_provider_hops=1,
-        model_standard="primary-model",
-    )
-    monkeypatch.setenv("VDS_AUDIT_ACTIVE_PROFILE", "primary-openai")
-    monkeypatch.setenv("VDS_AUDIT_ROW_FAILOVER_AVAILABLE_PROFILES", "fallback-openai")
-    monkeypatch.setenv("VDS_AUDIT_LLM__ROW_FAILOVER_PROFILES", "")
-
-    async def primary_model(messages, info):
-        raise RuntimeError("status_code: 503, body: {'error': {'type': 'service_unavailable'}}")
-
-    primary = FunctionModel(primary_model)
-    agent = DummyAgent(settings=settings, model_override=primary)
-
-    with pytest.raises(RuntimeError):
-        await agent._run_pydantic_agent(
-            system_prompt="Return status",
-            user_prompt="Provide status",
-            result_type=DummyOutput,
-            complexity="standard",
-            prompt_type="row_eval",
-            row_failover_context=RowFailoverContext(
-                row_id="row-1",
-                check_id="CL-011",
-                original_provider="primary-openai",
-                failover_count=1,
-                failover_chain=["primary-openai", "fallback-openai"],
-            ),
-        )
-
-    telemetry = agent._last_agent_run_telemetry
-    assert telemetry["timeout_kind"] == "timeout_failover_exhausted"
-    assert telemetry["provider_failover_final_provider"] == "primary-openai"
-    assert telemetry["provider_failover_profiles_considered"] == ["fallback-openai"]
-
-
-@pytest.mark.asyncio
-async def test_pydantic_agent_retries_malformed_tool_call_json_until_success() -> None:
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        agent_event_stream_enabled=False,
-        agent_stream=False,
-        agent_tool_call_retry_attempts=1,
-        agent_tool_call_retry_backoff_seconds=0.0,
-    )
-    attempts = {"count": 0}
-
-    async def model_function(messages, info):
-        attempts["count"] += 1
-        if attempts["count"] == 1:
-            raise RuntimeError("EOF while parsing an object at line 1 column 63")
-        return ModelResponse(
-            parts=[TextPart(json.dumps({"status": "ok"}))],
-            usage=RequestUsage(input_tokens=1, output_tokens=1),
-        )
-
-    model = FunctionModel(model_function)
-    agent = DummyAgent(settings=settings, model_override=model)
-
-    output = await agent._run_pydantic_agent(
-        system_prompt="Return status",
-        user_prompt="Provide status",
-        result_type=DummyOutput,
-        complexity="standard",
-    )
-
-    assert output.status == "ok"
-    assert attempts["count"] == 2
-
-
-@pytest.mark.asyncio
-async def test_pydantic_agent_openai_codex_auth_retry_refreshes_and_recovers(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    models.ALLOW_MODEL_REQUESTS = False
-
-    settings = LLMSettings(
-        enabled=True,
-        use_pydantic_ai=True,
-        protocol=LLMProtocolType.OPENAI_CODEX,
-        agent_event_stream_enabled=False,
-    )
-    agent = DummyAgent(settings=settings)
-    agent._pydantic_model = TestModel(custom_output_args={"status": "ok"})
-
-    calls = {"dispatch": 0, "operation": 0, "refresh": 0, "rebuild_model": 0}
-
-    class _FakeRunResult:
-        output = {"status": "ok"}
-
-        def usage(self) -> RequestUsage:
-            return RequestUsage(input_tokens=1, output_tokens=1)
-
-        def all_messages(self, *, output_tool_return_content: str | None = None) -> list[object]:
-            return []
-
-    def _fake_build_agent_operation(
-        self: BaseAgent,
-        *,
-        agent: object,
-        user_prompt: str,
-        run_kwargs: dict[str, object],
-    ):
-        calls["dispatch"] += 1
-
-        async def _operation():
-            calls["operation"] += 1
-            if calls["operation"] == 1:
-                raise RuntimeError("status_code: 401 unauthorized")
-            return _FakeRunResult()
-
-        return _operation()
-
-    def _refresh_stub(*_args: object, **_kwargs: object) -> SimpleNamespace:
-        calls["refresh"] += 1
-        return SimpleNamespace(access_token="token-new", account_id="acct-1", is_refreshed=True)
-
-    def _rebuild_model(self: BaseAgent, model_name: str):
-        calls["rebuild_model"] += 1
-        return TestModel(custom_output_args={"status": "ok"})
-
-    monkeypatch.setattr(BaseAgent, "_build_agent_operation", _fake_build_agent_operation)
-    monkeypatch.setattr("vds_audit_orchestrator.llm.codex_oauth.force_refresh_codex_credentials", _refresh_stub)
-    monkeypatch.setattr(BaseAgent, "_build_pydantic_ai_model", _rebuild_model)
-
-    output = await agent._run_pydantic_agent(
-        system_prompt="Return status",
-        user_prompt="Provide status",
-        result_type=DummyOutput,
-        complexity="standard",
-    )
-
-    assert output.status == "ok"
-    assert calls["refresh"] == 1
-    assert calls["rebuild_model"] == 1
-    assert calls["operation"] == 2
-    assert calls["dispatch"] == 2