@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/audit_orchestrator/src/vds_audit_orchestrator/llm/engine.py

@@ -1,751 +0,0 @@
-"""Unified LLM Engine for Audit Orchestrator (TSK-040).
-
-This module provides the main LLMEngine class that unifies all LLM operations
-for the audit orchestrator. It serves as the primary interface for:
-- Template analysis (extracting evidence requirements from criteria)
-- Content evaluation (evaluating documentation, code, configs)
-- Gap analysis (comparing evidence against requirements)
-- Documentation quality checks (README, API docs, security docs)
-
-Reference: design.md Section 5.1
-"""
-
-from __future__ import annotations
-
-import asyncio
-import json
-from typing import TYPE_CHECKING, Any
-
-from structlog import get_logger
-from vds_agent_core.llm.cache import LLMResponseCache
-from vds_agent_core.llm.schemas import ProbeScore
-
-from vds_audit_orchestrator.llm.audit_schemas import (
-    AlignedPracticeFinding,
-    ApplicabilitySchema,
-    ContentEvaluationResult,
-    GapAnalysisResult,
-    GapFinding,
-    TemplateAnalysisResult,
-)
-from vds_audit_orchestrator.llm.cost_tracker import UsageSummary, global_tracker
-from vds_audit_orchestrator.llm.evaluator import LLMMultiProbeEvaluator
-from vds_audit_orchestrator.llm.prompts.template_analysis import TEMPLATE_ANALYSIS_SYSTEM, TEMPLATE_ANALYSIS_USER
-from vds_audit_orchestrator.llm.provider import LLMSettings
-from vds_audit_orchestrator.observability.metrics import record_llm_usage
-
-if TYPE_CHECKING:
-    from vds_audit_orchestrator.models.evidence import EvidenceBundle
-
-logger = get_logger(__name__)
-
-
-class LLMEngine:
-    """Unified LLM evaluation engine for audit orchestrator.
-
-    The LLMEngine provides a consistent interface for all LLM-based operations
-    in the audit orchestrator. It handles:
-    - Template analysis for extracting evidence requirements
-    - Content evaluation for documentation and code quality
-    - Gap analysis for compliance checking
-    - Specialized documentation evaluations (README, API, security)
-
-    All methods are async and include built-in caching, cost tracking,
-    and graceful error handling with fallback results.
-
-    Example:
-        >>> engine = LLMEngine()
-        >>> result = await engine.analyze_template("Project must have README")
-        >>> print(result.evidence_requirements)
-    """
-
-    def __init__(self, settings: LLMSettings | None = None):
-        """Initialize the LLM engine.
-
-        Args:
-            settings: LLM configuration settings. If None, uses default settings.
-        """
-        self.settings = settings or LLMSettings()
-        self.cache = LLMResponseCache()
-        self.cost_tracker = global_tracker
-
-        # Initialize client if enabled
-        if self.settings.enabled:
-            try:
-                self.client = self.settings.get_client()
-            except Exception as e:
-                logger.warning("llm_client_init_failed", error=str(e))
-                self.client = None
-        else:
-            self.client = None
-
-        # Initialize multi-probe evaluator
-        self.evaluator = LLMMultiProbeEvaluator(self.settings)
-
-    @property
-    def is_enabled(self) -> bool:
-        """Check if LLM engine is enabled and available.
-
-        Returns:
-            True if LLM is enabled and client is available.
-        """
-        return self.settings.enabled and self.client is not None
-
-    @property
-    def settings(self) -> LLMSettings:
-        """Get current LLM settings.
-
-        Returns:
-            The LLM settings configuration.
-        """
-        return self._settings
-
-    @settings.setter
-    def settings(self, value: LLMSettings) -> None:
-        """Set LLM settings and reinitialize client.
-
-        Args:
-            value: New LLM settings.
-        """
-        self._settings = value
-        if value.enabled:
-            try:
-                self.client = value.get_client()
-            except Exception as e:
-                logger.warning("llm_client_init_failed", error=str(e))
-                self.client = None
-        else:
-            self.client = None
-
-    async def analyze_template(self, criterion: str) -> TemplateAnalysisResult:
-        """Analyze a criterion to extract evidence requirements.
-
-        Uses LLM to understand natural language criteria and extract structured
-        evidence requirements including sources, patterns, and applicability.
-
-        Args:
-            criterion: Natural language criterion text to analyze.
-
-        Returns:
-            TemplateAnalysisResult with extracted evidence requirements and
-            applicability information.
-        """
-        if not self.client:
-            return self._fallback_template_result("LLM evaluation disabled or unavailable")
-
-        # Check cache
-        cache_key = f"template_analysis:{criterion}"
-        cached = self.cache.get(cache_key, self.settings.model_simple)
-        if cached:
-            try:
-                return TemplateAnalysisResult.model_validate(json.loads(cached.response))
-            except Exception as e:
-                logger.warning("cache_parse_failed", error=str(e))
-
-        # Build prompt
-        system_prompt = TEMPLATE_ANALYSIS_SYSTEM
-        user_prompt = TEMPLATE_ANALYSIS_USER.format(criterion=criterion)
-        user_prompt += "\n\nOutput JSON matching this schema:\n"
-        user_prompt += json.dumps(TemplateAnalysisResult.model_json_schema(), indent=2)
-
-        try:
-            result = await self._call_llm_json(
-                system_prompt=system_prompt,
-                user_prompt=user_prompt,
-                model=self.settings.model_simple,
-                schema_class=TemplateAnalysisResult,
-                prompt_type="template_analysis",
-                cache_key=cache_key,
-            )
-            return result
-
-        except Exception as e:
-            logger.exception("template_analysis_failed", criterion=criterion, error=str(e))
-            return self._fallback_template_result(f"Analysis failed: {e!s}")
-
-    async def evaluate_content(
-        self,
-        content: str,
-        requirement: str,
-    ) -> ContentEvaluationResult:
-        """Evaluate content against a requirement.
-
-        Evaluates documentation, code, or configuration content against
-        a specified requirement using multi-probe analysis.
-
-        Args:
-            content: The content to evaluate (e.g., README content, code, config).
-            requirement: The requirement or criterion to evaluate against.
-
-        Returns:
-            ContentEvaluationResult with probe scores, gaps, strengths,
-            and overall assessment.
-        """
-        if not self.client:
-            return self._fallback_content_result("LLM evaluation disabled or unavailable")
-
-        # Truncate content if too long
-        max_content_length = 10000
-        truncated_content = content[:max_content_length]
-        if len(content) > max_content_length:
-            truncated_content += "\n\n[Content truncated due to length...]"
-
-        # Check cache
-        cache_key = f"content_eval:{hash(requirement)}:{hash(truncated_content)}"
-        cached = self.cache.get(cache_key, self.settings.model_standard)
-        if cached:
-            try:
-                return ContentEvaluationResult.model_validate(json.loads(cached.response))
-            except Exception as e:
-                logger.warning("cache_parse_failed", error=str(e))
-
-        # Build prompt
-        system_prompt = """You are an expert auditor evaluating content against requirements.
-Analyze the content and provide scores for different aspects of compliance."""
-        user_prompt = f"""Requirement: {requirement}
-
-Content to evaluate:
-{truncated_content}
-
-Provide evaluation with:
-1. Probe scores for different aspects (0-10 each)
-2. Overall score (0-10)
-3. Identified gaps with severity and recommendations
-4. Strengths and aligned practices
-5. Detailed reasoning
-
-Output JSON matching this schema:
-"""
-        user_prompt += json.dumps(ContentEvaluationResult.model_json_schema(), indent=2)
-
-        try:
-            result = await self._call_llm_json(
-                system_prompt=system_prompt,
-                user_prompt=user_prompt,
-                model=self.settings.model_standard,
-                schema_class=ContentEvaluationResult,
-                prompt_type="content_evaluation",
-                cache_key=cache_key,
-            )
-            return result
-
-        except Exception as e:
-            logger.exception("content_evaluation_failed", requirement=requirement, error=str(e))
-            return self._fallback_content_result(f"Evaluation failed: {e!s}")
-
-    async def analyze_gaps(
-        self,
-        evidence: EvidenceBundle,
-        requirement: str,
-    ) -> GapAnalysisResult:
-        """Analyze gaps between evidence and requirements.
-
-        Compares collected evidence against a requirement to identify gaps,
-        aligned practices, and provide actionable recommendations.
-
-        Args:
-            evidence: Evidence bundle containing collected evidence.
-            requirement: The requirement to check against.
-
-        Returns:
-            GapAnalysisResult with score, gaps, aligned practices,
-            and detailed reasoning.
-        """
-        if not self.client:
-            return self._fallback_gap_result("LLM evaluation disabled or unavailable")
-
-        # Use multi-probe evaluator for gap analysis
-        try:
-            # The evaluator returns ContentEvaluationResult, we convert to GapAnalysisResult
-            content_analysis = await self.evaluator.evaluate(evidence, requirement)
-
-            # Convert ContentEvaluationResult to GapAnalysisResult
-            # gaps are already GapFinding objects in ContentEvaluationResult
-            gaps = content_analysis.gaps
-
-            # Convert strengths to AlignedPracticeFinding
-            aligned_practices = [
-                AlignedPracticeFinding(
-                    description=strength,
-                    best_practice_level="Good",
-                )
-                for strength in content_analysis.strengths
-            ]
-
-            result = GapAnalysisResult(
-                score=content_analysis.overall_score,  # Already 0-10
-                gaps=gaps,
-                aligned_practices=aligned_practices,
-                reasoning=content_analysis.reasoning,
-                confidence=content_analysis.confidence,
-            )
-
-            return result
-
-        except Exception as e:
-            logger.exception("gap_analysis_failed", requirement=requirement, error=str(e))
-            return self._fallback_gap_result(f"Analysis failed: {e!s}")
-
-    async def evaluate_readme(self, readme_content: str) -> ContentEvaluationResult:
-        """Evaluate README documentation quality.
-
-        Evaluates a README file for completeness, clarity, and usability
-        based on documentation best practices.
-
-        Args:
-            readme_content: The README file content to evaluate.
-
-        Returns:
-            ContentEvaluationResult with specific probes for README quality.
-        """
-        if not self.client:
-            return self._fallback_content_result("LLM evaluation disabled or unavailable")
-
-        from vds_audit_orchestrator.llm.prompts.readme_evaluation import (
-            README_EVALUATION_SYSTEM,
-            README_EVALUATION_USER,
-        )
-
-        # Truncate if needed
-        max_content_length = 8000
-        truncated_content = readme_content[:max_content_length]
-        if len(readme_content) > max_content_length:
-            truncated_content += "\n\n[Content truncated...]"
-
-        # Check cache
-        cache_key = f"readme_eval:{hash(truncated_content)}"
-        cached = self.cache.get(cache_key, self.settings.model_standard)
-        if cached:
-            try:
-                return ContentEvaluationResult.model_validate(json.loads(cached.response))
-            except Exception as e:
-                logger.warning("cache_parse_failed", error=str(e))
-
-        # Build prompt
-        user_prompt = README_EVALUATION_USER.format(content=truncated_content)
-        user_prompt += "\n\nOutput JSON matching this schema:\n"
-        user_prompt += json.dumps(ContentEvaluationResult.model_json_schema(), indent=2)
-
-        try:
-            result = await self._call_llm_json(
-                system_prompt=README_EVALUATION_SYSTEM,
-                user_prompt=user_prompt,
-                model=self.settings.model_standard,
-                schema_class=ContentEvaluationResult,
-                prompt_type="readme_evaluation",
-                cache_key=cache_key,
-            )
-            return result
-
-        except Exception as e:
-            logger.exception("readme_evaluation_failed", error=str(e))
-            return self._fallback_content_result(f"Evaluation failed: {e!s}")
-
-    async def evaluate_api_docs(self, docs_content: str) -> ContentEvaluationResult:
-        """Evaluate API documentation quality.
-
-        Evaluates API documentation for completeness, OpenAPI/Swagger
-        compliance, and developer usability.
-
-        Args:
-            docs_content: The API documentation content to evaluate.
-
-        Returns:
-            ContentEvaluationResult with specific probes for API docs quality.
-        """
-        if not self.client:
-            return self._fallback_content_result("LLM evaluation disabled or unavailable")
-
-        from vds_audit_orchestrator.llm.prompts.api_docs_evaluation import (
-            API_DOCS_EVALUATION_SYSTEM,
-            API_DOCS_EVALUATION_USER,
-        )
-
-        # Truncate if needed
-        max_content_length = 10000
-        truncated_content = docs_content[:max_content_length]
-        if len(docs_content) > max_content_length:
-            truncated_content += "\n\n[Content truncated...]"
-
-        # Check cache
-        cache_key = f"api_docs_eval:{hash(truncated_content)}"
-        cached = self.cache.get(cache_key, self.settings.model_standard)
-        if cached:
-            try:
-                return ContentEvaluationResult.model_validate(json.loads(cached.response))
-            except Exception as e:
-                logger.warning("cache_parse_failed", error=str(e))
-
-        # Build prompt
-        user_prompt = API_DOCS_EVALUATION_USER.format(content=truncated_content)
-        user_prompt += "\n\nOutput JSON matching this schema:\n"
-        user_prompt += json.dumps(ContentEvaluationResult.model_json_schema(), indent=2)
-
-        try:
-            result = await self._call_llm_json(
-                system_prompt=API_DOCS_EVALUATION_SYSTEM,
-                user_prompt=user_prompt,
-                model=self.settings.model_standard,
-                schema_class=ContentEvaluationResult,
-                prompt_type="api_docs_evaluation",
-                cache_key=cache_key,
-            )
-            return result
-
-        except Exception as e:
-            logger.exception("api_docs_evaluation_failed", error=str(e))
-            return self._fallback_content_result(f"Evaluation failed: {e!s}")
-
-    async def evaluate_security_docs(self, docs_content: str) -> ContentEvaluationResult:
-        """Evaluate security documentation quality.
-
-        Evaluates security documentation for threat modeling coverage,
-        data flow documentation, access controls, and compliance mapping.
-
-        Args:
-            docs_content: The security documentation content to evaluate.
-
-        Returns:
-            ContentEvaluationResult with specific probes for security docs quality.
-        """
-        if not self.client:
-            return self._fallback_content_result("LLM evaluation disabled or unavailable")
-
-        from vds_audit_orchestrator.llm.prompts.security_docs_evaluation import (
-            SECURITY_DOCS_EVALUATION_SYSTEM,
-            SECURITY_DOCS_EVALUATION_USER,
-        )
-
-        # Truncate if needed
-        max_content_length = 10000
-        truncated_content = docs_content[:max_content_length]
-        if len(docs_content) > max_content_length:
-            truncated_content += "\n\n[Content truncated...]"
-
-        # Check cache
-        cache_key = f"security_docs_eval:{hash(truncated_content)}"
-        cached = self.cache.get(cache_key, self.settings.model_standard)
-        if cached:
-            try:
-                return ContentEvaluationResult.model_validate(json.loads(cached.response))
-            except Exception as e:
-                logger.warning("cache_parse_failed", error=str(e))
-
-        # Build prompt
-        user_prompt = SECURITY_DOCS_EVALUATION_USER.format(content=truncated_content)
-        user_prompt += "\n\nOutput JSON matching this schema:\n"
-        user_prompt += json.dumps(ContentEvaluationResult.model_json_schema(), indent=2)
-
-        try:
-            result = await self._call_llm_json(
-                system_prompt=SECURITY_DOCS_EVALUATION_SYSTEM,
-                user_prompt=user_prompt,
-                model=self.settings.model_complex,
-                schema_class=ContentEvaluationResult,
-                prompt_type="security_docs_evaluation",
-                cache_key=cache_key,
-            )
-            return result
-
-        except Exception as e:
-            logger.exception("security_docs_evaluation_failed", error=str(e))
-            return self._fallback_content_result(f"Evaluation failed: {e!s}")
-
-    def get_usage_summary(self) -> UsageSummary:
-        """Get summary of LLM usage and costs.
-
-        Returns:
-            UsageSummary with total requests, tokens, costs, and breakdowns.
-        """
-        return self.cost_tracker.get_summary()
-
-    async def _call_llm_json(
-        self,
-        system_prompt: str,
-        user_prompt: str,
-        model: str,
-        schema_class: type,
-        prompt_type: str,
-        cache_key: str,
-    ) -> Any:
-        """Call LLM and parse JSON response.
-
-        Args:
-            system_prompt: System prompt for the LLM.
-            user_prompt: User prompt for the LLM.
-            model: Model name to use.
-            schema_class: Pydantic schema class for response validation.
-            prompt_type: Type of prompt for cost tracking.
-            cache_key: Cache key for storing response.
-
-        Returns:
-            Parsed and validated response as schema_class instance.
-        """
-        if not self.client:
-            raise RuntimeError("LLM client not initialized")
-
-        client = self.client
-
-        def _make_call():
-            return client.messages.create(
-                model=model,
-                max_tokens=4096,
-                temperature=0.0,
-                system=system_prompt,
-                messages=[{"role": "user", "content": user_prompt}],
-            )
-
-        message = await asyncio.to_thread(_make_call)
-
-        # Track cost
-        if message.usage:
-            record = self.cost_tracker.record_usage(
-                model=model,
-                prompt_type=prompt_type,
-                input_tokens=message.usage.input_tokens,
-                output_tokens=message.usage.output_tokens,
-            )
-            record_llm_usage(
-                model=model,
-                prompt_tokens=record.input_tokens,
-                completion_tokens=record.output_tokens,
-                cost=record.cost,
-            )
-
-        # Parse response
-        content_text = ""
-        for block in message.content:
-            text = getattr(block, "text", None)
-            if text:
-                content_text = text
-                break
-
-        # Extract JSON if wrapped in markdown
-        if "```json" in content_text:
-            json_start = content_text.find("```json") + 7
-            json_end = content_text.find("```", json_start)
-            json_str = content_text[json_start:json_end].strip()
-        elif "```" in content_text:
-            json_start = content_text.find("```") + 3
-            json_end = content_text.find("```", json_start)
-            json_str = content_text[json_start:json_end].strip()
-        else:
-            json_str = content_text.strip()
-
-        # Parse and validate
-        data = json.loads(json_str)
-        result = schema_class.model_validate(data)
-
-        # Cache the result
-        self.cache.set(
-            prompt=cache_key,
-            model=model,
-            response=json_str,
-            input_tokens=message.usage.input_tokens if message.usage else 0,
-            output_tokens=message.usage.output_tokens if message.usage else 0,
-        )
-
-        return result
-
-    def _fallback_template_result(self, reason: str) -> TemplateAnalysisResult:
-        """Return fallback template analysis result.
-
-        Args:
-            reason: Reason for fallback.
-
-        Returns:
-            Default TemplateAnalysisResult indicating analysis was skipped.
-        """
-        return TemplateAnalysisResult(
-            evidence_requirements=[],
-            applicability=ApplicabilitySchema(),
-            gap_severity_if_missing="unknown",
-            gap_category="unknown",
-            check_type="llm_content_evaluation",
-            confidence=0.0,
-        )
-
-    def _fallback_content_result(self, reason: str) -> ContentEvaluationResult:
-        """Return fallback content evaluation result.
-
-        Args:
-            reason: Reason for fallback.
-
-        Returns:
-            Default ContentEvaluationResult indicating evaluation was skipped.
-        """
-        return ContentEvaluationResult(
-            probes=[
-                ProbeScore(
-                    name="system",
-                    score=0.0,
-                    max_score=10.0,
-                    reasoning=reason,
-                    evidence_refs=[],
-                )
-            ],
-            overall_score=0.0,
-            maturity_level=1,
-            gaps=[],
-            strengths=[],
-            reasoning=reason,
-            confidence=0.0,
-        )
-
-    def _fallback_gap_result(self, reason: str) -> GapAnalysisResult:
-        """Return fallback gap analysis result.
-
-        Args:
-            reason: Reason for fallback.
-
-        Returns:
-            Default GapAnalysisResult indicating analysis was skipped.
-        """
-        return GapAnalysisResult(
-            score=0.0,
-            gaps=[
-                GapFinding(
-                    description=reason,
-                    severity="unknown",
-                    category="system",
-                    recommendation="Enable LLM evaluation to perform gap analysis.",
-                )
-            ],
-            aligned_practices=[],
-            reasoning=reason,
-            confidence=0.0,
-        )
-
-
-# Convenience functions for easy import.
-# Note: These helpers create a new LLMEngine per call; reuse LLMEngine for batches.
-
-
-async def analyze_template(
-    criterion: str,
-    settings: LLMSettings | None = None,
-) -> TemplateAnalysisResult:
-    """Analyze a criterion to extract evidence requirements.
-
-    Convenience function that creates a new LLMEngine instance per call.
-    This is intentional to keep call sites lightweight and avoid shared
-    mutable state across concurrent callers.
-
-    Args:
-        criterion: Natural language criterion text to analyze.
-        settings: Optional LLM settings. Uses defaults if None.
-
-    Returns:
-        TemplateAnalysisResult with extracted evidence requirements.
-
-    Example:
-        >>> result = await analyze_template("Project must have a README")
-        >>> print(result.evidence_requirements)
-    """
-    engine = LLMEngine(settings)
-    return await engine.analyze_template(criterion)
-
-
-async def evaluate_content(
-    content: str,
-    requirement: str,
-    settings: LLMSettings | None = None,
-) -> ContentEvaluationResult:
-    """Evaluate content against a requirement.
-
-    Convenience function that creates an LLMEngine instance and calls
-    evaluate_content.
-
-    Args:
-        content: The content to evaluate.
-        requirement: The requirement to evaluate against.
-        settings: Optional LLM settings. Uses defaults if None.
-
-    Returns:
-        ContentEvaluationResult with probe scores and findings.
-
-    Example:
-        >>> readme = "# My Project\\n..."
-        >>> result = await evaluate_content(readme, "Must have installation instructions")
-        >>> print(result.overall_score)
-    """
-    engine = LLMEngine(settings)
-    return await engine.evaluate_content(content, requirement)
-
-
-async def evaluate_readme(
-    readme_content: str,
-    settings: LLMSettings | None = None,
-) -> ContentEvaluationResult:
-    """Evaluate README documentation quality.
-
-    Convenience function for README evaluation.
-
-    Args:
-        readme_content: The README file content.
-        settings: Optional LLM settings. Uses defaults if None.
-
-    Returns:
-        ContentEvaluationResult with README-specific evaluation.
-    """
-    engine = LLMEngine(settings)
-    return await engine.evaluate_readme(readme_content)
-
-
-async def evaluate_api_docs(
-    docs_content: str,
-    settings: LLMSettings | None = None,
-) -> ContentEvaluationResult:
-    """Evaluate API documentation quality.
-
-    Convenience function for API docs evaluation.
-
-    Args:
-        docs_content: The API documentation content.
-        settings: Optional LLM settings. Uses defaults if None.
-
-    Returns:
-        ContentEvaluationResult with API docs-specific evaluation.
-    """
-    engine = LLMEngine(settings)
-    return await engine.evaluate_api_docs(docs_content)
-
-
-async def evaluate_security_docs(
-    docs_content: str,
-    settings: LLMSettings | None = None,
-) -> ContentEvaluationResult:
-    """Evaluate security documentation quality.
-
-    Convenience function for security docs evaluation.
-
-    Args:
-        docs_content: The security documentation content.
-        settings: Optional LLM settings. Uses defaults if None.
-
-    Returns:
-        ContentEvaluationResult with security docs-specific evaluation.
-    """
-    engine = LLMEngine(settings)
-    return await engine.evaluate_security_docs(docs_content)
-
-
-async def analyze_gaps(
-    evidence: EvidenceBundle,
-    requirement: str,
-    settings: LLMSettings | None = None,
-) -> GapAnalysisResult:
-    """Analyze gaps between evidence and requirements.
-
-    Convenience function for gap analysis.
-
-    Args:
-        evidence: Evidence bundle containing collected evidence.
-        requirement: The requirement to check against.
-        settings: Optional LLM settings. Uses defaults if None.
-
-    Returns:
-        GapAnalysisResult with gaps and aligned practices.
-    """
-    engine = LLMEngine(settings)
-    return await engine.analyze_gaps(evidence, requirement)