@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/audit_orchestrator/src/vds_audit_orchestrator/engine/ad_hoc_analyzer.py

@@ -1,399 +0,0 @@
-"""Ad-hoc requirement analyzer for TSK-252.
-
-Enables analysis of arbitrary requirements against evidence without a template.
-Builds synthetic AuditCheck and reuses PostgresEvidenceMatcher, RowEvaluator, and GroundingValidator.
-
-FR-25: Dynamic Analysis Support (Phase 25)
-"""
-
-from __future__ import annotations
-
-import hashlib
-from dataclasses import dataclass
-from datetime import UTC, datetime
-from pathlib import Path
-from typing import TYPE_CHECKING, Literal, cast
-
-import structlog
-from pydantic import BaseModel, Field
-
-from vds_audit_orchestrator.config import get_config
-from vds_audit_orchestrator.engine.deterministic_evaluator import DeterministicEvaluator
-from vds_audit_orchestrator.engine.grounding_validator import GroundingConfig, GroundingValidator
-from vds_audit_orchestrator.engine.llm_row_evaluator import LLMRowEvaluator
-from vds_audit_orchestrator.engine.row_evaluator_types import (
-    RowEvaluationResult,
-    RowEvaluatorConfig,
-    RowLLMMode,
-    estimate_evidence_clarity,
-)
-from vds_audit_orchestrator.evidence.code_pack import CodeEvidencePack, CodePackBuilder
-from vds_audit_orchestrator.evidence.loader import (
-    EvidenceManifestLoader,
-    ProjectChunkLoader,
-    StateManifestLoader,
-)
-from vds_audit_orchestrator.evidence.postgres_matcher import PostgresEvidenceMatcher
-from vds_audit_orchestrator.models.checklist import EvidenceAnchor, RowProvenance, RowStatus, ScoreBreakdown
-from vds_audit_orchestrator.models.enums import Severity
-from vds_audit_orchestrator.models.template import AuditCheck
-
-if TYPE_CHECKING:
-    from vds_audit_orchestrator.evidence.corpus import EvidenceCorpus
-    from vds_audit_orchestrator.evidence.matcher import MatcherProtocol, RowEvidenceContext
-
-logger = structlog.get_logger(__name__)
-
-RetrievalMode = Literal["lexical", "vector", "hybrid"]
-
-
-@dataclass
-class AdHocAnalysisConfig:
-    """Configuration for ad-hoc analysis."""
-
-    max_evidence_chars: int = 8000
-    strict_grounding: bool = False
-    evidence_project: str | None = None
-    evidence_project_key: str | None = None
-    repo_storage_key: str | None = None
-    llm_mode: RowLLMMode = RowLLMMode.SELECTIVE
-    confidence_threshold: float = 0.6
-    retrieval_mode: str = "hybrid"
-    # Phase 64 default-on behavior: run tool-first row investigation by default.
-    tool_first_enabled: bool = True
-
-
-class AdHocAnalysisResult(BaseModel):
-    """Result of ad-hoc requirement analysis."""
-
-    requirement: str = Field(description="The analyzed requirement text")
-    status: RowStatus = Field(description="Evaluation status (PASS/PARTIAL/FAIL/ERROR/NA)")
-    score: float = Field(ge=0, le=100, description="Score as percentage (0-100)")
-    score_1_5: int = Field(ge=1, le=5, description="Score on Vietnamese 1-5 scale")
-    score_breakdown: ScoreBreakdown = Field(description="Detailed score breakdown")
-    reason: str = Field(description="Vietnamese reason (Lý do)")
-    finding: str = Field(description="Vietnamese finding (Phát hiện)")
-    evidence_anchors: list[EvidenceAnchor] = Field(default_factory=list, description="Cited evidence anchors")
-    provenance: RowProvenance = Field(description="Evaluation provenance metadata")
-    recommendation: str | None = Field(default=None, description="Remediation recommendation")
-    error_message: str | None = Field(default=None, description="Error message if status is ERROR")
-    evidence_context_chars: int = Field(default=0, description="Total evidence context characters used")
-    matched_docs_count: int = Field(default=0, description="Number of matched Confluence documents")
-    matched_code_count: int = Field(default=0, description="Number of matched code files")
-
-    class Config:
-        """Pydantic config."""
-
-        use_enum_values = True
-
-
-def _build_synthetic_check(requirement: str) -> AuditCheck:
-    """Build a synthetic AuditCheck from a requirement string.
-
-    Args:
-        requirement: The requirement text to analyze.
-
-    Returns:
-        AuditCheck instance with synthetic ID and the requirement as description.
-    """
-    # Generate a stable ID from the requirement text
-    req_hash = hashlib.sha256(requirement.encode()).hexdigest()[:8]
-    check_id = f"ADHOC-{req_hash.upper()}"
-
-    return AuditCheck(
-        id=check_id,
-        name=requirement[:100],  # Truncate for name
-        description=requirement,
-        max_points=100,
-        weight=100.0,  # Single check = 100% weight
-        check_type="llm_content_evaluation",  # Semantic evaluation
-        section_id="ad-hoc",
-        severity=Severity.MEDIUM,
-    )
-
-
-def _compute_template_hash(requirement: str) -> str:
-    """Compute a hash representing the 'template' (just the requirement)."""
-    return hashlib.sha256(f"adhoc:{requirement}".encode()).hexdigest()[:16]
-
-
-def _compute_evidence_hash(corpus: EvidenceCorpus, code_pack: CodeEvidencePack | None) -> str:
-    """Compute a hash representing the evidence corpus."""
-    parts = []
-    for doc in corpus.documents[:50]:  # Sample first 50 docs
-        parts.append(doc.title or "")
-        parts.append(doc.source_path or "")
-    if code_pack:
-        for f in code_pack.files[:20]:
-            parts.append(f.file_path)
-    return hashlib.sha256("|".join(parts).encode()).hexdigest()[:16]
-
-
-class AdHocAnalyzer:
-    """Analyzer for ad-hoc requirements without a template.
-
-    Builds synthetic AuditCheck and reuses existing evaluation pipeline:
-    - PostgresEvidenceMatcher for targeted evidence retrieval
-    - RowEvaluator (deterministic or LLM) for scoring
-    - GroundingValidator for citation validation
-    """
-
-    def __init__(
-        self,
-        evidence_dir: Path | None = None,
-        repo_path: Path | None = None,
-        config: AdHocAnalysisConfig | None = None,
-        state_dsn: str | None = None,
-        state_manifest_loader: StateManifestLoader | None = None,
-        project_chunk_loader: ProjectChunkLoader | None = None,
-    ):
-        """Initialize the analyzer."""
-        self.evidence_dir = Path(evidence_dir).expanduser().resolve(strict=False) if evidence_dir else None
-        self.repo_path = Path(repo_path) if repo_path else None
-        self.config = config or AdHocAnalysisConfig()
-        self.state_dsn = state_dsn
-        self.state_manifest_loader = state_manifest_loader
-        self.project_chunk_loader = project_chunk_loader
-
-        # Lazy-loaded components
-        self._corpus: EvidenceCorpus | None = None
-        self._code_pack: CodeEvidencePack | None = None
-
-    def _load_evidence(self) -> tuple[EvidenceCorpus, CodeEvidencePack | None]:
-        """Load evidence corpus and optional code pack."""
-        if self._corpus is None:
-            if not self.state_dsn or not self.state_dsn.strip() or self.state_manifest_loader is None:
-                raise RuntimeError("Centralized state is required for evidence loading.")
-            loader = EvidenceManifestLoader(
-                state_dsn=self.state_dsn,
-                state_manifest_loader=self.state_manifest_loader,
-                project_chunk_loader=self.project_chunk_loader,
-            )
-            self._corpus = loader.load_runtime_corpus(project_storage_key=self.config.evidence_project_key)
-            assert self._corpus is not None
-            logger.info(
-                "evidence_corpus_loaded",
-                doc_count=len(self._corpus.documents),
-            )
-
-        if self._code_pack is None and self.repo_path and self.repo_path.exists():
-            builder = CodePackBuilder(self.repo_path)
-            self._code_pack = builder.build()
-            assert self._code_pack is not None
-            logger.info(
-                "code_pack_loaded",
-                repo_path=str(self.repo_path),
-                file_count=self._code_pack.total_files_included,
-            )
-
-        return self._corpus, self._code_pack
-
-    async def analyze(self, requirement: str) -> AdHocAnalysisResult:
-        """Analyze a requirement against the evidence.
-
-        Args:
-            requirement: The requirement text to analyze.
-
-        Returns:
-            AdHocAnalysisResult with evaluation details.
-        """
-        log = logger.bind(requirement=requirement[:100])
-        log.info("ad_hoc_analysis_started")
-
-        # Load evidence
-        corpus, code_pack = self._load_evidence()
-
-        if not corpus.documents and not code_pack:
-            log.warning("no_evidence_available")
-            return self._create_no_evidence_result(requirement)
-
-        # Build synthetic check
-        check = _build_synthetic_check(requirement)
-        template_hash = _compute_template_hash(requirement)
-        evidence_hash = _compute_evidence_hash(corpus, code_pack)
-
-        # Match evidence to requirement
-        normalized_retrieval_mode = cast("RetrievalMode", self.config.retrieval_mode)
-        matcher: MatcherProtocol = PostgresEvidenceMatcher(
-            state_dsn=self.state_dsn,
-            project_name=self.config.evidence_project,
-            project_storage_key=self.config.evidence_project_key,
-            repo_name=self.repo_path.name if self.repo_path else None,
-            repo_storage_key=self.config.repo_storage_key,
-            code_pack=code_pack,
-            max_context_chars=self.config.max_evidence_chars,
-            retrieval_mode=normalized_retrieval_mode,
-        )
-
-        row_context = matcher.match_row(
-            row_id=check.id,
-            requirement_text=requirement,
-            section_id="ad-hoc",
-        )
-
-        log.info(
-            "evidence_matched",
-            matched_docs=len(row_context.matched_docs),
-            matched_code=len(row_context.matched_code),
-            total_chars=row_context.total_chars,
-            retrieval_trace=row_context.retrieval_trace or {},
-        )
-
-        # Evaluate the requirement
-        result = await self._evaluate(
-            check=check,
-            row_context=row_context,
-            template_hash=template_hash,
-            evidence_hash=evidence_hash,
-        )
-
-        # Apply grounding validation
-        grounding_config = GroundingConfig(
-            strict_mode=self.config.strict_grounding,
-            verify_excerpts=True,
-        )
-        grounding_validator = GroundingValidator(grounding_config)
-
-        evidence_context = row_context.format_for_llm(self.config.max_evidence_chars)
-        result = grounding_validator.validate(
-            result,
-            allowed_refs=row_context.evidence_refs,
-            evidence_context=evidence_context,
-        )
-
-        log.info(
-            "ad_hoc_analysis_completed",
-            status=result.status.value,
-            score=result.score,
-            evidence_anchors=len(result.evidence_anchors),
-        )
-
-        # Convert to AdHocAnalysisResult
-        return AdHocAnalysisResult(
-            requirement=requirement,
-            status=result.status,
-            score=result.score,
-            score_1_5=result.score_1_5 or 1,
-            score_breakdown=result.score_breakdown,
-            reason=result.reason,
-            finding=result.finding,
-            evidence_anchors=result.evidence_anchors,
-            provenance=result.provenance,
-            recommendation=result.recommendation,
-            error_message=result.error_message,
-            evidence_context_chars=row_context.total_chars,
-            matched_docs_count=len(row_context.matched_docs),
-            matched_code_count=len(row_context.matched_code),
-        )
-
-    async def _evaluate(
-        self,
-        check: AuditCheck,
-        row_context: RowEvidenceContext,
-        template_hash: str,
-        evidence_hash: str,
-    ) -> RowEvaluationResult:
-        """Evaluate the check using appropriate evaluator.
-
-        Uses LLM for semantic evaluation (ad-hoc checks are always semantic).
-        """
-        evidence_context = row_context.format_for_llm(self.config.max_evidence_chars)
-        evidence_clarity = estimate_evidence_clarity(
-            requirement_text=check.description,
-            evidence_context=evidence_context,
-            evidence_refs=row_context.evidence_refs,
-        )
-
-        # Ad-hoc checks are semantic by nature, so use LLM unless mode is OFF
-        use_llm = self.config.llm_mode != RowLLMMode.OFF
-
-        if use_llm and (row_context.matched_docs or row_context.matched_code):
-            # Use LLM evaluator
-            evaluator_config = RowEvaluatorConfig(
-                mode=self.config.llm_mode,
-                max_evidence_chars=self.config.max_evidence_chars,
-                confidence_threshold=self.config.confidence_threshold,
-                tool_first_enabled=self.config.tool_first_enabled,
-                runtime_context={
-                    "state_dsn": self.state_dsn,
-                    "project_key": self.config.evidence_project_key or self.config.evidence_project,
-                    "repo_key": self.config.repo_storage_key or (self.repo_path.name if self.repo_path else None),
-                    "repo_path": str(self.repo_path) if self.repo_path is not None else None,
-                    "requirement_scope_enabled": bool(get_config().features.requirement_scope_enabled),
-                    "manual_check_gating_enabled": bool(get_config().features.manual_check_gating_enabled),
-                    "cross_repo_config_enabled": bool(get_config().features.cross_repo_config_enabled),
-                    "shared_resource_indexing_enabled": bool(get_config().features.shared_resource_indexing_enabled),
-                    "instruction_analysis_enabled": bool(get_config().features.instruction_analysis_enabled),
-                },
-            )
-
-            llm_evaluator = LLMRowEvaluator(
-                config=evaluator_config,
-                template_hash=template_hash,
-                evidence_hash=evidence_hash,
-            )
-
-            return await llm_evaluator.aevaluate(
-                check=check,
-                evidence_context=evidence_context,
-                evidence_refs=row_context.evidence_refs,
-                row_id=check.id,
-                project_profile=None,
-            )
-        else:
-            # Use deterministic evaluator (no evidence or LLM disabled)
-            det_evaluator = DeterministicEvaluator(template_hash=template_hash)
-
-            # Create a synthetic Evidence object for deterministic evaluation
-            import hashlib
-
-            from vds_audit_orchestrator.models.evidence import Evidence, EvidenceStatus
-
-            if row_context.matched_docs or row_context.matched_code:
-                # Some evidence found
-                evidence = Evidence(
-                    source="confluence",
-                    source_id="ad-hoc-evidence",
-                    title="Ad-hoc Evidence",
-                    content=evidence_context[:500],
-                    content_hash=hashlib.sha256(evidence_context.encode()).hexdigest()[:16],
-                    status=EvidenceStatus.FOUND if evidence_clarity > 0.5 else EvidenceStatus.PARTIAL,
-                )
-            else:
-                evidence = None
-
-            return det_evaluator.evaluate(
-                check=check,
-                evidence=evidence,
-                row_id=check.id,
-            )
-
-    def _create_no_evidence_result(self, requirement: str) -> AdHocAnalysisResult:
-        """Create a result when no evidence is available."""
-        _build_synthetic_check(requirement)
-        template_hash = _compute_template_hash(requirement)
-
-        return AdHocAnalysisResult(
-            requirement=requirement,
-            status=RowStatus.FAIL,
-            score=0.0,
-            score_1_5=1,
-            score_breakdown=ScoreBreakdown.compute(raw_score=0.0, coverage=0.0, confidence=1.0),
-            reason="Không tìm thấy bằng chứng trong thư mục evidence.",
-            finding="Thiếu tài liệu hoặc mã nguồn để đánh giá yêu cầu này.",
-            evidence_anchors=[],
-            provenance=RowProvenance(
-                row_llm_mode="off",
-                protocol=None,
-                model=None,
-                template_hash=template_hash,
-                rubric_version="1.0",
-                evidence_hash="none",
-                evaluated_at=datetime.now(UTC),
-            ),
-            recommendation="Bổ sung tài liệu Confluence hoặc mã nguồn vào thư mục evidence.",
-            error_message=None,
-            evidence_context_chars=0,
-            matched_docs_count=0,
-            matched_code_count=0,
-        )

package/tools/vds-scripts/audit_orchestrator/src/vds_audit_orchestrator/engine/aggregator.py

@@ -1,220 +0,0 @@
-"""Multi-repository audit aggregation."""
-
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Any
-
-import structlog
-from pydantic import BaseModel, Field
-
-logger = structlog.get_logger(__name__)
-
-
-class RepoAuditSummary(BaseModel):
-    """Summary of single repo audit."""
-
-    repository: str
-    overall_score: float
-    maturity_level: int
-    critical_count: int = 0
-    high_count: int = 0
-    medium_count: int = 0
-    low_count: int = 0
-    aligned_count: int = 0
-    all_na: bool = False
-
-
-class AggregatedResult(BaseModel):
-    """Aggregated result across multiple repositories."""
-
-    project: str
-    audit_date: datetime = Field(default_factory=datetime.now)
-
-    # Repository summaries
-    repositories: list[RepoAuditSummary] = Field(default_factory=list)
-
-    # Aggregate scores
-    avg_score: float = 0.0
-    min_score: float = 0.0
-    max_score: float = 0.0
-
-    # Maturity distribution
-    maturity_distribution: dict[int, int] = Field(default_factory=dict)
-
-    # Gap aggregation
-    total_critical: int = 0
-    total_high: int = 0
-    total_gaps: int = 0
-
-    # Common gaps (appear in multiple repos)
-    common_gaps: list[dict] = Field(default_factory=list)
-
-    # Best practices (can be shared)
-    shareable_practices: list[dict] = Field(default_factory=list)
-
-    # Recommendations
-    org_recommendations: list[str] = Field(default_factory=list)
-
-
-class MultiRepoAggregator:
-    """Aggregate audit results across multiple repositories."""
-
-    def __init__(self):
-        self.logger = structlog.get_logger(__name__)
-
-    def aggregate(
-        self,
-        project: str,
-        audit_results: list[dict],
-    ) -> AggregatedResult:
-        """Aggregate multiple repository audits."""
-        if not audit_results:
-            return AggregatedResult(project=project)
-
-        summaries: list[RepoAuditSummary] = []
-        all_gaps: list[dict[str, Any]] = []
-        all_practices: list[dict[str, Any]] = []
-
-        for result in audit_results:
-            repo = result.get("repository", "unknown")
-            score = result.get("overall_score", 0.0)
-            row_count = result.get("row_count", -1)
-
-            gaps = result.get("gaps", [])
-            practices = result.get("aligned_practices", [])
-
-            critical = len([g for g in gaps if g.get("severity") == "critical"])
-            high = len([g for g in gaps if g.get("severity") == "high"])
-            medium = len([g for g in gaps if g.get("severity") == "medium"])
-            low = len([g for g in gaps if g.get("severity") == "low"])
-
-            # Derive all_na: explicit flag takes precedence; fall back to
-            # (row_count == 0 and score == 0.0) heuristic.
-            all_na_flag = result.get("all_na_rows")
-            if all_na_flag is True:
-                all_na = True
-            elif all_na_flag is False:
-                all_na = False
-            else:
-                all_na = row_count == 0 and score == 0.0
-
-            summaries.append(
-                RepoAuditSummary(
-                    repository=repo,
-                    overall_score=score,
-                    maturity_level=result.get("maturity_level", 3),
-                    critical_count=critical,
-                    high_count=high,
-                    medium_count=medium,
-                    low_count=low,
-                    aligned_count=len(practices),
-                    all_na=all_na,
-                )
-            )
-
-            for g in gaps:
-                g["_repo"] = repo
-                all_gaps.append(g)
-
-            for p in practices:
-                p["_repo"] = repo
-                all_practices.append(p)
-
-        # Calculate aggregates — exclude repos where all rows were N/A
-        # (all_na=True). Repos that scored 0% but are not all-NA are included.
-        scorable = [summary.overall_score for summary in summaries if not summary.all_na]
-        avg_score = sum(scorable) / len(scorable) if scorable else 0.0
-        min_score = min(scorable) if scorable else 0.0
-        max_score = max(scorable) if scorable else 0.0
-
-        # Maturity distribution
-        maturity_dist: dict[int, int] = {}
-        for s in summaries:
-            m = s.maturity_level
-            maturity_dist[m] = maturity_dist.get(m, 0) + 1
-
-        # Find common gaps
-        common_gaps = self._find_common_gaps(all_gaps)
-
-        # Find shareable practices
-        shareable = self._find_shareable_practices(all_practices)
-
-        # Generate org recommendations
-        recommendations = self._generate_org_recommendations(summaries, common_gaps, avg_score)
-
-        return AggregatedResult(
-            project=project,
-            repositories=summaries,
-            avg_score=round(avg_score, 2),
-            min_score=round(min_score, 2),
-            max_score=round(max_score, 2),
-            maturity_distribution=maturity_dist,
-            total_critical=sum(s.critical_count for s in summaries),
-            total_high=sum(s.high_count for s in summaries),
-            total_gaps=len(all_gaps),
-            common_gaps=common_gaps[:10],
-            shareable_practices=shareable[:10],
-            org_recommendations=recommendations,
-        )
-
-    def _find_common_gaps(self, gaps: list[dict[str, Any]]) -> list[dict[str, Any]]:
-        """Find gaps that appear in multiple repositories."""
-        gap_counts: dict[tuple[str, str], dict[str, Any]] = {}
-        for g in gaps:
-            key = (g.get("category", ""), g.get("description", "")[:50])
-            if key not in gap_counts:
-                gap_counts[key] = {"gap": g, "repos": set()}
-            gap_counts[key]["repos"].add(g.get("_repo", "unknown"))
-
-        common: list[dict[str, Any]] = []
-        for value in gap_counts.values():
-            repos = value["repos"]
-            if len(repos) <= 1:
-                continue
-            gap = value["gap"]
-            common.append(
-                {
-                    **gap,
-                    "affected_repos": sorted(repos),
-                    "occurrence_count": len(repos),
-                }
-            )
-
-        return sorted(common, key=lambda x: x["occurrence_count"], reverse=True)
-
-    def _find_shareable_practices(self, practices: list[dict]) -> list[dict]:
-        """Find best practices that can be shared."""
-        excellent = [p for p in practices if p.get("best_practice_level") in ["Excellent", "Exemplary"]]
-        return excellent
-
-    def _generate_org_recommendations(
-        self,
-        summaries: list[RepoAuditSummary],
-        common_gaps: list[dict],
-        avg_score: float,
-    ) -> list[str]:
-        """Generate organization-level recommendations."""
-        recs = []
-
-        # Critical gaps
-        total_critical = sum(s.critical_count for s in summaries)
-        if total_critical > 0:
-            recs.append(f"URGENT: Address {total_critical} critical findings across repositories")
-
-        # Common gaps
-        if common_gaps:
-            recs.append(
-                f"Standardize: {len(common_gaps)} issues appear in multiple repositories - consider org-wide fix"
-            )
-
-        # Low maturity repos
-        low_maturity = [s for s in summaries if s.maturity_level <= 2]
-        if low_maturity:
-            recs.append(f"Focus: {len(low_maturity)} repositories at maturity level 2 or below need attention")
-
-        # Score improvement
-        if avg_score < 6.0:
-            recs.append("Overall score below 6.0 - prioritize documentation and security improvements")
-
-        return recs