@ngocsangairvds/vsaf 3.2.14 → 3.2.15

package/tools/vds-scripts/progress_report_orchestrator/src/progress_report_orchestrator/pipeline/structural.py

@@ -1,384 +0,0 @@
-"""Structural analysis phase — deterministic dimensions 1-6 + hex pre-check.
-
-Dim 1 — service_exists:        repo is present in cache AND has ≥1 source file.
-Dim 2 — doc_chain_complete:    all 4 required doc types present in fr-to-docs.yaml.
-Dim 3 — brd_traced:            FR has a BRD document entry with non-zero line range.
-Dim 4 — code_activity_score:   git log recency + volume + contributor diversity (0-100).
-Dim 5 — (informational only):  code size / endpoint URLs; stored in RepoAnalysis.
-Dim 6 — test_coverage_score:   test_loc / source_loc ratio, scaled 0-100.
-Dim 7 pre-check — deterministic_arch_score: hexagonal compliance floor for Wave C LLM.
-
-All six dimensions are computed without LLM calls. File-system I/O is dispatched
-to a thread-pool executor so asyncio.gather can run FRs in parallel without
-blocking the event loop.
-"""
-
-from __future__ import annotations
-
-import asyncio
-from pathlib import Path
-from typing import Any
-
-import structlog
-
-from progress_report_orchestrator.models.analysis import FRAssessment, RepoAnalysis, compute_overall_score
-from progress_report_orchestrator.models.topology import FRModule, ProjectTopology
-
-log = structlog.get_logger(__name__)
-
-# All four doc types required for a complete documentation chain (Dim 2).
-_REQUIRED_DOC_TYPES: frozenset[str] = frozenset({"BRD", "PRD", "SRS", "Architecture"})
-
-
-# ---------------------------------------------------------------------------
-# Per-repo metrics
-# ---------------------------------------------------------------------------
-
-
-def _collect_repo_analysis(slug: str, repo_path: Path | None) -> RepoAnalysis:
-    """Return a fully-populated RepoAnalysis for a single repo.
-
-    Calls all deterministic analyzers (git_activity, test_scanner,
-    endpoint_scanner) in addition to the base exists/has_code check.
-    """
-    if repo_path is None or not repo_path.is_dir():
-        return RepoAnalysis(
-            repo_slug=slug,
-            exists=False,
-            has_code=False,
-            branch=None,
-            last_commit_date=None,
-            commit_count_30d=0,
-            contributors=[],
-            java_files=0,
-            ts_files=0,
-            total_loc=0,
-            test_files=0,
-            test_loc=0,
-            controllers=0,
-            services=0,
-            repositories=0,
-            endpoint_paths=[],
-        )
-
-    # Dim 4 — git activity
-    from progress_report_orchestrator.analyzers.git_activity import (
-        score_repo_activity,
-    )
-
-    git_metrics, _activity_score = score_repo_activity(repo_path)
-
-    # Dim 5 (informational) + endpoint paths for Dim 8 LLM
-    from progress_report_orchestrator.analyzers.endpoint_scanner import (
-        scan_code_size,
-    )
-
-    size_metrics: dict[str, Any] = scan_code_size(repo_path)
-
-    # Dim 6 — test coverage
-    from progress_report_orchestrator.analyzers.test_scanner import (
-        scan_tests,
-    )
-
-    test_metrics: dict[str, Any] = scan_tests(repo_path)
-
-    java_files: int = size_metrics.get("java_files", 0)
-    ts_files: int = size_metrics.get("ts_files", 0)
-
-    return RepoAnalysis(
-        repo_slug=slug,
-        exists=True,
-        has_code=(java_files + ts_files) > 0,
-        branch=None,  # not fetched at structural phase
-        last_commit_date=git_metrics.get("last_commit_date"),
-        commit_count_30d=git_metrics.get("commit_count_30d", 0),
-        contributors=list(git_metrics.get("contributors", [])),
-        java_files=java_files,
-        ts_files=ts_files,
-        total_loc=size_metrics.get("total_loc", 0),
-        test_files=test_metrics.get("test_files", 0),
-        test_loc=test_metrics.get("test_loc", 0),
-        controllers=size_metrics.get("controllers", 0),
-        services=size_metrics.get("services", 0),
-        repositories=size_metrics.get("repositories", 0),
-        endpoint_paths=list(size_metrics.get("endpoint_paths", [])),
-    )
-
-
-# ---------------------------------------------------------------------------
-# Dimension checks
-# ---------------------------------------------------------------------------
-
-
-def _check_service_exists(
-    fr: FRModule,
-    repo_paths: dict[str, Path],
-) -> tuple[bool, list[RepoAnalysis]]:
-    """Dim 1: service_exists = at least one target service repo has source code."""
-    analyses: list[RepoAnalysis] = []
-    any_has_code = False
-
-    for slug in fr.target_services:
-        analysis = _collect_repo_analysis(slug, repo_paths.get(slug))
-        analyses.append(analysis)
-        if analysis.has_code:
-            any_has_code = True
-
-    return any_has_code, analyses
-
-
-def _check_doc_chain_complete(fr: FRModule) -> bool:
-    """Dim 2: doc_chain_complete = all 4 required doc types are present."""
-    present = {doc.doc_type for doc in fr.documents}
-    return _REQUIRED_DOC_TYPES.issubset(present)
-
-
-def _detect_brd_sections(
-    brd_path: Path,
-    *,
-    fr_id: str,
-    fr_title: str = "",
-) -> list[tuple[str, int]]:
-    """Scan *brd_path* markdown for headings that mention *fr_id* or *fr_title*.
-
-    Returns a list of (heading_text, line_number) tuples for every matching
-    heading.  Returns an empty list on any I/O error or when no match is found.
-    """
-    if not brd_path.is_file():
-        return []
-    try:
-        fr_id_lower = fr_id.lower()
-        title_lower = fr_title.lower() if fr_title else ""
-        matches: list[tuple[str, int]] = []
-        with brd_path.open(encoding="utf-8", errors="replace") as fh:
-            for lineno, line in enumerate(fh, 1):
-                stripped = line.lstrip()
-                if not stripped.startswith("#"):
-                    continue
-                heading_lower = stripped.lower()
-                if fr_id_lower in heading_lower or (title_lower and title_lower in heading_lower):
-                    matches.append((stripped.rstrip(), lineno))
-        return matches
-    except Exception:
-        return []
-
-
-def _check_brd_traced(fr: FRModule, bmad_base_dir: Path | None = None) -> bool:
-    """Dim 3: brd_traced — multi-level BRD evidence check.
-
-    Level 1: BRD document exists with a non-empty path → True.
-    Level 2: BRD document has sections with valid line ranges → True.
-    Level 3: BRD markdown contains a heading mentioning the FR ID → True.
-
-    Levels are checked in order; first match wins.
-    """
-    for doc in fr.documents:
-        if doc.doc_type != "BRD":
-            continue
-        # Level 1: existence-based — non-empty path is enough
-        if doc.path:
-            return True
-        # Level 2: section line ranges (original logic)
-        for section in doc.sections:
-            start, end = section.lines
-            if end > start and end > 0:
-                return True
-        # Level 3: heading scan via absolute path resolution
-        if bmad_base_dir and doc.path:
-            full = Path(doc.path) if Path(doc.path).is_absolute() else bmad_base_dir / doc.path
-            if _detect_brd_sections(full, fr_id=fr.fr_id):
-                return True
-    return False
-
-
-def _aggregate_activity_score(analyses: list[RepoAnalysis]) -> float:
-    """Dim 4: average code_activity_score across repos that have code.
-
-    Computes per-repo score from stored RepoAnalysis fields (avoids re-running
-    git commands).  Returns 0.0 if no repos have code.
-    """
-    from progress_report_orchestrator.analyzers.git_activity import (
-        _W_DIVERSITY,
-        _W_RECENCY,
-        _W_VOLUME,
-        _diversity_score,
-        _recency_score,
-        _volume_score,
-    )
-
-    scores: list[float] = []
-    for repo in analyses:
-        if not repo.has_code:
-            continue
-        r = _recency_score(repo.last_commit_date)
-        v = _volume_score(repo.commit_count_30d)
-        d = _diversity_score(len(repo.contributors))
-        scores.append(round(_W_RECENCY * r + _W_VOLUME * v + _W_DIVERSITY * d, 2))
-
-    return round(sum(scores) / len(scores), 2) if scores else 0.0
-
-
-def _aggregate_test_score(analyses: list[RepoAnalysis]) -> float:
-    """Dim 6: weighted-average test_coverage_score across repos with source code.
-
-    Weight = java_files + ts_files (source file count, proxy for repo size).
-    Returns 0.0 if no repos have source code.
-    """
-    from progress_report_orchestrator.analyzers.test_scanner import (
-        _RATIO_MULTIPLIER,
-    )
-
-    total_source: int = 0
-    total_test: int = 0
-    for repo in analyses:
-        if not repo.has_code:
-            continue
-        # Approximate source = total files minus test files
-        source_files = max(repo.java_files + repo.ts_files - repo.test_files, 0)
-        total_source += source_files
-        total_test += repo.test_files
-
-    if total_source == 0:
-        return 0.0
-
-    # Use test_files / source_files ratio as surrogate (same multiplier as test_scanner)
-    ratio = total_test / total_source
-    return round(min(ratio * _RATIO_MULTIPLIER, 100.0), 2)
-
-
-def _compute_deterministic_arch_score(analyses: list[RepoAnalysis]) -> float | None:
-    """Dim 7 pre-check: hexagonal compliance floor.
-
-    Returns None when no repos have Java code (hex analysis not applicable).
-    Returns 0.0-100.0 when at least one repo has Java code.
-    """
-
-    java_repos = [a for a in analyses if a.java_files > 0]
-    if not java_repos:
-        return None  # No Java — hex analysis not applicable; LLM may still score
-
-    scores: list[float] = []
-    for _repo in java_repos:
-        # We need the repo_path — it's not stored in RepoAnalysis, so we
-        # re-derive from the slug. Callers that need this should pass paths.
-        # For aggregation, we pass 0.0 and let the LLM be sole authority.
-        # The per-FR full hex check is done in analyze_structural_fr where
-        # repo_paths is available.
-        scores.append(0.0)
-
-    return round(sum(scores) / len(scores), 2) if scores else None
-
-
-def _compute_hex_score_from_paths(
-    analyses: list[RepoAnalysis],
-    repo_paths: dict[str, Path],
-) -> float | None:
-    """Compute deterministic hex score using actual repo paths.
-
-    Returns None when no repos have Java code.
-    """
-    from progress_report_orchestrator.analyzers.hexagonal import (
-        score_repo_hexagonal,
-    )
-
-    java_repos = [a for a in analyses if a.java_files > 0]
-    if not java_repos:
-        return None
-
-    scores: list[float] = []
-    for repo in java_repos:
-        path = repo_paths.get(repo.repo_slug)
-        if path is None or not path.is_dir():
-            scores.append(0.0)
-            continue
-        scores.append(score_repo_hexagonal(path))
-
-    return round(sum(scores) / len(scores), 2) if scores else None
-
-
-# ---------------------------------------------------------------------------
-# Per-FR analysis
-# ---------------------------------------------------------------------------
-
-
-def analyze_structural_fr(
-    fr: FRModule,
-    repo_paths: dict[str, Path],
-    bmad_base_dir: Path | None = None,
-) -> FRAssessment:
-    """Compute dims 1-6 + deterministic_arch_score for one FR.
-
-    Called from a thread-pool executor — must be thread-safe (pure function
-    aside from logging).
-    """
-    service_exists, repo_analyses = _check_service_exists(fr, repo_paths)
-    doc_chain_complete = _check_doc_chain_complete(fr)
-    brd_traced = _check_brd_traced(fr, bmad_base_dir)
-
-    # Dims 4 and 6: aggregate from per-repo RepoAnalysis
-    code_activity_score = _aggregate_activity_score(repo_analyses)
-    test_coverage_score = _aggregate_test_score(repo_analyses)
-
-    # Dim 7 pre-check: deterministic hexagonal compliance floor
-    deterministic_arch_score = _compute_hex_score_from_paths(repo_analyses, repo_paths)
-
-    assessment = FRAssessment(
-        fr_id=fr.fr_id,
-        service_exists=service_exists,
-        service_has_code=service_exists,
-        doc_chain_complete=doc_chain_complete,
-        brd_traced=brd_traced,
-        code_activity_score=code_activity_score,
-        test_coverage_score=test_coverage_score,
-        deterministic_arch_score=deterministic_arch_score,
-        # Dims 7-8 remain None until LLM phase (or 0.0 for empty repos there)
-        repo_analyses=repo_analyses,
-    )
-    assessment.overall_score = compute_overall_score(assessment)
-
-    log.debug(
-        "fr_structural_done",
-        fr_id=fr.fr_id,
-        service_exists=service_exists,
-        doc_chain_complete=doc_chain_complete,
-        brd_traced=brd_traced,
-        code_activity=round(code_activity_score, 1),
-        test_coverage=round(test_coverage_score, 1),
-        deterministic_arch=deterministic_arch_score,
-        score=assessment.overall_score,
-    )
-    return assessment
-
-
-# ---------------------------------------------------------------------------
-# Phase entry point
-# ---------------------------------------------------------------------------
-
-
-async def run_structural_phase(
-    topology: ProjectTopology,
-    repo_paths: dict[str, Path],
-    max_concurrent: int = 4,
-    bmad_base_dir: Path | None = None,
-) -> list[FRAssessment]:
-    """Phase 3: compute dims 1-6 + hex pre-check for all FRs.
-
-    File-system globbing and subprocess calls are synchronous I/O, so each
-    FR analysis is dispatched to the default thread-pool executor to avoid
-    blocking the event loop.
-
-    TSK-E.2: bounded_gather enforces the max_concurrent limit so the thread
-    pool is not saturated when FR count > max_concurrent.
-    """
-    from progress_report_orchestrator.pipeline.concurrency import bounded_gather
-
-    loop = asyncio.get_running_loop()
-    fr_modules = topology.fr_modules
-    log.info("structural_phase_start", fr_count=len(fr_modules), max_concurrent=max_concurrent)
-
-    async def _run_in_thread(fr: FRModule) -> FRAssessment:
-        return await loop.run_in_executor(None, analyze_structural_fr, fr, repo_paths, bmad_base_dir)
-
-    assessments = await bounded_gather([_run_in_thread(fr) for fr in fr_modules], max_concurrent)
-    log.info("structural_phase_done", fr_count=len(assessments))
-    return assessments

package/tools/vds-scripts/progress_report_orchestrator/src/progress_report_orchestrator/pipeline/sync.py

@@ -1,143 +0,0 @@
-"""Repo sync phase — thin wrapper around the vds-audit sync ecosystem.
-
-ADR-6 compliance: do NOT rebuild sync logic.  Primary path delegates to
-``vds-cli audit sync repos`` via subprocess.  Fallback activates only when
-the ``vds-cli audit`` subcommand is unavailable (e.g. standalone installation).
-"""
-
-from __future__ import annotations
-
-import shutil
-import subprocess
-from pathlib import Path
-
-import structlog
-
-log = structlog.get_logger(__name__)
-
-
-def sync_repos(
-    bitbucket_project: str,
-    cache_dir: Path,
-    concurrency: int = 5,
-) -> dict[str, Path]:
-    """Clone/pull all project repos to the local cache.
-
-    Primary path:
-        Delegates to ``vds-cli audit sync repos`` — battle-tested 200-line
-        implementation in audit_orchestrator/sync_cmds.py.
-
-    Fallback path (vds-audit absent):
-        Uses ``vds_bitbucket_orchestrator.create_client()`` + git subprocess.
-
-    Repo discovery:
-        After sync, walks ``cache_dir/<project>-project/`` for subdirectories
-        that contain ``.git/``.  Stdout from vds-audit is NOT parsed.
-
-    Returns:
-        ``{repo_slug: path}`` mapping for all discovered repos.
-    """
-    project_dir = cache_dir / f"{bitbucket_project}-project"
-
-    vds_audit_bin = shutil.which("vds-audit")
-    if vds_audit_bin:
-        _sync_via_vds_audit(bitbucket_project, project_dir, vds_audit_bin, concurrency)
-    else:
-        log.warning(
-            "vds_audit_not_found",
-            reason="falling back to direct Bitbucket client",
-        )
-        _sync_via_bitbucket_client(bitbucket_project, project_dir)
-
-    return _discover_repos(project_dir)
-
-
-# ---------------------------------------------------------------------------
-# Internal helpers
-# ---------------------------------------------------------------------------
-
-
-def _sync_via_vds_audit(
-    project: str,
-    project_dir: Path,
-    vds_audit_bin: str,
-    concurrency: int,
-) -> None:
-    """Call ``vds-cli audit sync repos`` and degrade gracefully on failure."""
-    cmd = [
-        vds_audit_bin,
-        "sync",
-        "repos",
-        "--bitbucket-project",
-        project,
-        "--workspace",
-        str(project_dir) + "/",
-        "--concurrency",
-        str(concurrency),
-    ]
-    log.info("sync_via_vds_audit_start", cmd=" ".join(cmd))
-
-    result = subprocess.run(cmd, capture_output=True, text=True)
-
-    if result.returncode != 0:
-        log.warning(
-            "vds_audit_sync_failed",
-            exit_code=result.returncode,
-            stderr=result.stderr[:2000],
-            reason="degrading to last-cached state; staleness warning",
-        )
-    else:
-        log.info("vds_audit_sync_success")
-
-
-def _sync_via_bitbucket_client(project: str, project_dir: Path) -> None:
-    """Fallback: direct BitbucketCliClient + git clone/pull."""
-    try:
-        from vds_bitbucket_orchestrator import create_client  # type: ignore[import]
-    except ImportError:
-        log.warning(
-            "bitbucket_client_unavailable",
-            reason="neither vds-audit nor vds_bitbucket_orchestrator available; using cache only",
-        )
-        return
-
-    client = create_client()
-    repos: list[dict] = client.list_repos(project_key=project)
-    project_dir.mkdir(parents=True, exist_ok=True)
-
-    for repo in repos:
-        slug: str = repo.get("slug", "")
-        clone_url: str = repo.get("clone_url", "")
-        dest = project_dir / slug
-
-        if dest.is_dir():
-            log.info("repo_pull", slug=slug)
-            subprocess.run(
-                ["git", "-C", str(dest), "pull", "--ff-only"],
-                capture_output=True,
-            )
-        elif clone_url:
-            log.info("repo_clone", slug=slug, dest=str(dest))
-            subprocess.run(["git", "clone", clone_url, str(dest)], capture_output=True)
-        else:
-            log.warning("repo_skip_no_url", slug=slug)
-
-
-def _discover_repos(project_dir: Path) -> dict[str, Path]:
-    """Walk project_dir for subdirs containing ``.git/`` → ``{slug: path}``."""
-    repos: dict[str, Path] = {}
-
-    if not project_dir.is_dir():
-        log.warning("project_dir_not_found", path=str(project_dir))
-        return repos
-
-    for candidate in sorted(project_dir.iterdir()):
-        if candidate.is_dir() and (candidate / ".git").is_dir():
-            repos[candidate.name] = candidate
-
-    log.info(
-        "repos_discovered",
-        count=len(repos),
-        project_dir=str(project_dir),
-    )
-    return repos

package/tools/vds-scripts/progress_report_orchestrator/src/progress_report_orchestrator/recommendations/__init__.py

@@ -1,5 +0,0 @@
-"""Recommendation engine — parameterized, score-aware recommendations per FR."""
-
-from .engine import Recommendation, RecommendationContext, RecommendationEngine
-
-__all__ = ["RecommendationEngine", "Recommendation", "RecommendationContext"]

package/tools/vds-scripts/progress_report_orchestrator/src/progress_report_orchestrator/recommendations/engine.py

@@ -1,105 +0,0 @@
-"""Recommendation engine — core data models and orchestration.
-
-Generates scored, parameterized recommendations from an FRAssessment.
-Recommendations are sorted descending by impact_score so callers can
-pick the top-N most impactful items.
-"""
-
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import Any
-
-from progress_report_orchestrator.models.analysis import FRAssessment
-from progress_report_orchestrator.models.topology import FRModule
-
-# ---------------------------------------------------------------------------
-# Dimension weight registry
-# ---------------------------------------------------------------------------
-
-# Percentage-scale weights (values sum to 100.0).
-# Mirrors compute_overall_score() in models/analysis.py — MUST stay in sync.
-# code_size weight is 0 (informational only; excluded from overall_score).
-DIMENSION_WEIGHTS: dict[str, float] = {
-    "service_existence": 20.0,
-    "doc_coverage": 10.0,
-    "brd_traceability": 5.0,
-    "code_activity": 15.0,
-    "code_size": 0.0,  # informational only
-    "test_coverage": 15.0,
-    "architecture_compliance": 20.0,
-    "endpoint_coverage": 15.0,
-}
-
-# Structural mode: dims 7-8 are absent; renormalize over the remaining total.
-_STRUCTURAL_DIMS: frozenset[str] = frozenset(
-    {"service_existence", "doc_coverage", "brd_traceability", "code_activity", "test_coverage"}
-)
-_STRUCTURAL_TOTAL: float = sum(DIMENSION_WEIGHTS[d] for d in _STRUCTURAL_DIMS)  # 65.0
-
-
-# ---------------------------------------------------------------------------
-# Data models
-# ---------------------------------------------------------------------------
-
-
-@dataclass(frozen=True)
-class Recommendation:
-    """A scored, parameterized recommendation for a single FR and dimension."""
-
-    dimension: str  # matches DIMENSION_WEIGHTS keys
-    severity: str  # "high" | "medium" | "low"
-    impact_score: float  # percentage points gained if dimension reaches target
-    summary: str  # one-line action item (FR ID + action)
-    detail: str  # multi-line with repo names, file counts, specific guidance
-    target_value: Any = None  # target score / label (e.g., 80.0 or "≥1 source files")
-    current_value: Any = None  # current score / label (e.g., 14.3 or "0 files")
-
-
-@dataclass(frozen=True)
-class RecommendationContext:
-    """All data needed to generate recommendations for one FR."""
-
-    assessment: FRAssessment  # has repo_analyses: list[RepoAnalysis]
-    fr: FRModule | None = None  # topology data (documents, target_services, use_cases)
-
-
-# ---------------------------------------------------------------------------
-# Engine
-# ---------------------------------------------------------------------------
-
-
-class RecommendationEngine:
-    """Generate parameterized recommendations sorted descending by impact."""
-
-    def generate(self, ctx: RecommendationContext) -> list[Recommendation]:
-        """Return all applicable recommendations, sorted descending by impact_score."""
-        from progress_report_orchestrator.recommendations.templates import (
-            rec_architecture_compliance,
-            rec_brd_traced,
-            rec_code_activity,
-            rec_code_size,
-            rec_doc_chain,
-            rec_endpoint_coverage,
-            rec_service_existence,
-            rec_test_coverage,
-        )
-
-        template_fns = [
-            rec_service_existence,
-            rec_doc_chain,
-            rec_brd_traced,
-            rec_code_activity,
-            rec_code_size,
-            rec_test_coverage,
-            rec_architecture_compliance,
-            rec_endpoint_coverage,
-        ]
-
-        a = ctx.assessment
-        recs = [fn(a) for fn in template_fns]
-        return sorted(
-            (r for r in recs if r is not None),
-            key=lambda r: r.impact_score,
-            reverse=True,
-        )