@ngocsangairvds/vsaf 3.2.16 → 4.0.0

package/tools/vds-scripts/scheduler_orchestrator/src/vds_scheduler_orchestrator/workflows/collectors/test_suite_collector.py

@@ -1,231 +0,0 @@
-"""Test-suite health collector.
-
-Gathers test counts, cached failures, and packages without tests.
-
-Strategy (hybrid, fastest-first):
-
-1. Cache-first - read pytest's ``.pytest_cache/v/cache/{nodeids,lastfailed}``
-   directly. This avoids spawning ``pytest --collect-only``, which can take
-   >10 s on the full ``scheduler_orchestrator`` suite and blow the per-collector
-   timeout. Sub-100 ms when the cache is fresh.
-2. Subprocess fallback - if the cache is missing or stale, spawn
-   ``pytest --collect-only -q`` with a short internal timeout so the surrounding
-   ``collector_timeout_s`` envelope (default 10 s) is never exceeded.
-
-The cache search probes both the package root (``.../scheduler_orchestrator``)
-and the legacy ``tests/`` location, since pytest stores its cache in its
-rootdir (where ``pyproject.toml`` lives) while older code looked under
-``tests/``.
-"""
-
-from __future__ import annotations
-
-import asyncio
-import contextlib
-import json
-import re
-import time
-from pathlib import Path
-
-from vds_scheduler_orchestrator.config import get_ecosystem_config
-from vds_scheduler_orchestrator.workflows.collectors.protocol import (
-    CollectorResult,
-    GoldenSignals,
-    SignalEntry,
-)
-
-# Cache freshness window - older than this we'd rather re-collect than mislead.
-# 7 days picked to handle weekend gaps without forcing a slow subprocess hop.
-_CACHE_MAX_AGE_S = 7 * 24 * 60 * 60
-
-# Internal subprocess timeout - must stay well below ``collector_timeout_s``
-# (default 10 s) so the cache-miss fallback still completes inside the envelope.
-_SUBPROCESS_TIMEOUT_S = 8.0
-
-
-def _parse_pytest_co(stdout: str) -> int:
-    """Count test functions from ``pytest --co -q`` output.
-
-    First tries the canonical ``N test[s] collected`` summary line that pytest
-    prints at the end of collection. Falls back to counting ``::`` lines for
-    compact outputs that omit the summary.
-    """
-    m = re.search(r"(\d+)\s+test[s]?\s+collected", stdout)
-    if m:
-        return int(m.group(1))
-    # Fallback: count nodeid lines (path::test_name)
-    count = 0
-    for line in stdout.strip().splitlines():
-        line = line.strip()
-        if not line or line.startswith("====="):
-            continue
-        if "::" in line:
-            count += 1
-    return count
-
-
-def _candidate_cache_dirs(test_dir: Path) -> list[Path]:
-    """Locations pytest may store its ``.pytest_cache`` for this suite.
-
-    pytest writes the cache to its rootdir (where ``pytest.ini`` /
-    ``pyproject.toml`` lives), which for the scheduler suite is the package
-    directory ``scheduler_orchestrator/``. Older code looked one level deeper
-    inside ``tests/``; we keep that as a fallback.
-    """
-    return [test_dir.parent, test_dir]
-
-
-def _read_lastfailed(cache_dir: Path) -> list[str]:
-    """Read ``.pytest_cache/v/cache/lastfailed`` JSON."""
-    path = cache_dir / ".pytest_cache" / "v" / "cache" / "lastfailed"
-    if not path.exists():
-        return []
-    try:
-        with open(path, encoding="utf-8") as fh:
-            data = json.load(fh)
-        return list(data.keys())
-    except Exception:
-        return []
-
-
-def _read_lastfailed_any(test_dir: Path) -> list[str]:
-    """Read ``lastfailed`` from the first candidate cache dir that has it."""
-    for candidate in _candidate_cache_dirs(test_dir):
-        failures = _read_lastfailed(candidate)
-        if failures:
-            return failures
-    return []
-
-
-def _read_nodeids(cache_dir: Path) -> int:
-    """Count test nodeids from ``.pytest_cache/v/cache/nodeids``."""
-    path = cache_dir / ".pytest_cache" / "v" / "cache" / "nodeids"
-    if not path.exists():
-        return 0
-    try:
-        text = path.read_text(encoding="utf-8").strip()
-        if not text:
-            return 0
-        # nodeids is JSON-encoded list - fall back to line-count for legacy layouts.
-        try:
-            data = json.loads(text)
-            if isinstance(data, list):
-                return len(data)
-        except json.JSONDecodeError:
-            pass
-        return len(text.splitlines())
-    except Exception:
-        return 0
-
-
-def _read_fresh_nodeids(test_dir: Path, max_age_s: float = _CACHE_MAX_AGE_S) -> int:
-    """Return nodeid count from the freshest cache within ``max_age_s``.
-
-    Returns 0 when no candidate cache is recent enough; the caller then falls
-    back to a pytest subprocess.
-    """
-    now = time.time()
-    for candidate in _candidate_cache_dirs(test_dir):
-        path = candidate / ".pytest_cache" / "v" / "cache" / "nodeids"
-        if not path.exists():
-            continue
-        try:
-            mtime = path.stat().st_mtime
-        except OSError:
-            continue
-        if now - mtime > max_age_s:
-            continue
-        count = _read_nodeids(candidate)
-        if count:
-            return count
-    return 0
-
-
-async def _collect_via_subprocess(test_dir: Path, timeout_s: float) -> int:
-    """Run ``pytest --collect-only -q`` with a hard internal timeout."""
-    try:
-        proc = await asyncio.create_subprocess_exec(
-            "pytest",
-            str(test_dir),
-            "--co",
-            "-q",
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-        )
-    except OSError:
-        return 0
-    try:
-        stdout_bytes, _ = await asyncio.wait_for(
-            proc.communicate(), timeout=timeout_s
-        )
-    except TimeoutError:
-        with contextlib.suppress(ProcessLookupError):
-            proc.kill()
-        return 0
-    return _parse_pytest_co(stdout_bytes.decode())
-
-
-class TestSuiteCollector:
-    """Collect test-suite metrics via pytest cache + subprocess fallback."""
-
-    section = "Test Suite"
-
-    async def collect(self) -> CollectorResult:
-        cfg = get_ecosystem_config()
-        base = cfg.monorepo_root
-
-        test_dir = base / "WHO-project" / "vds-scripts" / "scheduler_orchestrator" / "tests"
-
-        # Tier 1 - fresh cache (sub-100 ms).
-        total_tests = _read_fresh_nodeids(test_dir)
-
-        # Tier 2 - subprocess fallback when cache is absent or stale.
-        if total_tests == 0:
-            total_tests = await _collect_via_subprocess(
-                test_dir, _SUBPROCESS_TIMEOUT_S
-            )
-
-        # Tier 3 - last-resort stale cache read; better than reporting 0.
-        if total_tests == 0:
-            for candidate in _candidate_cache_dirs(test_dir):
-                stale_count = _read_nodeids(candidate)
-                if stale_count:
-                    total_tests = stale_count
-                    break
-
-        try:
-            lastfailed = _read_lastfailed_any(test_dir)
-        except OSError:
-            lastfailed = []
-
-        anomalies: list[str] = []
-        if lastfailed:
-            anomalies.append(f"{len(lastfailed)} cached test failure(s) from last run")
-
-        return CollectorResult(
-            section=self.section,
-            metrics={
-                "total_tests": total_tests,
-                "lastfailed_count": len(lastfailed),
-                "lastfailed": lastfailed[:10],
-            },
-            anomalies=anomalies,
-            golden_signals=GoldenSignals(
-                errors=[
-                    SignalEntry(
-                        label="lastfailed",
-                        value=float(len(lastfailed)),
-                        display=f"{len(lastfailed)} cached failures",
-                        polarity="lower-is-better",
-                    )
-                ],
-                traffic=[
-                    SignalEntry(
-                        label="total_tests",
-                        value=float(total_tests),
-                        display=f"{total_tests} tests",
-                        polarity="higher-is-better",
-                    )
-                ],
-            ),
-        )

package/tools/vds-scripts/scheduler_orchestrator/src/vds_scheduler_orchestrator/workflows/collectors/trend_forecast.py

@@ -1,417 +0,0 @@
-"""Trend forecasting collector (Tier 3 — Phase 3 Wave 2).
-
-Queries 7-day history from ``sched.ecosystem_report_runs``, computes a
-moving z-score for the four numerical workflow KPIs, classifies each
-metric's trend direction, and renders a Unicode sparkline per metric.
-
-Bootstrap behavior (NFR-11): until ``ecosystem_report_runs`` has at least
-3 distinct historical rows, the collector emits an ``[unavailable]``
-anomaly and returns an empty metric block. This is **expected** during
-the first 7 days after deploy and is not itself an anomaly that should
-deduct health points.
-
-Z-score thresholds (AC-P3.4):
-- ``|z| > 2.0`` → ``unusual``
-- ``|z| > 3.0`` → ``anomalous``
-
-If ``scipy`` and ``numpy`` are importable, the collector uses a
-Shapiro-Wilk normality gate (``p < 0.05``) to fall back to an IQR-based
-outlier check (multiplier 1.5). When those libraries are not available
-the IQR path is silently skipped — the moving z-score is the canonical
-signal regardless.
-"""
-
-from __future__ import annotations
-
-import math
-import statistics
-from datetime import date, timedelta
-from typing import TYPE_CHECKING, Any
-
-import structlog
-from sqlalchemy import desc, select
-from sqlalchemy.exc import SQLAlchemyError
-
-from vds_scheduler_orchestrator.db.models import EcosystemReportRun
-from vds_scheduler_orchestrator.workflows.collectors.protocol import (
-    CollectorResult,
-    GoldenSignals,
-    SignalEntry,
-)
-from vds_scheduler_orchestrator.workflows.registry import _make_session
-
-if TYPE_CHECKING:  # pragma: no cover - typing only
-    from sqlalchemy.orm import Session
-
-logger = structlog.get_logger(__name__)
-
-# Sparkline alphabet — 8 evenly spaced Unicode block characters.
-_SPARK_BARS = "▁▂▃▄▅▆▇█"
-
-# Numerical metrics to trend over the 7-day history window. The keys map
-# the model column name to the human-readable label shown in the report.
-_TRENDED_METRICS: dict[str, str] = {
-    "error_24h": "errors_24h",
-    "success_24h": "success_24h",
-    "pending_depth": "pending_depth",
-    "enqueued_depth": "enqueued_depth",
-    "health_score": "health_score",
-}
-
-_HISTORY_WINDOW_DAYS = 7
-_MIN_HISTORY_FOR_ZSCORE = 3
-_UNUSUAL_Z = 2.0
-_ANOMALOUS_Z = 3.0
-
-
-class TrendForecastCollector:
-    """Compute 7-day moving z-scores and sparklines per numerical metric.
-
-    Implements the ``EcosystemCollector`` protocol. Stateless — every call
-    to :meth:`collect` opens its own SQLAlchemy session via
-    :func:`vds_scheduler_orchestrator.workflows.registry._make_session`.
-    """
-
-    section = "Trend Forecast"
-
-    # ── static analytic helpers ──────────────────────────────────────────────
-
-    @staticmethod
-    def _moving_zscore(series: list[float], window: int = 7) -> float:
-        """Return the moving z-score for the latest value in ``series``.
-
-        The mean and standard deviation are computed over the last
-        ``window`` historical points (excluding the most recent). Returns
-        ``0.0`` when fewer than 2 points are available or when the
-        historical std-dev is zero.
-        """
-        if len(series) < 2:
-            return 0.0
-        history = series[-window - 1:-1] if len(series) > window else series[:-1]
-        if len(history) < 1:
-            return 0.0
-        mean = statistics.mean(history)
-        std = statistics.stdev(history) if len(history) > 1 else 0.0
-        if std == 0:
-            return 0.0
-        return (series[-1] - mean) / std
-
-    @staticmethod
-    def _iqr_anomaly(series: list[float], multiplier: float = 1.5) -> bool:
-        """Return ``True`` when the latest value is an IQR outlier.
-
-        Activated only when Shapiro-Wilk reports the historical
-        distribution as non-normal (``p < 0.05``). Requires ``numpy`` and
-        ``scipy`` — the caller is expected to have already verified that
-        both imports succeed.
-        """
-        if len(series) < 4:
-            return False
-        try:
-            import numpy as np  # local import — optional dep
-        except ImportError:  # pragma: no cover - guarded by caller
-            return False
-        history = series[:-1]
-        q1, q3 = np.percentile(history, [25, 75])
-        iqr = float(q3 - q1)
-        if iqr == 0:
-            return False
-        lower = float(q1) - multiplier * iqr
-        upper = float(q3) + multiplier * iqr
-        latest = float(series[-1])
-        return bool(latest < lower or latest > upper)
-
-    @staticmethod
-    def _sparkline(series: list[float]) -> str:
-        """Render a Unicode sparkline (8 levels) from a numeric series."""
-        if not series:
-            return ""
-        mn, mx = min(series), max(series)
-        if mn == mx:
-            return _SPARK_BARS[3] * len(series)
-        span = mx - mn
-        return "".join(
-            _SPARK_BARS[int((v - mn) / span * 7)] for v in series
-        )
-
-    @staticmethod
-    def _trend_direction(series: list[float]) -> str:
-        """Classify ``series`` as ``up``, ``down``, or ``flat``.
-
-        Compares the average of the second half of the series to the
-        average of the first half. Constant or near-constant series
-        (delta ≤ 5% of the historical mean magnitude) are reported as
-        ``flat``.
-        """
-        if len(series) < 2:
-            return "flat"
-        midpoint = len(series) // 2
-        first = series[:midpoint] if midpoint > 0 else series[:1]
-        second = series[midpoint:] if midpoint > 0 else series[1:]
-        if not first or not second:
-            return "flat"
-        first_avg = statistics.mean(first)
-        second_avg = statistics.mean(second)
-        delta = second_avg - first_avg
-        baseline = max(abs(first_avg), 1e-9)
-        if abs(delta) / baseline < 0.05:
-            return "flat"
-        return "up" if delta > 0 else "down"
-
-    @staticmethod
-    def _classify_zscore(z: float) -> str:
-        """Bucket a z-score into ``normal``/``unusual``/``anomalous``."""
-        absz = abs(z)
-        if absz > _ANOMALOUS_Z:
-            return "anomalous"
-        if absz > _UNUSUAL_Z:
-            return "unusual"
-        return "normal"
-
-    # ── DB access (mockable in tests) ────────────────────────────────────────
-
-    def _load_history(
-        self,
-        session: Session,
-        today: date,
-        window_days: int = _HISTORY_WINDOW_DAYS,
-    ) -> list[Any]:
-        """Fetch up to ``window_days`` of recent runs (oldest → newest).
-
-        Uses an **explicit column projection** (SQLAlchemy Core ``select``)
-        rather than ``session.query(EcosystemReportRun)`` so the SELECT
-        only references columns this collector actually needs. This keeps
-        TrendForecast resilient when newer columns (e.g. Phase 3
-        ``dora_summary``, ``trend_summary``) exist in the ORM model but
-        have not yet been added to the live DB by an alembic upgrade.
-
-        Returns lightweight ``Row`` objects whose labeled attributes are
-        consumed by :meth:`_series_for` via ``getattr``.
-        """
-        cutoff = today - timedelta(days=window_days)
-        stmt = (
-            select(
-                EcosystemReportRun.report_date,
-                EcosystemReportRun.error_24h,
-                EcosystemReportRun.success_24h,
-                EcosystemReportRun.pending_depth,
-                EcosystemReportRun.enqueued_depth,
-                EcosystemReportRun.health_score,
-            )
-            .where(EcosystemReportRun.report_date >= cutoff)
-            .order_by(desc(EcosystemReportRun.report_date))
-            .limit(window_days)
-        )
-        rows = session.execute(stmt).all()
-        # Re-sort ascending so sparklines read left=oldest, right=newest.
-        return list(reversed(rows))
-
-    @staticmethod
-    def _series_for(rows: list[Any], column: str) -> list[float]:
-        """Extract a numeric series for ``column`` from history rows.
-
-        Missing values, ``None``, ``NaN``, and missing columns are
-        filtered out — the returned series only contains finite floats.
-        """
-        series: list[float] = []
-        for row in rows:
-            raw = getattr(row, column, None)
-            if raw is None:
-                continue
-            try:
-                value = float(raw)
-            except (TypeError, ValueError):
-                continue
-            if not math.isfinite(value):
-                continue
-            series.append(value)
-        return series
-
-    # ── public protocol method ───────────────────────────────────────────────
-
-    async def collect(self) -> CollectorResult:
-        """Compute trends per metric and return a :class:`CollectorResult`.
-
-        On bootstrap (history < 3 rows) returns an ``[unavailable]``
-        anomaly with zero health deductions. On any DB failure logs the
-        error and emits a 5-point health deduction with ``[unavailable]``.
-        """
-        try:
-            session = _make_session()
-        except Exception as exc:
-            logger.warning("trend_forecast.session_unavailable", error=str(exc))
-            return CollectorResult(
-                section=self.section,
-                anomalies=[
-                    "[unavailable] trend forecast: session factory unavailable "
-                    f"({exc})"
-                ],
-                health_deductions=0,
-            )
-
-        try:
-            today = date.today()
-            rows = self._load_history(session, today)
-        except SQLAlchemyError as exc:
-            # Log full error internally, but surface a clean single-line
-            # human-readable summary. Avoid leaking SQL/schema/stack traces
-            # into the public report (defense layer 1 — collector level).
-            logger.warning("trend_forecast.query_failed", error=str(exc))
-            return CollectorResult(
-                section=self.section,
-                anomalies=[
-                    "[unavailable] trend forecast: history query failed "
-                    "(likely missing Phase-3 columns; run alembic upgrade)"
-                ],
-                health_deductions=5,
-            )
-        except Exception as exc:
-            logger.warning("trend_forecast.query_failed", error=str(exc))
-            return CollectorResult(
-                section=self.section,
-                anomalies=[
-                    "[unavailable] trend forecast: history query failed "
-                    "(unexpected error)"
-                ],
-                health_deductions=5,
-            )
-        finally:
-            session.close()
-
-        # Bootstrap — NFR-11: not enough history yet.
-        if len(rows) < _MIN_HISTORY_FOR_ZSCORE:
-            return CollectorResult(
-                section=self.section,
-                metrics={
-                    "history_days": len(rows),
-                    "min_required": _MIN_HISTORY_FOR_ZSCORE,
-                    "trends": {},
-                },
-                anomalies=[
-                    "[unavailable] trend forecast: bootstrap window "
-                    f"({len(rows)}/{_MIN_HISTORY_FOR_ZSCORE} days collected; "
-                    "expected for the first 7 days after deploy)"
-                ],
-                health_deductions=0,
-            )
-
-        trends: dict[str, dict[str, Any]] = {}
-        anomalies: list[str] = []
-        health_deductions = 0
-        traffic_signals: list[SignalEntry] = []
-        error_signals: list[SignalEntry] = []
-
-        normality_available = self._normality_module_available()
-
-        for column, label in _TRENDED_METRICS.items():
-            series = self._series_for(rows, column)
-            if len(series) < _MIN_HISTORY_FOR_ZSCORE:
-                trends[label] = {
-                    "series": series,
-                    "sparkline": self._sparkline(series),
-                    "zscore": 0.0,
-                    "classification": "insufficient_data",
-                    "direction": self._trend_direction(series),
-                    "iqr_outlier": False,
-                }
-                continue
-
-            z = self._moving_zscore(series)
-            classification = self._classify_zscore(z)
-            direction = self._trend_direction(series)
-            sparkline = self._sparkline(series)
-
-            iqr_outlier = False
-            if normality_available:
-                iqr_outlier = self._maybe_iqr_outlier(series)
-
-            trends[label] = {
-                "series": series,
-                "sparkline": sparkline,
-                "zscore": round(z, 3),
-                "classification": classification,
-                "direction": direction,
-                "iqr_outlier": iqr_outlier,
-            }
-
-            if classification == "anomalous":
-                anomalies.append(
-                    f"{label} anomalous (z={z:+.2f}, trend {direction}) `{sparkline}`"
-                )
-                health_deductions += 5
-            elif classification == "unusual":
-                anomalies.append(
-                    f"{label} unusual (z={z:+.2f}, trend {direction}) `{sparkline}`"
-                )
-                health_deductions += 2
-            elif iqr_outlier:
-                anomalies.append(
-                    f"{label} IQR outlier (non-normal distribution) `{sparkline}`"
-                )
-                health_deductions += 2
-
-            entry = SignalEntry(
-                label=label,
-                value=float(z),
-                display=f"{label} {sparkline} (z={z:+.2f}, {direction})",
-                polarity="neutral",
-            )
-            if column in {"error_24h", "pending_depth", "enqueued_depth"}:
-                entry.polarity = "lower-is-better"
-                error_signals.append(entry)
-            else:
-                traffic_signals.append(entry)
-
-        return CollectorResult(
-            section=self.section,
-            metrics={
-                "history_days": len(rows),
-                "min_required": _MIN_HISTORY_FOR_ZSCORE,
-                "trends": trends,
-            },
-            anomalies=anomalies,
-            golden_signals=GoldenSignals(
-                errors=error_signals,
-                traffic=traffic_signals,
-            ),
-            health_deductions=min(health_deductions, 25),
-        )
-
-    # ── optional-dependency helpers ──────────────────────────────────────────
-
-    @staticmethod
-    def _normality_module_available() -> bool:
-        """Return ``True`` when both numpy and scipy.stats can be imported."""
-        try:
-            import numpy
-            import scipy.stats
-        except ImportError:
-            return False
-        return True
-
-    @classmethod
-    def _maybe_iqr_outlier(cls, series: list[float]) -> bool:
-        """Run Shapiro-Wilk; if non-normal (p < 0.05) return IQR outlier flag.
-
-        Silently returns ``False`` if scipy/numpy are unavailable or if
-        the test cannot be performed (n < 3, constant series, etc).
-        """
-        if len(series) < 4:
-            return False
-        try:
-            import numpy as np
-            from scipy import stats as scipy_stats
-        except ImportError:  # pragma: no cover - optional dep
-            return False
-        history = np.asarray(series[:-1], dtype=float)
-        # Shapiro-Wilk is undefined for fewer than 3 samples or a
-        # constant series — bail out without flagging.
-        if history.size < 3 or float(history.std()) == 0.0:
-            return False
-        try:
-            _, p_value = scipy_stats.shapiro(history)
-        except Exception:
-            return False
-        if p_value >= 0.05:
-            return False
-        return cls._iqr_anomaly(series)