@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/progress_report_orchestrator/src/progress_report_orchestrator/state/schema.py

@@ -1,625 +0,0 @@
-from __future__ import annotations
-
-import structlog
-
-log = structlog.get_logger(__name__)
-
-# ---------------------------------------------------------------------------
-# Schema ownership — phase P4 (Alembic Migration Standardization).
-#
-# DDL formerly inlined here (the ``_TABLES`` dict + ``init_schema()``) has been
-# subsumed by Alembic. See ``progress_report_orchestrator/alembic/versions/
-# 0001_initial_progress_schema.py`` for the squashed initial migration.
-#
-# Operators must run::
-#
-#     alembic -c progress_report_orchestrator/alembic.ini stamp 0001
-#     alembic -c progress_report_orchestrator/alembic.ini upgrade head
-#
-# (Or ``make run-progress-migrate`` once that target lands.) The CLI no longer
-# auto-creates tables on first run.
-# ---------------------------------------------------------------------------
-
-# Canonical ordered list — progress_runs must exist before FK-referencing tables.
-# Kept as data so check_schema_status() and the ``vds-progress db status`` UI
-# can verify alembic-managed tables are actually present.
-_TABLE_ORDER = ["progress_runs", "fr_assessments", "document_hashes", "service_analyses", "daily_scores"]
-
-EXPECTED_TABLES: frozenset[str] = frozenset(_TABLE_ORDER)
-
-
-def _connect(dsn: str):
-    """Open a psycopg connection and pin search_path to the progress schema.
-
-    Post-P8 (v2.15.6): the legacy ``vds_progress`` standalone database (port
-    5434) was retired. Connections now route through PgBouncer's `vds_progress`
-    stanza into the shared `vds_platform` database. PgBouncer's per-stanza
-    `connect_query` only fires once per backend lifetime, and the global
-    `server_reset_query=DISCARD ALL` resets session state (including
-    `search_path`) to the system default between client checkouts in session
-    mode. Unqualified table references like ``SELECT FROM progress_runs`` then
-    fail because the post-reset search_path is ``"$user", public`` rather than
-    ``progress, public``.
-
-    This helper performs ``SET search_path TO progress, public`` immediately
-    after every connect so the rest of the module can keep using unqualified
-    table names. Schema-qualified DDL in the alembic migrations is unaffected.
-    """
-    import psycopg
-
-    conn = psycopg.connect(dsn)
-    with conn.cursor() as cur:
-        cur.execute("SET search_path TO progress, public")
-    return conn
-
-
-def check_schema_status(dsn: str) -> dict[str, bool]:
-    """Return mapping of table_name → exists for all expected tables.
-
-    Looks in the ``progress`` schema (per ADR-INF-10/11). Falls back to
-    ``public`` for legacy databases that pre-date the Alembic migration so
-    operators can still run ``vds-progress db status`` against a not-yet-
-    migrated cluster and see what's missing.
-    """
-
-    with _connect(dsn) as conn, conn.cursor() as cur:
-        cur.execute(
-            """
-                SELECT tablename FROM pg_tables
-                WHERE schemaname IN ('progress', 'public')
-                  AND tablename = ANY(%s)
-                """,
-            (list(EXPECTED_TABLES),),
-        )
-        present = {row[0] for row in cur.fetchall()}
-
-    return {table: table in present for table in _TABLE_ORDER}
-
-
-# ---------------------------------------------------------------------------
-# TSK-D.1: Delta query helpers
-# ---------------------------------------------------------------------------
-
-
-def get_previous_run_scores(
-    dsn: str,
-    project_key: str,
-    exclude_run_id: str | None = None,
-) -> tuple[str | None, dict[str, float]]:
-    """Return (prev_run_id, {fr_id: overall_score}) for the most recent completed run.
-
-    Excludes ``exclude_run_id`` (the current run) so a run is never its own baseline.
-    Returns (None, {}) when no prior completed run exists.
-    """
-
-    with _connect(dsn) as conn, conn.cursor() as cur:
-        query = """
-                SELECT run_id FROM progress_runs
-                WHERE project_key = %s AND status = 'complete'
-            """
-        params: list[object] = [project_key]
-        if exclude_run_id:
-            query += " AND run_id != %s"
-            params.append(exclude_run_id)
-        query += " ORDER BY started_at DESC LIMIT 1"
-
-        cur.execute(query, params)
-        row = cur.fetchone()
-        if not row:
-            return None, {}
-        prev_run_id: str = row[0]
-
-        cur.execute(
-            "SELECT fr_id, overall_score FROM fr_assessments WHERE run_id = %s",
-            (prev_run_id,),
-        )
-        scores = {r[0]: float(r[1]) for r in cur.fetchall() if r[1] is not None}
-
-    return prev_run_id, scores
-
-
-# ---------------------------------------------------------------------------
-# TSK-L.8: Dimension trend history for sparkline rendering
-# ---------------------------------------------------------------------------
-
-# Maps DimensionCoverageRecord key → fr_assessments column name
-_DIM_TREND_COLS: dict[str, str] = {
-    "service_exists": "service_exists",
-    "doc_chain_complete": "doc_chain_complete",
-    "brd_traced": "brd_traced",
-    "code_activity_avg": "code_activity_score",
-    "test_coverage_avg": "test_coverage_score",
-    "architecture_compliance_avg": "architecture_compliance",
-    "endpoint_coverage_avg": "endpoint_coverage",
-}
-
-
-def get_dimension_trend_history(
-    dsn: str,
-    project_key: str,
-    dimension: str,
-    limit: int = 5,
-) -> list[float]:
-    """Return per-run avg values for a dimension over the last ``limit`` completed runs.
-
-    Returns values oldest-first (chronological order) so sparklines read left→right.
-    Returns an empty list when no completed runs exist.
-
-    ``dimension`` must be a key in ``_DIM_TREND_COLS`` (matches DimensionCoverageRecord.key).
-    """
-    col = _DIM_TREND_COLS.get(dimension)
-    if not col:
-        return []
-
-    with _connect(dsn) as conn, conn.cursor() as cur:
-        cur.execute(
-            f"""
-                SELECT AVG(fa.{col})
-                FROM fr_assessments fa
-                JOIN progress_runs pr ON fa.run_id = pr.run_id
-                WHERE pr.project_key = %s
-                  AND pr.status = 'complete'
-                  AND fa.{col} IS NOT NULL
-                GROUP BY pr.run_id, pr.started_at
-                ORDER BY pr.started_at DESC
-                LIMIT %s
-                """,
-            (project_key, limit),
-        )
-        rows = cur.fetchall()
-
-    # Reverse to chronological order (oldest first for sparkline)
-    return [float(row[0]) for row in reversed(rows)]
-
-
-# ---------------------------------------------------------------------------
-# TSK-M.5: Explicit run baseline lookup for --baseline CLI flag
-# ---------------------------------------------------------------------------
-
-
-def get_run_scores_by_id(
-    dsn: str,
-    run_id: str,
-) -> tuple[str | None, dict[str, float]]:
-    """Return (run_id, {fr_id: overall_score}) for an explicit run_id.
-
-    Parallel to get_previous_run_scores() but takes an exact run_id rather
-    than "most recent". Returns (None, {}) when run_id does not exist or
-    has status != 'complete'.
-
-    Used by --baseline CLI flag to override default "previous run" comparison.
-    """
-
-    with _connect(dsn) as conn, conn.cursor() as cur:
-        cur.execute(
-            "SELECT run_id FROM progress_runs WHERE run_id = %s AND status = 'complete'",
-            (run_id,),
-        )
-        row = cur.fetchone()
-        if not row:
-            return None, {}
-
-        cur.execute(
-            "SELECT fr_id, overall_score FROM fr_assessments WHERE run_id = %s",
-            (run_id,),
-        )
-        scores = {r[0]: float(r[1]) for r in cur.fetchall() if r[1] is not None}
-
-    return run_id, scores
-
-
-# ---------------------------------------------------------------------------
-# TSK-D.3: Checkpoint helpers
-# ---------------------------------------------------------------------------
-
-_PHASE_ORDER = ("sync", "parse_kg", "structural", "llm_eval", "report")
-
-
-def create_run(dsn: str, run_id: str, project_key: str, config_json: dict | None = None) -> None:
-    """Insert a new progress_runs row with status='running'."""
-    import json
-
-    with _connect(dsn) as conn:
-        with conn.cursor() as cur:
-            cur.execute(
-                """
-                INSERT INTO progress_runs (run_id, project_key, status, config_json)
-                VALUES (%s, %s, 'running', %s)
-                ON CONFLICT (run_id) DO NOTHING
-                """,
-                (run_id, project_key, json.dumps(config_json) if config_json else None),
-            )
-        conn.commit()
-    log.debug("run_created", run_id=run_id, project_key=project_key)
-
-
-def update_run_phase(dsn: str, run_id: str, phase: str) -> None:
-    """Set completed_phase checkpoint after a phase finishes."""
-
-    with _connect(dsn) as conn:
-        with conn.cursor() as cur:
-            cur.execute(
-                "UPDATE progress_runs SET completed_phase = %s WHERE run_id = %s",
-                (phase, run_id),
-            )
-        conn.commit()
-    log.debug("checkpoint_updated", run_id=run_id, phase=phase)
-
-
-def complete_run(dsn: str, run_id: str, overall_score: float) -> None:
-    """Mark a run as complete with a final overall score."""
-
-    with _connect(dsn) as conn:
-        with conn.cursor() as cur:
-            cur.execute(
-                """
-                UPDATE progress_runs
-                SET status = 'complete', completed_at = NOW(), overall_score = %s
-                WHERE run_id = %s
-                """,
-                (overall_score, run_id),
-            )
-        conn.commit()
-    log.info("run_complete", run_id=run_id, overall_score=overall_score)
-
-
-def fail_run(dsn: str, run_id: str, error: str) -> None:
-    """Mark a run as failed with an error message."""
-
-    with _connect(dsn) as conn:
-        with conn.cursor() as cur:
-            cur.execute(
-                """
-                UPDATE progress_runs
-                SET status = 'failed', completed_at = NOW(), error = %s
-                WHERE run_id = %s
-                """,
-                (error[:4000], run_id),  # guard against oversized error strings
-            )
-        conn.commit()
-    log.warning("run_failed", run_id=run_id)
-
-
-def get_run(dsn: str, run_id: str) -> dict | None:
-    """Fetch a run row as a plain dict. Returns None when not found."""
-
-    with _connect(dsn) as conn, conn.cursor() as cur:
-        cur.execute(
-            """
-                SELECT run_id, project_key, status, completed_phase, error, overall_score
-                FROM progress_runs WHERE run_id = %s
-                """,
-            (run_id,),
-        )
-        row = cur.fetchone()
-        if not row:
-            return None
-        return {
-            "run_id": row[0],
-            "project_key": row[1],
-            "status": row[2],
-            "completed_phase": row[3],
-            "error": row[4],
-            "overall_score": row[5],
-        }
-
-
-def phase_already_done(completed_phase: str | None, target_phase: str) -> bool:
-    """Return True if *target_phase* was completed in a prior checkpoint.
-
-    Uses the canonical ``_PHASE_ORDER`` list so ordering is never hard-coded
-    at call sites.
-    """
-    if completed_phase is None:
-        return False
-    if completed_phase not in _PHASE_ORDER or target_phase not in _PHASE_ORDER:
-        return False
-    return _PHASE_ORDER.index(completed_phase) >= _PHASE_ORDER.index(target_phase)
-
-
-def load_fr_assessments(dsn: str, run_id: str) -> list:
-    """Load FRAssessment objects from fr_assessments for a given run_id.
-
-    TSK-E.1: used on resume when structural phase was already checkpointed so
-    the pipeline works with real values instead of an empty list.
-    """
-    from progress_report_orchestrator.models.analysis import FRAssessment
-
-    with _connect(dsn) as conn, conn.cursor() as cur:
-        cur.execute(
-            """
-                SELECT DISTINCT ON (fr_id)
-                       fr_id, service_exists, doc_chain_complete, brd_traced,
-                       code_activity_score, test_coverage_score,
-                       architecture_compliance, endpoint_coverage,
-                       overall_score, gap_summary, payload
-                FROM fr_assessments WHERE run_id = %s
-                ORDER BY fr_id, id DESC
-                """,
-            (run_id,),
-        )
-        rows = cur.fetchall()
-
-    assessments: list[FRAssessment] = []
-    for row in rows:
-        (
-            fr_id,
-            service_exists,
-            doc_chain_complete,
-            brd_traced,
-            code_activity_score,
-            test_coverage_score,
-            architecture_compliance,
-            endpoint_coverage,
-            overall_score,
-            gap_summary,
-            payload,
-        ) = row
-
-        a = FRAssessment(
-            fr_id=fr_id,
-            service_exists=bool(service_exists) if service_exists is not None else False,
-            service_has_code=bool(service_exists) if service_exists is not None else False,
-            doc_chain_complete=bool(doc_chain_complete) if doc_chain_complete is not None else False,
-            brd_traced=bool(brd_traced) if brd_traced is not None else False,
-            code_activity_score=float(code_activity_score) if code_activity_score is not None else 0.0,
-            test_coverage_score=float(test_coverage_score) if test_coverage_score is not None else 0.0,
-            architecture_compliance=float(architecture_compliance) if architecture_compliance is not None else None,
-            endpoint_coverage=float(endpoint_coverage) if endpoint_coverage is not None else None,
-            overall_score=float(overall_score) if overall_score is not None else 0.0,
-            gap_summary=gap_summary or "",
-        )
-
-        # Supplement from payload JSONB when available (future-proofing for richer saves)
-        if payload and isinstance(payload, dict) and "deterministic_arch_score" in payload:
-            a.deterministic_arch_score = payload["deterministic_arch_score"]
-
-        assessments.append(a)
-
-    log.debug("fr_assessments_loaded", run_id=run_id, count=len(assessments))
-    return assessments
-
-
-def save_fr_assessments(dsn: str, run_id: str, assessments: list) -> None:
-    """Persist a list of FRAssessment objects to the fr_assessments table.
-
-    ``payload`` JSONB preserves extra structural-phase data (notably
-    ``deterministic_arch_score``) so checkpoint resume can restore the full
-    FRAssessment state via ``load_fr_assessments``.
-    """
-    import json
-
-    with _connect(dsn) as conn:
-        with conn.cursor() as cur:
-            for a in assessments:
-                # Build payload with extra fields not mapped to explicit columns
-                payload_data: dict = {}
-                det = getattr(a, "deterministic_arch_score", None)
-                if det is not None:
-                    payload_data["deterministic_arch_score"] = det
-
-                cur.execute(
-                    """
-                    INSERT INTO fr_assessments (
-                        run_id, fr_id, service_exists, doc_chain_complete,
-                        brd_traced, code_activity_score, test_coverage_score,
-                        architecture_compliance, endpoint_coverage, overall_score,
-                        gap_summary, payload
-                    ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
-                    """,
-                    (
-                        run_id,
-                        a.fr_id,
-                        a.service_exists,
-                        a.doc_chain_complete,
-                        a.brd_traced,
-                        a.code_activity_score,
-                        a.test_coverage_score,
-                        a.architecture_compliance,
-                        a.endpoint_coverage,
-                        a.overall_score,
-                        getattr(a, "gap_summary", "") or "",
-                        json.dumps(payload_data) if payload_data else None,
-                    ),
-                )
-        conn.commit()
-    log.debug("fr_assessments_saved", run_id=run_id, count=len(assessments))
-
-
-# ---------------------------------------------------------------------------
-# TSK-F.1: Document hash helpers (incremental analysis support)
-# ---------------------------------------------------------------------------
-
-
-def save_document_hashes(dsn: str, run_id: str, hashes: list[dict]) -> None:
-    """Batch INSERT document content hashes for the given run.
-
-    Each dict must contain: doc_id, doc_path, git_sha, changed_since_last.
-    Write semantics: INSERT per (run_id, doc_id) — one row per document per run.
-    *changed_since_last* is True when the SHA differs from the previous run (or
-    no prior run exists); False when SHA matches (doc unchanged → LLM skip eligible).
-    """
-
-    if not hashes:
-        return
-
-    with _connect(dsn) as conn:
-        with conn.cursor() as cur:
-            for h in hashes:
-                cur.execute(
-                    """
-                    INSERT INTO document_hashes (run_id, doc_id, doc_path, git_sha, changed_since_last)
-                    VALUES (%s, %s, %s, %s, %s)
-                    """,
-                    (
-                        run_id,
-                        h["doc_id"],
-                        h.get("doc_path", ""),
-                        h.get("git_sha", ""),
-                        h.get("changed_since_last", True),
-                    ),
-                )
-        conn.commit()
-    log.debug("document_hashes_saved", run_id=run_id, count=len(hashes))
-
-
-def get_previous_document_hashes(dsn: str, project_key: str) -> dict[str, str]:
-    """Return {doc_id: git_sha} from the most recent completed run for *project_key*.
-
-    Returns an empty dict when no prior completed run exists.
-    Used to set *changed_since_last* flags for the current run.
-    """
-
-    with _connect(dsn) as conn, conn.cursor() as cur:
-        cur.execute(
-            """
-                SELECT run_id FROM progress_runs
-                WHERE project_key = %s AND status = 'complete'
-                ORDER BY started_at DESC LIMIT 1
-                """,
-            (project_key,),
-        )
-        row = cur.fetchone()
-        if not row:
-            return {}
-        prev_run_id: str = row[0]
-
-        cur.execute(
-            "SELECT doc_id, git_sha FROM document_hashes WHERE run_id = %s",
-            (prev_run_id,),
-        )
-        return {r[0]: r[1] for r in cur.fetchall() if r[1]}
-
-
-# ---------------------------------------------------------------------------
-# TSK-F.1: Service analysis helpers (append-only per-repo storage)
-# ---------------------------------------------------------------------------
-
-
-def save_service_analyses(dsn: str, run_id: str, assessments: list) -> None:
-    """Batch INSERT RepoAnalysis metrics as JSONB rows (append-only history).
-
-    Write semantics: one row per (run_id, repo_slug) — no upsert, history preserved.
-    Accepts a list of FRAssessment objects; deduplicates by repo_slug so each
-    repo appears once even when it covers multiple FRs.
-    """
-    import dataclasses
-    import json
-
-    seen: set[str] = set()
-    rows: list[tuple[str, str]] = []
-    for assessment in assessments:
-        for repo in getattr(assessment, "repo_analyses", []):
-            slug = repo.repo_slug
-            if slug not in seen:
-                seen.add(slug)
-                rows.append((slug, json.dumps(dataclasses.asdict(repo))))
-
-    if not rows:
-        return
-
-    with _connect(dsn) as conn:
-        with conn.cursor() as cur:
-            for repo_slug, analysis_json in rows:
-                cur.execute(
-                    """
-                    INSERT INTO service_analyses (run_id, repo_slug, repo_analysis_json)
-                    VALUES (%s, %s, %s)
-                    """,
-                    (run_id, repo_slug, analysis_json),
-                )
-        conn.commit()
-    log.debug("service_analyses_saved", run_id=run_id, repo_count=len(rows))
-
-
-# ---------------------------------------------------------------------------
-# TSK-F.1: Daily score helpers (UPSERT — latest run wins per calendar day)
-# ---------------------------------------------------------------------------
-
-
-def save_daily_scores(
-    dsn: str,
-    project_key: str,
-    run_date: str,
-    overall_score: float,
-    fr_scores: dict[str, float],
-) -> None:
-    """UPSERT composite scores for *project_key* on *run_date*.
-
-    Write semantics: ON CONFLICT (project_key, run_date) DO UPDATE — the latest
-    run on a given calendar day overwrites earlier rows for that day.
-    *fr_scores* is {fr_id: overall_score} for all assessed FRs.
-    """
-    import json
-
-    with _connect(dsn) as conn:
-        with conn.cursor() as cur:
-            cur.execute(
-                """
-                INSERT INTO daily_scores (project_key, run_date, overall_score, fr_scores_json)
-                VALUES (%s, %s::date, %s, %s)
-                ON CONFLICT (project_key, run_date)
-                DO UPDATE SET overall_score   = EXCLUDED.overall_score,
-                              fr_scores_json  = EXCLUDED.fr_scores_json
-                """,
-                (project_key, run_date, overall_score, json.dumps(fr_scores)),
-            )
-        conn.commit()
-    log.debug(
-        "daily_scores_upserted",
-        project_key=project_key,
-        run_date=run_date,
-        overall_score=overall_score,
-    )
-
-
-# ---------------------------------------------------------------------------
-# TSK-G.2: DB summary helper for the status command
-# ---------------------------------------------------------------------------
-
-
-def get_db_summary(dsn: str) -> dict:
-    """Return run count and last-run info for display in `vds-progress db status`."""
-
-    with _connect(dsn) as conn, conn.cursor() as cur:
-        cur.execute("SELECT COUNT(*) FROM progress_runs WHERE status = 'complete'")
-        run_count: int = cur.fetchone()[0] or 0  # type: ignore[index]
-
-        cur.execute(
-            """
-                SELECT started_at, overall_score FROM progress_runs
-                WHERE status = 'complete'
-                ORDER BY started_at DESC LIMIT 1
-                """
-        )
-        last_row = cur.fetchone()
-
-    return {
-        "run_count": run_count,
-        "last_run_at": str(last_row[0]) if last_row else None,
-        "last_overall_score": round(float(last_row[1]), 2) if last_row and last_row[1] is not None else None,
-    }
-
-
-# ---------------------------------------------------------------------------
-# TSK-B3: Document hash loading for LLM phase cache invalidation
-# ---------------------------------------------------------------------------
-
-
-def load_document_hashes(dsn: str, run_id: str) -> dict[str, str]:
-    """Load document hashes for a run. Returns {doc_id: content_hash}.
-
-    Reads from the document_hashes table populated by Phase 2.
-    Returns empty dict if table doesn't exist or run has no hashes.
-    """
-
-    try:
-        with _connect(dsn) as conn, conn.cursor() as cur:
-            cur.execute(
-                "SELECT doc_id, git_sha FROM document_hashes WHERE run_id = %s",
-                (run_id,),
-            )
-            return {row[0]: row[1] for row in cur.fetchall()}
-    except Exception as exc:
-        log.warning("load_document_hashes_failed", run_id=run_id, error=str(exc))
-        return {}