@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/audit_orchestrator/src/vds_audit_orchestrator/engine/loader.py

@@ -1,1070 +0,0 @@
-from __future__ import annotations
-
-import contextlib
-import json
-import re
-import subprocess
-import tempfile
-import unicodedata
-from pathlib import Path
-from typing import Any
-
-import httpx
-import pandas as pd
-from structlog import get_logger
-from vds_excel_orchestrator.converters import ExcelConverter
-
-from vds_audit_orchestrator.engine.mapping import (
-    CHECKLIST_GID,
-    CHECKLIST_SHEET_NAME,
-    RubricMapping,
-    load_mapping_file,
-    resolve_default_mapping,
-)
-from vds_audit_orchestrator.errors import TemplateLoadError
-from vds_audit_orchestrator.models.enums import MigrationEffort, Priority, Severity
-from vds_audit_orchestrator.models.template import (
-    AuditCheck,
-    AuditSection,
-    AuditTemplate,
-    CheckApplicability,
-)
-
-# Optional import for hierarchical rubric parser (FR-002.13, FR-002.16)
-try:
-    from vds_audit_orchestrator.engine.rubric import (
-        RubricConfig,
-        parse_rubric_with_mapping,
-    )
-
-    HIERARCHICAL_RUBRIC_AVAILABLE = True
-except ImportError:
-    HIERARCHICAL_RUBRIC_AVAILABLE = False  # type: ignore[reportConstantRedefinition]
-
-logger = get_logger()
-
-DEFAULT_SPREADSHEET_URL = "https://docs.google.com/spreadsheets/d/13aZlARMCDaBod8FaObEK0FwzOtqKYCDEcmF1m1jpzZE/edit"
-
-
-def resolve_mapping_path(
-    mapping_spec: str | Path | None,
-    *,
-    _bundle_name: str = "default_checklist_mapping.json",
-) -> Path | None:
-    """Resolve a mapping specification to a file path.
-
-    Args:
-        mapping_spec: Either a file path, a bundled mapping name, or None
-        bundle_name: Default bundled mapping name to use if mapping_spec is a known name
-
-    Returns:
-        Path to the mapping file, or None if no mapping should be used
-
-    Examples:
-        >>> # Full path
-        >>> resolve_mapping_path("/path/to/custom.json")
-        Path('/path/to/custom.json')
-
-        >>> # Bundled mapping name (without .json extension)
-        >>> resolve_mapping_path("default_checklist_mapping")
-        Path('.../mappings/default_checklist_mapping.json')
-
-        >>> # None
-        >>> resolve_mapping_path(None)
-        None
-    """
-    if mapping_spec is None:
-        return None
-
-    mapping_path = Path(mapping_spec) if not isinstance(mapping_spec, Path) else mapping_spec
-
-    # If it's an existing file, return it
-    if mapping_path.exists():
-        return mapping_path
-
-    # Try as a bundled mapping name (add .json if not present)
-    mapping_str = str(mapping_spec)
-    if not mapping_str.endswith(".json"):
-        mapping_str = f"{mapping_str}.json"
-
-    try:
-        from vds_audit_orchestrator.mappings import get_bundled_mapping_path
-
-        bundled = get_bundled_mapping_path(mapping_str)
-        if bundled:
-            return bundled
-    except ImportError:
-        pass
-
-    # Return as-is and let the caller handle the error
-    return mapping_path
-
-
-class ExcelLoader:
-    """Loads Audit Templates from Excel files or Google Sheets."""
-
-    def load(self, path: Path) -> AuditTemplate:
-        """Load template from Excel file."""
-        return load_template_from_excel(path)
-
-
-def load_template_from_excel(path: Path) -> AuditTemplate:
-    """Load a standard Sections/Checks/Metadata template from Excel."""
-    try:
-        # Try loading straightforward "Checklist" format first (MVP compat)
-        try:
-            return _load_simple_checklist(path)
-        except Exception as e:
-            logger.warning("simple_checklist_load_failed", error=str(e))
-
-        # Fallback to Advanced "Sections/Checks" format
-        sections_payload = ExcelConverter.to_json(path, sheet_name="Sections")
-        checks_payload = ExcelConverter.to_json(path, sheet_name="Checks")
-    except Exception as exc:
-        message = str(exc)
-        if "Sheet 'Sections'" in message or "Sheet 'Checks'" in message:
-            raise TemplateLoadError(
-                "Template workbook missing Sections or Checks sheet",
-                context={"path": str(path), "error": message},
-            ) from exc
-        raise TemplateLoadError(
-            "Failed to read template workbook",
-            context={"path": str(path), "error": str(exc)},
-        ) from exc
-
-    sections_rows = sections_payload.get("Sections", []) if isinstance(sections_payload, dict) else []
-    checks_rows = checks_payload.get("Checks", []) if isinstance(checks_payload, dict) else []
-
-    if not sections_rows:
-        raise TemplateLoadError(
-            "Template workbook missing Sections or Checks sheet",
-            context={"path": str(path)},
-        )
-
-    sections_by_id: dict[str, AuditSection] = {}
-    for row in sections_rows:
-        row_norm = {str(k).strip().lower(): v for k, v in row.items()}
-        section_id = _value_str(row_norm.get("id"))
-        name = _value_str(row_norm.get("name") or row_norm.get("title") or row_norm.get("section"))
-        description = _value_str(row_norm.get("description") or "")
-        weight = _parse_weight(row_norm.get("weight"))
-        if not section_id or not name:
-            continue
-        sections_by_id[section_id] = AuditSection(
-            id=section_id,
-            name=name,
-            description=description or name,
-            weight=weight,
-            checks=[],
-        )
-
-    for row in checks_rows:
-        row_norm = {str(k).strip().lower(): v for k, v in row.items()}
-        check_id = _value_str(row_norm.get("id"))
-        section_id = _value_str(row_norm.get("section_id") or row_norm.get("section"))
-        name = _value_str(row_norm.get("name") or row_norm.get("description"))
-        description = _value_str(row_norm.get("description") or name)
-        weight = _parse_optional_weight(row_norm.get("weight"))
-        if not check_id or not section_id or not description:
-            continue
-        section = sections_by_id.get(section_id)
-        if not section:
-            continue
-        applicability_dict = _value_dict(row_norm.get("applicability"))
-        applicability_obj = CheckApplicability(**applicability_dict) if applicability_dict else None
-        section.checks.append(
-            AuditCheck(
-                id=check_id,
-                name=name or description,
-                description=description,
-                max_points=_parse_int(row_norm.get("max_points"), default=100),
-                weight=weight or 0.0,
-                check_type=_value_str(
-                    row_norm.get("check_type") or row_norm.get("plugin") or row_norm.get("plugin_name")
-                )
-                or None,
-                section_id=section_id,
-                check_config=_value_dict(row_norm.get("check_config") or row_norm.get("plugin_config")),
-                score_mode=_value_str(row_norm.get("score_mode")) or None,
-                score_config=_value_dict(row_norm.get("score_config")),
-                required=_parse_bool(row_norm.get("required")),
-                tags=_value_list(row_norm.get("tags")),
-                severity=_parse_enum(row_norm.get("severity")),  # type: ignore[arg-type]
-                priority=_parse_enum(row_norm.get("priority"), enum_name="priority"),  # type: ignore[arg-type]
-                migration_effort=_parse_enum(row_norm.get("migration_effort"), enum_name="migration_effort"),  # type: ignore[arg-type]
-                enabled=_parse_bool(row_norm.get("enabled")) if row_norm.get("enabled") is not None else True,
-                remediation_steps=_value_str(row_norm.get("remediation")),
-                doc_link=_value_str(row_norm.get("doc_link")),
-                applicability=applicability_obj,
-            )
-        )
-
-    sections = list(sections_by_id.values())
-    _normalize_section_and_check_weights(sections)
-
-    return AuditTemplate(name=path.stem, sections=sections)
-
-
-def _load_simple_checklist(path: Path) -> AuditTemplate:
-    """Load simple MVP 'Checklist' format."""
-    xls = pd.ExcelFile(path)
-    if "Checklist" not in xls.sheet_names:
-        raise ValueError("No Checklist sheet")
-
-    df = pd.read_excel(xls, "Checklist")
-    df.columns = [str(c).lower().strip().replace(" ", "_") for c in df.columns]
-
-    sections_map = {}
-    for _idx, row in df.iterrows():
-        if bool(pd.isna(row.get("check_id"))):
-            continue
-
-        sec_id = str(row.get("section", "DEFAULT")).strip()
-        sec_title = str(row.get("section_title", sec_id)).strip()
-        sec_weight = float(row.get("section_weight") or 0)
-
-        if sec_id not in sections_map:
-            sections_map[sec_id] = AuditSection(
-                id=sec_id,
-                name=sec_title,
-                description=sec_title,
-                weight=sec_weight,
-                checks=[],
-            )
-
-        config_str = str(row.get("plugin_config", "{}"))
-        if config_str and config_str.lower() != "nan":
-            with contextlib.suppress(Exception):
-                _config = json.loads(config_str)
-
-        w = _parse_weight(row.get("weight"))
-        desc = str(row["description"]).strip()
-
-        check = AuditCheck(
-            id=str(row["check_id"]).strip(),
-            name=desc,
-            description=desc,
-            max_points=100,
-            weight=w,
-            check_type=_value_str(row.get("check_type") or row.get("plugin") or row.get("plugin_name")) or None,
-            section_id=sec_id,
-            check_config=_value_dict(row.get("check_config") or row.get("plugin_config")),
-            severity=_parse_enum(row.get("severity"), enum_name="severity"),  # type: ignore[arg-type]
-            priority=_parse_enum(row.get("priority"), enum_name="priority"),  # type: ignore[arg-type]
-            migration_effort=_parse_enum(row.get("effort"), enum_name="migration_effort"),  # type: ignore[arg-type]
-            required=bool(row.get("required", False)),
-            enabled=_parse_bool(row.get("enabled")) if row.get("enabled") is not None else True,
-            tags=_value_list(row.get("tags")),
-            remediation_steps=_value_str(row.get("remediation")),
-            doc_link=_value_str(row.get("doc_link")),
-        )
-        sections_map[sec_id].checks.append(check)
-
-    sections = list(sections_map.values())
-    _normalize_section_and_check_weights(sections)
-    return AuditTemplate(name=path.stem, sections=sections)
-
-
-def load_template_from_google_sheet(
-    *,
-    spreadsheet_url: str = DEFAULT_SPREADSHEET_URL,
-    sheet_name: str | None = None,
-    sheet_gid: int | None = None,
-    doc_table_index: int | None = None,
-    mapping_path: Path | None = None,
-    template_map: RubricMapping | None = None,
-    auth_method: str | None = None,
-    credentials_path: Path | None = None,
-    oauth_credentials_path: Path | None = None,
-    api_key: str | None = None,
-) -> AuditTemplate:
-    """Load a template from Google Sheets via vds-cli."""
-    if _is_google_docs_url(spreadsheet_url):
-        return load_template_from_google_doc(
-            doc_url=spreadsheet_url,
-            table_index=0 if doc_table_index is None else doc_table_index,
-            mapping_path=mapping_path,
-            template_map=template_map,
-        )
-    mapping = template_map
-    if mapping_path:
-        mapping = load_mapping_file(mapping_path)
-    if not mapping:
-        mapping = resolve_default_mapping(sheet_name=sheet_name, sheet_gid=sheet_gid)
-    if mapping:
-        resolved_sheet = sheet_name
-        if not resolved_sheet and sheet_gid == CHECKLIST_GID:
-            resolved_sheet = CHECKLIST_SHEET_NAME
-        if not resolved_sheet:
-            worksheets = _list_google_sheet_worksheets(
-                spreadsheet_url=spreadsheet_url,
-                auth_method=auth_method,
-                credentials_path=credentials_path,
-                oauth_credentials_path=oauth_credentials_path,
-                api_key=api_key,
-            )
-            if worksheets:
-                resolved_sheet = worksheets[0]
-        if not resolved_sheet:
-            raise TemplateLoadError(
-                "Sheet name is required when template-gid is unknown",
-                context={"sheet_gid": sheet_gid, "sheet_name": sheet_name},
-            )
-
-        rows = _read_google_sheet_rows(
-            spreadsheet_url=spreadsheet_url,
-            sheet_name=resolved_sheet,
-            auth_method=auth_method,
-            credentials_path=credentials_path,
-            oauth_credentials_path=oauth_credentials_path,
-            api_key=api_key,
-        )
-        if not rows:
-            raise TemplateLoadError(
-                "Google Sheet returned no rows",
-                context={"sheet_name": resolved_sheet, "sheet_gid": sheet_gid},
-            )
-
-        sections_payload, checks_payload = _rubric_rows_to_standard(rows, mapping)
-        payload = {
-            "Sections": sections_payload,
-            "Checks": checks_payload,
-            "Metadata": [
-                {"key": "template_version", "value": "1.0.0"},
-                {"key": "source", "value": spreadsheet_url},
-                {"key": "sheet", "value": resolved_sheet},
-            ],
-        }
-        template = _load_template_from_google_payload(payload)
-        if mapping.template_name:
-            template.name = mapping.template_name
-        return template
-
-    if sheet_name or sheet_gid:
-        raise TemplateLoadError(
-            "Template mapping is required when specifying a sheet name or gid.",
-            context={"sheet_name": sheet_name, "sheet_gid": sheet_gid},
-        )
-
-    worksheets = _list_google_sheet_worksheets(
-        spreadsheet_url=spreadsheet_url,
-        auth_method=auth_method,
-        credentials_path=credentials_path,
-        oauth_credentials_path=oauth_credentials_path,
-        api_key=api_key,
-    )
-    missing = [name for name in ("Sections", "Checks") if name not in worksheets]
-    if not missing:
-        payload: dict[str, list[dict[str, Any]]] = {}
-        for name in ("Sections", "Checks", "Metadata"):
-            if name not in worksheets:
-                continue
-            rows = _read_google_sheet_rows(
-                spreadsheet_url=spreadsheet_url,
-                sheet_name=name,
-                auth_method=auth_method,
-                credentials_path=credentials_path,
-                oauth_credentials_path=oauth_credentials_path,
-                api_key=api_key,
-            )
-            payload[name] = _rows_to_dicts(rows)
-
-        return _load_template_from_google_payload(payload)
-
-    fallback_mapping = resolve_default_mapping(sheet_name=CHECKLIST_SHEET_NAME, sheet_gid=CHECKLIST_GID)
-    if not fallback_mapping:
-        raise TemplateLoadError(
-            "Google Sheet missing required worksheets for standard schema",
-            context={"missing": missing, "worksheets": worksheets},
-        )
-
-    rows = _read_google_sheet_rows(
-        spreadsheet_url=spreadsheet_url,
-        sheet_name=CHECKLIST_SHEET_NAME,
-        auth_method=auth_method,
-        credentials_path=credentials_path,
-        oauth_credentials_path=oauth_credentials_path,
-        api_key=api_key,
-    )
-    if not rows:
-        raise TemplateLoadError(
-            "Google Sheet returned no rows",
-            context={"sheet_name": CHECKLIST_SHEET_NAME},
-        )
-    sections_payload, checks_payload = _rubric_rows_to_standard(rows, fallback_mapping)
-    fallback_payload = {
-        "Sections": sections_payload,
-        "Checks": checks_payload,
-        "Metadata": [
-            {"key": "template_version", "value": "1.0.0"},
-            {"key": "source", "value": spreadsheet_url},
-            {"key": "sheet", "value": CHECKLIST_SHEET_NAME},
-        ],
-    }
-    template = _load_template_from_google_payload(fallback_payload)
-    if fallback_mapping.template_name:
-        template.name = fallback_mapping.template_name
-    return template
-
-
-def export_template_from_google_source(
-    *,
-    output_path: Path,
-    template_url: str = DEFAULT_SPREADSHEET_URL,
-    sheet_name: str | None = None,
-    sheet_gid: int | None = None,
-    doc_table_index: int | None = None,
-    template_map: RubricMapping | None = None,
-    auth_method: str | None = None,
-    credentials_path: Path | None = None,
-    oauth_credentials_path: Path | None = None,
-    api_key: str | None = None,
-) -> Path:
-    """Export a Google Sheet or Doc template to an Excel workbook."""
-    template = load_template_from_google_sheet(
-        spreadsheet_url=template_url,
-        sheet_name=sheet_name,
-        sheet_gid=sheet_gid,
-        doc_table_index=doc_table_index,
-        template_map=template_map,
-        auth_method=auth_method,
-        credentials_path=credentials_path,
-        oauth_credentials_path=oauth_credentials_path,
-        api_key=api_key,
-    )
-    sheet_label = sheet_name or ("Google Doc" if _is_google_docs_url(template_url) else "Google Sheet")
-    payload = _template_to_payload(template, source=template_url, sheet_label=sheet_label)
-    ExcelConverter.from_json(payload, output_path=output_path, sheet_name="Sections")
-    return output_path
-
-
-def _template_to_payload(
-    template: AuditTemplate,
-    *,
-    source: str,
-    sheet_label: str,
-) -> dict[str, list[dict[str, Any]]]:
-    sections_payload: list[dict[str, Any]] = []
-    checks_payload: list[dict[str, Any]] = []
-
-    for section in template.sections:
-        sections_payload.append(
-            {
-                "id": section.id,
-                "name": section.name,
-                "description": section.description,
-                "weight": section.weight,
-            }
-        )
-        for check in section.checks:
-            checks_payload.append(
-                {
-                    "id": check.id,
-                    "section_id": check.section_id,
-                    "name": check.name or check.description,
-                    "description": check.description,
-                    "weight": check.weight,
-                    "check_type": check.check_type,
-                    "check_config": check.check_config,
-                    "score_mode": check.score_mode,
-                    "score_config": check.score_config,
-                    "required": check.required,
-                    "tags": check.tags,
-                    "severity": check.severity.value if check.severity else None,
-                    "priority": check.priority.value if check.priority else None,
-                    "migration_effort": check.migration_effort.value if check.migration_effort else None,
-                    "enabled": check.enabled,
-                    "remediation": check.remediation_steps,
-                    "doc_link": check.doc_link,
-                    "applicability": check.applicability,
-                }
-            )
-
-    return {
-        "Sections": sections_payload,
-        "Checks": checks_payload,
-        "Metadata": [
-            {"key": "template_version", "value": template.version or "1.0.0"},
-            {"key": "source", "value": source},
-            {"key": "sheet", "value": sheet_label},
-        ],
-    }
-
-
-def load_template_from_google_doc(
-    *,
-    doc_url: str,
-    table_index: int = 0,
-    mapping_path: Path | None = None,
-    template_map: RubricMapping | None = None,
-) -> AuditTemplate:
-    """Load a template from a Google Docs table export."""
-    mapping = template_map
-    if mapping_path:
-        mapping = load_mapping_file(mapping_path)
-    if not mapping:
-        mapping = resolve_default_mapping(sheet_name=CHECKLIST_SHEET_NAME, sheet_gid=CHECKLIST_GID)
-    if not mapping:
-        raise TemplateLoadError(
-            "Template mapping is required for Google Docs templates",
-            context={"doc_url": doc_url},
-        )
-
-    rows = _read_google_doc_rows(doc_url, table_index=table_index)
-    if not rows:
-        raise TemplateLoadError(
-            "Google Doc returned no table rows",
-            context={"doc_url": doc_url},
-        )
-
-    sections_payload, checks_payload = _rubric_rows_to_standard(rows, mapping)
-    payload = {
-        "Sections": sections_payload,
-        "Checks": checks_payload,
-        "Metadata": [
-            {"key": "template_version", "value": "1.0.0"},
-            {"key": "source", "value": doc_url},
-            {"key": "sheet", "value": "Google Doc"},
-        ],
-    }
-    template = _load_template_from_google_payload(payload)
-    if mapping.template_name:
-        template.name = mapping.template_name
-    return template
-
-
-def _load_template_from_google_payload(payload: dict[str, list[dict[str, Any]]]) -> AuditTemplate:
-    with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as handle:
-        temp_path = Path(handle.name)
-
-    try:
-        ExcelConverter.from_json(payload, output_path=temp_path, sheet_name="Sections")
-        return load_template_from_excel(temp_path)
-    finally:
-        temp_path.unlink(missing_ok=True)
-
-
-def _is_google_docs_url(url: str) -> bool:
-    return bool(re.search(r"docs\\.google\\.com/document/d/", url))
-
-
-def _extract_google_doc_id(url: str) -> str:
-    match = re.search(r"/document/d/([a-zA-Z0-9_-]+)", url)
-    if not match:
-        raise TemplateLoadError("Invalid Google Docs URL", context={"url": url})
-    return match.group(1)
-
-
-def _read_google_doc_rows(doc_url: str, table_index: int = 0) -> list[list[Any]]:
-    if table_index < 0:
-        raise TemplateLoadError(
-            "Table index must be zero or greater for Google Doc",
-            context={"table_index": table_index},
-        )
-    doc_id = _extract_google_doc_id(doc_url)
-    export_url = f"https://docs.google.com/document/d/{doc_id}/export?format=html"
-    response = httpx.get(export_url, timeout=30.0, follow_redirects=True)
-    if response.status_code != 200:
-        raise TemplateLoadError(
-            "Failed to fetch Google Doc export",
-            context={"status_code": response.status_code, "url": export_url},
-        )
-    tables = pd.read_html(response.text)
-    if not tables:
-        raise TemplateLoadError("No tables found in Google Doc", context={"url": doc_url})
-    if table_index >= len(tables):
-        raise TemplateLoadError(
-            "Table index out of range for Google Doc",
-            context={"table_index": table_index, "table_count": len(tables)},
-        )
-    table = tables[table_index]
-    headers = [str(col) for col in table.columns.tolist()]
-    rows = [headers]
-    for row in table.fillna("").astype(str).values.tolist():
-        rows.append(row)
-    return rows
-
-
-def _read_google_sheet_rows(
-    *,
-    spreadsheet_url: str,
-    sheet_name: str,
-    auth_method: str | None,
-    credentials_path: Path | None,
-    oauth_credentials_path: Path | None,
-    api_key: str | None,
-) -> list[list[Any]]:
-    project_dir = _resolve_vds_cli_project()
-    cmd = [
-        "uv",
-        "run",
-        "--project",
-        str(project_dir),
-        "vds-cli",
-        "google-sheets",
-        "--",
-        "--json-only",
-        "read",
-        "--spreadsheet-url",
-        spreadsheet_url,
-        "--sheet-name",
-        sheet_name,
-    ]
-    _add_google_auth_flags(
-        cmd,
-        auth_method=auth_method,
-        credentials_path=credentials_path,
-        oauth_credentials_path=oauth_credentials_path,
-        api_key=api_key,
-    )
-
-    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
-    if result.returncode != 0:
-        raise TemplateLoadError(
-            "Failed to read Google Sheet",
-            context={"stderr": result.stderr.strip(), "stdout": result.stdout.strip()},
-        )
-
-    try:
-        payload = json.loads(result.stdout)
-    except json.JSONDecodeError as exc:
-        raise TemplateLoadError(
-            "Google Sheets output was not valid JSON",
-            context={"stdout": result.stdout.strip(), "error": str(exc)},
-        ) from exc
-
-    rows = payload.get("rows")
-    return rows
-
-
-def _list_google_sheet_worksheets(
-    *,
-    spreadsheet_url: str,
-    auth_method: str | None,
-    credentials_path: Path | None,
-    oauth_credentials_path: Path | None,
-    api_key: str | None,
-) -> list[str]:
-    project_dir = _resolve_vds_cli_project()
-    cmd = [
-        "uv",
-        "run",
-        "--project",
-        str(project_dir),
-        "vds-cli",
-        "google-sheets",
-        "--",
-        "--json-only",
-        "list-worksheets",
-        "--spreadsheet-url",
-        spreadsheet_url,
-    ]
-    _add_google_auth_flags(
-        cmd,
-        auth_method=auth_method,
-        credentials_path=credentials_path,
-        oauth_credentials_path=oauth_credentials_path,
-        api_key=api_key,
-    )
-
-    result = subprocess.run(cmd, capture_output=True, text=True, check=False)
-    if result.returncode != 0:
-        raise TemplateLoadError(
-            "Failed to list Google Sheet worksheets",
-            context={"stderr": result.stderr.strip(), "stdout": result.stdout.strip()},
-        )
-    try:
-        payload = json.loads(result.stdout)
-    except json.JSONDecodeError as exc:
-        raise TemplateLoadError(
-            "Google Sheets list output was not valid JSON",
-            context={"stdout": result.stdout.strip(), "error": str(exc)},
-        ) from exc
-    worksheets = payload.get("worksheets")
-    if not isinstance(worksheets, list):
-        return []
-    return [str(name) for name in worksheets]
-
-
-def _add_google_auth_flags(
-    cmd: list[str],
-    *,
-    auth_method: str | None,
-    credentials_path: Path | None,
-    oauth_credentials_path: Path | None,
-    api_key: str | None,
-) -> None:
-    if auth_method:
-        cmd.extend(["--auth-method", auth_method])
-    if credentials_path:
-        cmd.extend(["--credentials-path", str(credentials_path)])
-    if oauth_credentials_path:
-        cmd.extend(["--oauth-credentials-path", str(oauth_credentials_path)])
-    if api_key:
-        cmd.extend(["--api-key", api_key])
-
-
-def _rows_to_dicts(rows: list[list[Any]] | None) -> list[dict[str, Any]]:
-    if not rows:
-        return []
-    headers = [str(cell).strip() if cell is not None else "" for cell in rows[0]]
-    data_rows = rows[1:]
-    output = []
-    for row in data_rows:
-        if not any(cell not in (None, "", " ") for cell in row):
-            continue
-        record = {headers[i]: row[i] if i < len(row) else "" for i in range(len(headers))}
-        output.append(record)
-    return output
-
-
-def _resolve_vds_cli_project() -> Path:
-    """Resolve the vds_cli project path relative to this package."""
-    current = Path(__file__).resolve()
-    for _ in range(6):
-        current = current.parent
-        candidate = current / "vds_cli"
-        if candidate.exists() and (candidate / "pyproject.toml").exists():
-            return candidate
-
-    # Fallback to sibling of current project
-    return Path(__file__).resolve().parents[4] / "vds_cli"
-
-
-def _rubric_rows_to_standard(
-    rows: list[list[Any]],
-    mapping: RubricMapping,
-    use_hierarchical: bool = True,
-) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
-    """Convert rubric rows to standard Sections/Checks format.
-
-    This function supports two modes:
-    1. use_hierarchical=True: Uses the new HierarchicalRubricParser with full
-       support for weight inheritance, group-level weight derivation (FR-002.13,
-       FR-002.16), and merged cell handling.
-    2. use_hierarchical=False: Uses the fallback algorithm with enhanced section
-       weight normalization.
-
-    Args:
-        rows: Raw rows from Google Sheet (including header row)
-        mapping: Column mapping configuration
-        use_hierarchical: Whether to use the new hierarchical parser
-
-    Returns:
-        Tuple of (sections_payload, checks_payload)
-    """
-    if not rows:
-        return [], []
-
-    # Try hierarchical parser if requested and available
-    if use_hierarchical and HIERARCHICAL_RUBRIC_AVAILABLE:
-        try:
-            config = RubricConfig(  # type: ignore[name-defined]
-                inherit_weights=True,
-                distribute_group_weights_evenly=True,
-                skip_summary_rows=True,
-                normalize_percent_weights=True,
-            )
-            return parse_rubric_with_mapping(rows, mapping, config)  # type: ignore[name-defined]
-        except Exception as e:
-            logger.warning(
-                "hierarchical_parser_failed",
-                error=str(e),
-                message="Falling back to rubric parser",
-            )
-
-    # Fallback algorithm with enhanced section weight normalization
-    return _rubric_rows_to_standard_fallback(rows, mapping)
-
-
-def _rubric_rows_to_standard_fallback(
-    rows: list[list[Any]],
-    mapping: RubricMapping,
-) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
-    """Fallback rubric to standard conversion with enhanced weight normalization.
-
-    Enhanced features:
-    - Forward-fill for blank rubric cells (merged cell handling)
-    - Group-level weight detection and distribution
-    - Section weight normalization to sum to 1.0 (FR-002.16)
-    - Summary row filtering (e.g., "TONG CONG", "Total")
-
-    """
-    if not rows:
-        return [], []
-
-    headers = [str(cell).strip() if cell is not None else "" for cell in rows[0]]
-    records = []
-    for row in rows[1:]:
-        if not any(cell not in (None, "", " ") for cell in row):
-            continue
-        record = {headers[i]: row[i] if i < len(row) else "" for i in range(len(headers))}
-        records.append(record)
-
-    summary_markers = mapping.summary_markers_normalized()
-    sections: dict[str, dict[str, Any]] = {}
-    checks: list[dict[str, Any]] = []
-    group_weights: dict[str, float] = {}
-    group_members: dict[str, list[int]] = {}
-    group_weight_candidates: dict[str, list[tuple[int, float]]] = {}
-    rule_to_section_id: dict[str, str] = {}
-
-    # Enhanced forward-fill with merged cell handling
-    last_values: dict[str, str] = {}
-    for record in records:
-        for column in mapping.forward_fill_columns:
-            if _value_str(record.get(column)):
-                last_values[column] = _value_str(record.get(column))
-            else:
-                record[column] = last_values.get(column)
-
-        rule = _value_str(record.get(mapping.rule_column))
-        group = _value_str(record.get(mapping.group_column))
-        check = _value_str(record.get(mapping.check_column))
-        weight = _parse_optional_weight(record.get(mapping.weight_column))
-        group_key = f"{rule}::{group}".strip(":")
-
-        # Detect group-level weight rows (rule+group+weight, no check text)
-        if not check:
-            if group and weight and weight > 0:
-                group_weights[group_key] = weight
-            continue
-
-        normalized_check = check.strip().lower()
-        if normalized_check in summary_markers:
-            continue
-
-        notes = _value_str(record.get(mapping.notes_column)) if mapping.notes_column else ""
-
-        if rule and rule in rule_to_section_id:
-            section_id = rule_to_section_id[rule]
-        else:
-            section_id = _section_id_for(rule, sections)
-            if rule:
-                rule_to_section_id[rule] = section_id
-        sections.setdefault(
-            section_id,
-            {
-                "id": section_id,
-                "name": rule or "Uncategorized",
-                "description": "",
-                "weight": 0.0,
-            },
-        )
-
-        check_id = _check_id_for(section_id, len(checks) + 1)
-        check_config: dict[str, Any] = {"group": group, "notes": notes} if group or notes else {}
-        checks.append(
-            {
-                "id": check_id,
-                "section_id": section_id,
-                "name": check,
-                "description": check,
-                "weight": weight,
-                "check_type": None,
-                "check_config": json.dumps(check_config) if check_config else "",
-            }
-        )
-
-        group_members.setdefault(group_key, []).append(len(checks) - 1)
-        if weight and weight > 0:
-            group_weight_candidates.setdefault(group_key, []).append((len(checks) - 1, weight))
-
-    # Enhanced group weight detection for merged cells
-    for group_key, candidates in group_weight_candidates.items():
-        indices = group_members.get(group_key, [])
-        if len(candidates) != 1 or not indices:
-            continue
-        candidate_idx, candidate_weight = candidates[0]
-        if any((checks[idx]["weight"] or 0.0) > 0 for idx in indices if idx != candidate_idx):
-            continue
-        group_weights[group_key] = candidate_weight
-        checks[candidate_idx]["weight"] = 0.0
-
-    # Distribute group weights evenly among checks
-    for group_key, indices in group_members.items():
-        weights = [checks[idx]["weight"] for idx in indices]
-        if any(weight > 0 for weight in weights):
-            continue
-        group_weight = group_weights.get(group_key)
-        if not group_weight:
-            continue
-        per_check = group_weight / len(indices)
-        for idx in indices:
-            checks[idx]["weight"] = per_check
-
-    # Derive section weights from check weights (FR-002.16)
-    section_weights: dict[str, float] = {}
-    for check in checks:
-        section_weights[check["section_id"]] = section_weights.get(check["section_id"], 0.0) + float(
-            check.get("weight") or 0.0
-        )
-
-    # Normalize section weights to sum to 100.0 (FR-002.16)
-    total_section_weight = sum(section_weights.values())
-    if total_section_weight > 0:
-        for section_id in section_weights:
-            section_weights[section_id] = (section_weights[section_id] / total_section_weight) * 100.0
-
-    for section_id, weight in section_weights.items():
-        if section_id in sections:
-            sections[section_id]["weight"] = weight
-
-    section_payload = list(sections.values())
-    return section_payload, checks
-
-
-def _normalize_section_and_check_weights(sections: list[AuditSection]) -> None:
-    for section in sections:
-        if not section.checks:
-            continue
-        if any(check.weight is None for check in section.checks):
-            # Keep explicit weights as provided; missing weights will use severity fallback.
-            continue
-        positive_weights = [check.weight for check in section.checks if (check.weight or 0.0) > 0.0]
-        if not positive_weights:
-            for check in section.checks:
-                check.weight = None
-        else:
-            check_weights = [check.weight or 0.0 for check in section.checks]
-            normalized = _normalize_weights(check_weights)
-            for check, weight in zip(section.checks, normalized, strict=False):
-                check.weight = weight
-
-    section_weights = [section.weight for section in sections]
-    normalized_sections = _normalize_weights(section_weights)
-    for section, weight in zip(sections, normalized_sections, strict=False):
-        section.weight = weight
-
-
-def _normalize_weights(values: list[float]) -> list[float]:
-    if not values:
-        return []
-    total = sum(values)
-    if total <= 0:
-        equal = 100.0 / len(values)
-        return [equal for _ in values]
-    return [(value / total) * 100.0 for value in values]
-
-
-def _parse_weight(value: Any) -> float:
-    if value is None:
-        return 0.0
-    val = 0.0
-    if isinstance(value, (int, float)):
-        val = float(value)
-    else:
-        text = str(value).strip()
-        if not text:
-            return 0.0
-        cleaned = text.replace("%", "").replace(",", ".").strip()
-        try:
-            val = float(cleaned)
-        except ValueError:
-            return 0.0
-
-    return val
-
-
-def _parse_optional_weight(value: Any) -> float | None:
-    if value is None:
-        return None
-    text = str(value).strip()
-    if not text:
-        return None
-    return _parse_weight(value)
-
-
-def _parse_int(value: Any, *, default: int = 0) -> int:
-    if value is None:
-        return default
-    try:
-        return int(float(str(value).strip()))
-    except (ValueError, TypeError):
-        return default
-
-
-def _parse_bool(value: Any) -> bool:
-    if isinstance(value, bool):
-        return value
-    if value is None:
-        return False
-    text = str(value).strip().lower()
-    return text in {"true", "1", "yes", "y", "t"}
-
-
-def _parse_enum(value: Any, enum_name: str = "severity"):
-    enum_map = {
-        "severity": (Severity, Severity.MEDIUM),
-        "priority": (Priority, Priority.P3),
-        "migration_effort": (MigrationEffort, MigrationEffort.MEDIUM),
-    }
-    enum_cls, default = enum_map.get(enum_name, (Severity, Severity.MEDIUM))
-
-    if value is None or not str(value).strip():
-        return default
-    text = str(value).strip()
-    try:
-        return enum_cls(text)
-    except ValueError:
-        lowered = text.lower()
-        for member in enum_cls:
-            if member.value.lower() == lowered or member.name.lower() == lowered:
-                return member
-    return default
-
-
-def _value_str(value: Any) -> str:
-    if value is None:
-        return ""
-    return str(value).strip()
-
-
-def _value_dict(value: Any) -> dict[str, Any]:
-    if value is None:
-        return {}
-    if isinstance(value, dict):
-        return value
-    try:
-        return json.loads(str(value))
-    except Exception:
-        return {}
-
-
-def _value_list(value: Any) -> list[str]:
-    if value is None:
-        return []
-    if isinstance(value, list):
-        return [str(item).strip() for item in value if str(item).strip()]
-    text = str(value).strip()
-    if not text:
-        return []
-    return [part.strip() for part in text.split(",") if part.strip()]
-
-
-def _slugify(text: str) -> str:
-    normalized = unicodedata.normalize("NFKD", text)
-    ascii_text = normalized.encode("ascii", "ignore").decode()
-    cleaned = "".join(ch.lower() if ch.isalnum() else "-" for ch in ascii_text)
-    cleaned = "-".join(part for part in cleaned.split("-") if part)
-    return cleaned
-
-
-def _section_id_for(rule: str, existing: dict[str, dict[str, Any]]) -> str:
-    if rule:
-        slug = _slugify(rule)
-        candidate = f"section-{slug}" if slug else "section"
-    else:
-        candidate = "section"
-
-    if candidate not in existing:
-        return candidate
-
-    index = 2
-    while f"{candidate}-{index}" in existing:
-        index += 1
-    return f"{candidate}-{index}"
-
-
-def _check_id_for(section_id: str, index: int) -> str:
-    return f"{section_id}-check-{index:03d}"