@ngocsangairvds/vsaf 4.0.6 → 4.0.7

package/skills/vds-skill/vds-scripts/excel_orchestrator/src/vds_excel_orchestrator/validators.py

@@ -0,0 +1,164 @@
+"""Validation utilities for Excel files and data."""
+
+from __future__ import annotations
+
+import zipfile
+from pathlib import Path
+from typing import Any
+
+from openpyxl import load_workbook
+
+from .errors import ExcelFileInvalidError, ExcelFileNotFoundError, ExcelValidationError
+from .logging import get_logger
+
+logger = get_logger(__name__)
+
+
+class ExcelValidator:
+    """Utility class for validating Excel files and data."""
+
+    @staticmethod
+    def validate_file(file_path: str | Path, max_size_mb: int = 100) -> dict[str, Any]:
+        """Validate Excel file structure and format.
+
+        Args:
+            file_path: Path to Excel file
+            max_size_mb: Maximum file size in MB
+
+        Returns:
+            Dictionary with validation results
+
+        Raises:
+            ExcelFileNotFoundError: If file doesn't exist
+            ExcelFileInvalidError: If file is invalid
+        """
+        file_path = Path(file_path)
+
+        if not file_path.exists():
+            raise ExcelFileNotFoundError(f"File not found: {file_path}", context={"path": str(file_path)})
+
+        # Check file size
+        file_size_mb = file_path.stat().st_size / (1024 * 1024)
+        if file_size_mb > max_size_mb:
+            raise ExcelFileInvalidError(
+                f"File size ({file_size_mb:.2f} MB) exceeds maximum ({max_size_mb} MB)",
+                context={
+                    "path": str(file_path),
+                    "size_mb": file_size_mb,
+                    "max_mb": max_size_mb,
+                },
+            )
+
+        # Check if it's a valid ZIP file (xlsx is a ZIP archive)
+        try:
+            with zipfile.ZipFile(file_path, "r") as zip_file:
+                zip_file.testzip()
+        except zipfile.BadZipFile as exc:
+            raise ExcelFileInvalidError(
+                f"File is not a valid ZIP archive: {exc}",
+                context={"path": str(file_path)},
+            ) from exc
+
+        # Try to open with openpyxl
+        try:
+            workbook = load_workbook(filename=str(file_path), read_only=True, data_only=True)
+            sheet_count = len(workbook.sheetnames)
+            workbook.close()
+
+            result = {
+                "valid": True,
+                "path": str(file_path),
+                "size_mb": round(file_size_mb, 2),
+                "sheet_count": sheet_count,
+                "sheet_names": workbook.sheetnames,
+            }
+
+            logger.info("File validated", **result)
+            return result
+
+        except Exception as exc:
+            raise ExcelFileInvalidError(f"Failed to open Excel file: {exc}", context={"path": str(file_path)}) from exc
+
+    @staticmethod
+    def validate_sheet_name(sheet_name: str) -> bool:
+        """Validate sheet name.
+
+        Args:
+            sheet_name: Sheet name to validate
+
+        Returns:
+            True if valid
+
+        Raises:
+            ExcelValidationError: If sheet name is invalid
+        """
+        if not sheet_name:
+            raise ExcelValidationError("Sheet name cannot be empty")
+
+        if len(sheet_name) > 31:
+            raise ExcelValidationError(f"Sheet name too long (max 31 characters): {sheet_name}")
+
+        invalid_chars = [":", "\\", "/", "?", "*", "[", "]"]
+        for char in invalid_chars:
+            if char in sheet_name:
+                raise ExcelValidationError(f"Sheet name contains invalid character '{char}': {sheet_name}")
+
+        return True
+
+    @staticmethod
+    def validate_cell_ref(cell_ref: str) -> bool:
+        """Validate cell reference.
+
+        Args:
+            cell_ref: Cell reference (e.g., 'A1')
+
+        Returns:
+            True if valid
+
+        Raises:
+            ExcelValidationError: If cell reference is invalid
+        """
+        if not cell_ref:
+            raise ExcelValidationError("Cell reference cannot be empty")
+
+        # Basic validation - should match pattern like A1, AA100, etc.
+        import re
+
+        pattern = r"^[A-Z]+[0-9]+$"
+        if not re.match(pattern, cell_ref.upper()):
+            raise ExcelValidationError(f"Invalid cell reference format: {cell_ref}")
+
+        return True
+
+    @staticmethod
+    def validate_range_ref(range_ref: str) -> bool:
+        """Validate range reference.
+
+        Args:
+            range_ref: Range reference (e.g., 'A1:C10')
+
+        Returns:
+            True if valid
+
+        Raises:
+            ExcelValidationError: If range reference is invalid
+        """
+        if not range_ref:
+            raise ExcelValidationError("Range reference cannot be empty")
+
+        # Should contain colon separator
+        if ":" not in range_ref:
+            raise ExcelValidationError(f"Range reference must contain ':' separator: {range_ref}")
+
+        parts = range_ref.split(":")
+        if len(parts) != 2:
+            raise ExcelValidationError(f"Range reference must have exactly two parts: {range_ref}")
+
+        # Validate both cell references
+        ExcelValidator.validate_cell_ref(parts[0])
+        ExcelValidator.validate_cell_ref(parts[1])
+
+        return True
+
+
+__all__ = ["ExcelValidator"]

package/skills/vds-skill/vds-scripts/excel_orchestrator/sync_detail_and_total_sheets.py

@@ -0,0 +1,211 @@
+"""
+Sync VDS_Est_Detail_Target_State and VDS_Est_Detail_Services_Total sheets.
+- Calculate correct totals from Detail sheet
+- Update Services Total sheet with correct data
+- Fix remarks: only service rows should have remarks, task rows should be empty
+- Ensure all numbers match between sheets
+"""
+
+from openpyxl import load_workbook
+from openpyxl.styles import Alignment, Font, PatternFill
+
+FILE_PATH = "../../insurance-document/projects/INSURANCE/analysis/reference/vietin/estimation/Estimation_HTBH.xlsx"
+DETAIL_SHEET = "VDS_Est_Detail_Target_State"
+TOTAL_SHEET = "VDS_Est_Detail_Services_Total"
+
+
+def get_service_totals(ws_detail):
+    """Extract service totals from detail sheet."""
+    services = {}
+
+    for row in range(10, ws_detail.max_row + 1):
+        col2 = ws_detail.cell(row, 2).value
+        col3 = ws_detail.cell(row, 3).value
+        col6 = ws_detail.cell(row, 6).value
+
+        # Service header row
+        if isinstance(col2, (int, float)) and col2 > 0 and col3:
+            service_num = int(col2)
+            service_name = str(col3).strip()
+            remark = str(col6).strip() if col6 else ""
+
+            # Calculate totals from tasks below
+            total_days = 0.0
+            total_hours = 0.0
+
+            # Look ahead for tasks
+            for task_row in range(row + 1, ws_detail.max_row + 1):
+                task_col2 = ws_detail.cell(task_row, 2).value
+                task_col3 = ws_detail.cell(task_row, 3).value
+                task_col5 = ws_detail.cell(task_row, 5).value
+
+                # Stop at next service
+                if isinstance(task_col2, (int, float)) and task_col2 > 0:
+                    break
+
+                # Task row
+                if task_col3 and isinstance(task_col5, (int, float)):
+                    total_days += float(task_col5)
+                    total_hours += float(task_col5) * 8
+
+            services[service_num] = {
+                "name": service_name,
+                "days": total_days,
+                "hours": total_hours,
+                "remark": remark,
+            }
+
+    return services
+
+
+def clean_remarks(ws_detail):
+    """Clean remarks: only service rows should have remarks."""
+    cleaned = 0
+
+    for row in range(10, ws_detail.max_row + 1):
+        col2 = ws_detail.cell(row, 2).value
+        col3 = ws_detail.cell(row, 3).value
+        col6 = ws_detail.cell(row, 6).value
+
+        # Service row - keep remark if meaningful
+        if isinstance(col2, (int, float)) and col2 > 0 and col3:
+            service_name = str(col3).strip()
+            remark = str(col6).strip() if col6 else ""
+
+            # If remark is too short or looks like it belongs to a task, clear it
+            if remark and (len(remark) < 10 or remark.startswith("Mục tiêu:")):
+                ws_detail.cell(row, 6).value = None
+                cleaned += 1
+            # If no remark, add a default one based on service name
+            elif not remark:
+                # Extract main service name (before colon or parentheses)
+                main_name = service_name.split(":")[0].split("(")[0].strip()
+                ws_detail.cell(row, 6).value = f"Service for {main_name}"
+                cleaned += 1
+
+        # Task row - clear remark
+        elif col3 and not isinstance(col2, (int, float)):
+            if col6:
+                ws_detail.cell(row, 6).value = None
+                cleaned += 1
+
+    return cleaned
+
+
+def update_services_total_sheet(wb, services):
+    """Update or create Services Total sheet with correct data."""
+    # Remove old sheet if exists
+    if TOTAL_SHEET in wb.sheetnames:
+        wb.remove(wb[TOTAL_SHEET])
+
+    ws_total = wb.create_sheet(TOTAL_SHEET)
+
+    # Copy header style from Detail sheet
+    ws_detail = wb[DETAIL_SHEET]
+    header_row = 8
+
+    # Headers
+    headers = ["#", "Task Name", "Coding Effort (hours)", "in days", "Remarks"]
+    for col_idx, header in enumerate(headers, start=1):
+        cell = ws_total.cell(1, col_idx)
+        cell.value = header
+
+        # Copy style from detail sheet header
+        src_cell = ws_detail.cell(header_row, col_idx + 1)  # Detail starts at col 2
+        if src_cell.font:
+            cell.font = Font(
+                bold=src_cell.font.bold,
+                size=src_cell.font.size,
+                color=src_cell.font.color,
+            )
+        if src_cell.fill:
+            cell.fill = PatternFill(patternType=src_cell.fill.patternType, fgColor=src_cell.fill.fgColor)
+        cell.alignment = Alignment(horizontal="center", vertical="center")
+
+    # Set column widths
+    ws_total.column_dimensions["A"].width = 8
+    ws_total.column_dimensions["B"].width = 50
+    ws_total.column_dimensions["C"].width = 25
+    ws_total.column_dimensions["D"].width = 12
+    ws_total.column_dimensions["E"].width = 40
+
+    # Add service rows
+    row = 2
+    for service_num in sorted(services.keys()):
+        service = services[service_num]
+
+        ws_total.cell(row, 1).value = service_num
+        ws_total.cell(row, 2).value = service["name"]
+        ws_total.cell(row, 3).value = service["hours"]
+        ws_total.cell(row, 4).value = service["days"]
+        ws_total.cell(row, 5).value = service["remark"]
+
+        # Format numbers
+        ws_total.cell(row, 3).number_format = "0.0"
+        ws_total.cell(row, 4).number_format = "0.0"
+
+        # Copy font style from detail sheet service rows
+        # Find corresponding row in detail sheet
+        for detail_row in range(10, ws_detail.max_row + 1):
+            detail_col2 = ws_detail.cell(detail_row, 2).value
+            if isinstance(detail_col2, (int, float)) and int(detail_col2) == service_num:
+                src_cell = ws_detail.cell(detail_row, 3)
+                if src_cell.font:
+                    ws_total.cell(row, 2).font = Font(
+                        bold=src_cell.font.bold,
+                        size=src_cell.font.size,
+                        color=src_cell.font.color,
+                    )
+                break
+
+        row += 1
+
+    # Add totals row
+    row += 1
+    ws_total.cell(row, 2).value = "Total"
+    ws_total.cell(row, 2).font = Font(bold=True)
+    ws_total.cell(row, 3).value = f"=SUM(C2:C{row - 1})"
+    ws_total.cell(row, 4).value = f"=SUM(D2:D{row - 1})"
+    ws_total.cell(row, 3).number_format = "0.0"
+    ws_total.cell(row, 4).number_format = "0.0"
+
+    return row - 1  # Return number of services
+
+
+def main():
+    wb = load_workbook(FILE_PATH, read_only=False)
+    ws_detail = wb[DETAIL_SHEET]
+
+    print("=== Step 1: Cleaning remarks in Detail sheet ===")
+    cleaned = clean_remarks(ws_detail)
+    print(f"✅ Cleaned {cleaned} remark cells")
+
+    print("\n=== Step 2: Calculating service totals from Detail sheet ===")
+    services = get_service_totals(ws_detail)
+    print(f"✅ Found {len(services)} services")
+
+    print("\n=== Step 3: Updating Services Total sheet ===")
+    service_count = update_services_total_sheet(wb, services)
+    print(f"✅ Updated Services Total sheet with {service_count} services")
+
+    # Verify totals match
+    print("\n=== Step 4: Verifying totals ===")
+    total_hours = sum(s["hours"] for s in services.values())
+    total_days = sum(s["days"] for s in services.values())
+    print(f"Detail sheet totals: {total_hours:.1f} hours, {total_days:.1f} days")
+
+    # Find totals row in detail sheet
+    for row in range(500, ws_detail.max_row + 1):
+        col3 = ws_detail.cell(row, 3).value
+        if isinstance(col3, str) and "total" in col3.lower():
+            detail_total_hours = ws_detail.cell(row, 4).value
+            detail_total_days = ws_detail.cell(row, 5).value
+            print(f"Detail sheet formula totals: {detail_total_hours} hours, {detail_total_days} days")
+            break
+
+    wb.save(FILE_PATH)
+    print("\n✅ All sheets synced and saved!")
+
+
+if __name__ == "__main__":
+    main()

package/skills/vds-skill/vds-scripts/excel_orchestrator/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for Excel orchestrator."""

package/skills/vds-skill/vds-scripts/excel_orchestrator/tests/conftest.py

@@ -0,0 +1,36 @@
+"""Pytest configuration and fixtures."""
+
+from __future__ import annotations
+
+import tempfile
+from pathlib import Path
+
+import pytest
+
+from vds_excel_orchestrator.excel_client import ExcelClient
+
+
+@pytest.fixture
+def temp_excel_file() -> Path:
+    """Create a temporary Excel file for testing."""
+    with tempfile.NamedTemporaryFile(suffix=".xlsx", delete=False) as f:
+        temp_path = Path(f.name)
+
+    # Create a simple workbook
+    with ExcelClient(temp_path) as client:
+        client.create()
+        client.set_cell("Sheet", "A1", "Test")
+        client.save()
+
+    yield temp_path
+
+    # Cleanup
+    if temp_path.exists():
+        temp_path.unlink()
+
+
+@pytest.fixture
+def temp_dir() -> Path:
+    """Create a temporary directory for testing."""
+    with tempfile.TemporaryDirectory() as tmpdir:
+        yield Path(tmpdir)