@ngocsangairvds/vsaf 4.0.6 → 4.0.7

package/skills/vds-skill/vds-scripts/excel_orchestrator/organize_and_deduplicate.py

@@ -0,0 +1,273 @@
+"""Organize content by category and subcategory, remove duplicates."""
+
+import re
+from collections import defaultdict
+
+from openpyxl import load_workbook
+
+# Service name mapping for generic names
+SERVICE_NAME_MAP = {
+    "Giao diện Khách hàng": "Customer Interface",
+    "Cổng thông tin Đại lý": "Agent Portal",
+    "Cổng quản trị nội bộ": "Internal Admin Portal",
+    "BFF Khách hàng": "Customer BFF",
+    "BFF Đại lý": "Agent BFF",
+    "BFF Quản trị": "Admin BFF",
+}
+
+# Category grouping
+CATEGORIES = {
+    "Core Services": [
+        "proposal",
+        "policy",
+        "customer",
+        "claim",
+        "agent",
+        "billing",
+        "promotion",
+    ],
+    "Engines": ["rating", "underwriting", "rules", "catalog", "analytics", "fraud"],
+    "BFF Services": ["bff", "customer bff", "agent bff", "admin bff"],
+    "Process Services": ["orchestrator", "workflow", "renewal", "timeout", "reconcile"],
+    "Integration Services": ["carrier", "payment", "hub", "gateway"],
+    "Infrastructure": [
+        "ci/cd",
+        "monitoring",
+        "observability",
+        "security",
+        "migration",
+        "test",
+    ],
+    "Other Services": [
+        "ocr",
+        "notification",
+        "document",
+        "socialized",
+        "content",
+        "schedule",
+        "crm",
+    ],
+}
+
+
+def extract_service_name(task_name):
+    """Extract service name from task name."""
+    if not task_name:
+        return None
+
+    task_str = str(task_name).strip()
+
+    # Check for workstream in parentheses
+    if "(" in task_str and ")" in task_str:
+        match = re.search(r"\(([^)]+)\)", task_str)
+        if match:
+            workstream = match.group(1)
+            # Try to map workstream to service name
+            if workstream in SERVICE_NAME_MAP:
+                return SERVICE_NAME_MAP[workstream]
+            # Extract from workstream
+            if "BFF" in workstream:
+                return workstream
+            if "Cổng" in workstream or "Giao diện" in workstream:
+                return workstream
+
+    # Try to extract insurance-xxx pattern
+    if "insurance-" in task_str:
+        match = re.search(r"insurance-([a-z-]+)", task_str)
+        if match:
+            return match.group(1).replace("-", " ").title()
+
+    # Extract from beginning
+    if task_str.startswith("  "):
+        clean = task_str.replace("  ", "").strip()
+        if "(" in clean:
+            clean = clean.split("(")[0].strip()
+        return clean
+
+    return None
+
+
+def get_category(service_name):
+    """Get category for a service."""
+    if not service_name:
+        return "Other Services"
+
+    service_lower = service_name.lower()
+    for category, keywords in CATEGORIES.items():
+        for keyword in keywords:
+            if keyword in service_lower:
+                return category
+    return "Other Services"
+
+
+if __name__ == "__main__":
+    wb = load_workbook(
+        "../../docs-nttc/projects/INSURANCE/analysis/reference/vietin/estimation/Estimation_HTBH.xlsx",
+        read_only=False,
+    )
+    detail = wb["VDS_Est_Detail_Target_State"]
+
+    print("=== Organizing and Deduplicating ===")
+
+    # Read all services with their tasks
+    services_data = []
+    current_service = None
+    current_tasks = []
+
+    for row in range(10, 527):
+        col2 = detail.cell(row, 2).value
+        col3 = detail.cell(row, 3).value
+        col4 = detail.cell(row, 4).value
+        col5 = detail.cell(row, 5).value
+
+        if col3:
+            task_str = str(col3).strip()
+            # Service header
+            if isinstance(col2, (int, float)):
+                # Save previous service
+                if current_service:
+                    services_data.append(
+                        {
+                            "row": current_service["row"],
+                            "section": current_service["section"],
+                            "name": current_service["name"],
+                            "days": current_service["days"],
+                            "hours_formula": current_service["hours_formula"],
+                            "tasks": current_tasks.copy(),
+                        }
+                    )
+                # New service
+                service_name = extract_service_name(task_str)
+                category = get_category(service_name)
+                current_service = {
+                    "row": row,
+                    "section": int(col2),
+                    "name": task_str,
+                    "service_name": service_name,
+                    "category": category,
+                    "days": col5,
+                    "hours_formula": col4,
+                }
+                current_tasks = []
+            else:
+                # Task
+                if current_service:
+                    current_tasks.append(
+                        {
+                            "row": row,
+                            "name": task_str,
+                            "days": col5,
+                            "hours_formula": col4,
+                        }
+                    )
+
+    # Save last service
+    if current_service:
+        services_data.append(
+            {
+                "row": current_service["row"],
+                "section": current_service["section"],
+                "name": current_service["name"],
+                "days": current_service["days"],
+                "hours_formula": current_service["hours_formula"],
+                "tasks": current_tasks.copy(),
+            }
+        )
+
+    print(f"Found {len(services_data)} services")
+
+    # Check for duplicates
+    seen_services = {}
+    duplicates = []
+    unique_services = []
+
+    for svc in services_data:
+        # Create unique key from service name
+        clean_name = svc["name"].split("(")[0].strip() if "(" in svc["name"] else svc["name"].strip()
+        key = clean_name.lower()
+
+        if key in seen_services:
+            duplicates.append((seen_services[key], svc))
+            print(f"  Duplicate: {svc['name'][:50]} (row {svc['row']})")
+        else:
+            seen_services[key] = svc
+            unique_services.append(svc)
+
+    print(f"\\nFound {len(duplicates)} duplicates, {len(unique_services)} unique services")
+
+    # Organize by category
+    print("\\n=== Organizing by Category ===")
+    by_category = defaultdict(list)
+    for svc in unique_services:
+        category = get_category(svc.get("service_name", ""))
+        by_category[category].append(svc)
+
+    print(f"Categories: {list(by_category.keys())}")
+    for category, svcs in by_category.items():
+        print(f"  {category}: {len(svcs)} services")
+
+    # Rebuild sheet organized by category
+    print("\\n=== Rebuilding Sheet Organized by Category ===")
+
+    # Clear data rows
+    for row in range(10, 527):
+        for col in range(1, 6):
+            detail.cell(row, col).value = None
+
+    # Write organized data
+    current_row = 10
+    section_num = 1
+
+    # Sort categories
+    category_order = [
+        "Core Services",
+        "Engines",
+        "BFF Services",
+        "Process Services",
+        "Integration Services",
+        "Infrastructure",
+        "Other Services",
+    ]
+
+    for category in category_order:
+        if category not in by_category:
+            continue
+
+        services = by_category[category]
+        # Sort services within category
+        services.sort(key=lambda x: x["name"])
+
+        for svc in services:
+            # Write service header
+            detail.cell(current_row, 2).value = section_num
+            detail.cell(current_row, 3).value = svc["name"]
+            detail.cell(current_row, 4).value = svc["hours_formula"]
+            detail.cell(current_row, 5).value = svc["days"]
+            current_row += 1
+            section_num += 1
+
+            # Write tasks
+            for task in svc["tasks"]:
+                detail.cell(current_row, 2).value = None
+                detail.cell(current_row, 3).value = task["name"]
+                detail.cell(current_row, 4).value = task["hours_formula"]
+                detail.cell(current_row, 5).value = task["days"]
+                current_row += 1
+
+    print(f"\\nRebuilt sheet: {section_num - 1} services, last row: {current_row - 1}")
+
+    # Update formulas
+    last_data_row = current_row - 1
+    detail.cell(9, 4).value = f"=SUM(D10:D{last_data_row})"
+    detail.cell(527, 4).value = f"=SUM(D10:D{last_data_row})"
+    detail.cell(527, 5).value = "=D527/8"
+    detail.cell(5, 4).value = "=D527"
+    detail.cell(5, 5).value = "=E527"
+
+    # Update summary
+    summary = wb["VDS_Summary_Target_State"]
+    summary.cell(10, 4).value = "='VDS_Est_Detail_Target_State'!D527"
+    summary.cell(10, 5).value = "='VDS_Est_Detail_Target_State'!D527"
+
+    wb.save("../../docs-nttc/projects/INSURANCE/analysis/reference/vietin/estimation/Estimation_HTBH.xlsx")
+    print("\\n✅ Organized by category and removed duplicates!")

package/skills/vds-skill/vds-scripts/excel_orchestrator/pyproject.toml

@@ -0,0 +1,64 @@
+[project]
+name = "vds-excel-orchestrator"
+version = "0.1.0"
+description = "Excel (.xlsx) file manipulation orchestrator for VDS platform"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+    "openpyxl>=3.1.5",           # Primary Excel manipulation (validated with pypi.org)
+    "xlsxwriter>=3.2.9",         # Optional: Advanced writing (validated with pypi.org)
+    "typer>=0.24.1",             # CLI framework (validated with PyPI as of 2026-02-24)
+    "pydantic>=2.12.5",          # Data validation (validated with pypi.org)
+    "pydantic-settings>=2.12.0", # Configuration
+    "structlog>=25.5.0",         # Logging (validated with pypi.org)
+    "rich>=14.3.3",              # Console output (validated with pypi.org)
+    "vds-cli-common",            # Shared environment loader and CLI primitives
+]
+
+[project.scripts]
+vds-excel = "vds_excel_orchestrator.cli:app"
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=9.0.2",
+    "pytest-cov>=7.0.0",
+    "ruff>=0.14.11",
+]
+
+[build-system]
+requires = ["hatchling>=1.20.0"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/vds_excel_orchestrator"]
+
+[tool.hatch.build.targets.sdist]
+include = ["src/vds_excel_orchestrator", "pyproject.toml", "README.md"]
+
+[tool.uv.sources]
+vds-cli-common = { workspace = true }
+
+[dependency-groups]
+dev = [
+    "pytest>=9.0.2",
+    "pytest-cov>=7.0.0",
+    "ruff>=0.14.11",
+]
+
+[tool.ruff]
+line-length = 120
+target-version = "py311"
+exclude = [
+    "tests",
+    ".venv",
+    "__pycache__",
+]
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "W", "B", "C4", "UP"]
+ignore = ["E501", "B008"]
+
+[tool.ruff.format]
+quote-style = "double"
+indent-style = "space"
+line-ending = "auto"

package/skills/vds-skill/vds-scripts/excel_orchestrator/rebuild_all_formulas.py

@@ -0,0 +1,146 @@
+"""Completely rebuild all formulas in columns D and E for absolute consistency."""
+
+from openpyxl import load_workbook
+
+if __name__ == "__main__":
+    wb = load_workbook(
+        "../../docs-nttc/projects/INSURANCE/analysis/reference/vietin/estimation/Estimation_HTBH.xlsx",
+        read_only=False,
+    )
+    detail = wb["VDS_Est_Detail_Target_State"]
+
+    print("=== Rebuilding All Formulas ===")
+
+    # Step 1: Clear all D and E columns for data rows
+    print("Step 1: Clearing all D and E values...")
+    for row in range(10, 450):
+        detail.cell(row, 4).value = None  # D
+        detail.cell(row, 5).value = None  # E
+
+    # Step 2: Rebuild based on row type
+    print("Step 2: Rebuilding formulas...")
+    tasks_rebuilt = 0
+
+    for row in range(10, 450):
+        col2 = detail.cell(row, 2).value
+        col3 = detail.cell(row, 3).value
+
+        if not col3:
+            continue
+
+        is_service_header = isinstance(col2, (int, float))
+
+        if is_service_header:
+            # Service header - keep D and E empty
+            detail.cell(row, 4).value = None
+            detail.cell(row, 5).value = None
+        else:
+            # Task row - we need to restore the days value
+            # Read from a backup or recalculate
+            # For now, let's check if we can get it from the task description
+            # Actually, we should preserve the E values, so let's read them first
+
+            # We'll need to preserve E values, so let's do this differently
+            pass
+
+    # Better approach: Read E values first, then rebuild D formulas
+    print("Step 2 (revised): Preserving E values and rebuilding D formulas...")
+
+    # First, read all E values
+    e_values = {}
+    for row in range(10, 450):
+        col2 = detail.cell(row, 2).value
+        col3 = detail.cell(row, 3).value
+        col5 = detail.cell(row, 5).value
+
+        if col3 and not isinstance(col2, (int, float)) and col5:
+            e_values[row] = float(col5) if isinstance(col5, (int, float)) else None
+
+    # Now clear and rebuild
+    for row in range(10, 450):
+        col2 = detail.cell(row, 2).value
+        col3 = detail.cell(row, 3).value
+
+        if not col3:
+            continue
+
+        is_service_header = isinstance(col2, (int, float))
+
+        if is_service_header:
+            # Service header - empty
+            detail.cell(row, 4).value = None
+            detail.cell(row, 5).value = None
+        else:
+            # Task - restore E and create D formula
+            if row in e_values:
+                days = e_values[row]
+                detail.cell(row, 5).value = days
+                detail.cell(row, 4).value = f"=E{row}*8"
+                tasks_rebuilt += 1
+            else:
+                # No E value - clear D
+                detail.cell(row, 4).value = None
+                detail.cell(row, 5).value = None
+
+    print(f"Rebuilt {tasks_rebuilt} task formulas")
+
+    # Step 3: Fix total rows
+    print("Step 3: Fixing total rows...")
+    last_data_row = 449
+
+    # Category total
+    detail.cell(9, 4).value = f"=SUM(D10:D{last_data_row})"
+    detail.cell(9, 5).value = "=D9/8"
+
+    # Grand total
+    detail.cell(527, 4).value = f"=SUM(D10:D{last_data_row})"
+    detail.cell(527, 5).value = "=D527/8"
+
+    # Row 5
+    detail.cell(5, 4).value = "=D527"
+    detail.cell(5, 5).value = "=E527"
+
+    # Update summary
+    summary = wb["VDS_Summary_Target_State"]
+    summary.cell(10, 4).value = "='VDS_Est_Detail_Target_State'!D527"
+    summary.cell(10, 5).value = "='VDS_Est_Detail_Target_State'!D527"
+
+    wb.save("../../docs-nttc/projects/INSURANCE/analysis/reference/vietin/estimation/Estimation_HTBH.xlsx")
+    print("\\n✅ All formulas rebuilt!")
+
+    # Verification
+    print("\\n=== Verification ===")
+    service_headers_ok = 0
+    tasks_ok = 0
+    issues = []
+
+    for row in range(10, 50):  # Sample check
+        col2 = detail.cell(row, 2).value
+        col3 = detail.cell(row, 3).value
+        col4 = detail.cell(row, 4).value
+        col5 = detail.cell(row, 5).value
+
+        if not col3:
+            continue
+
+        if isinstance(col2, (int, float)):
+            if not col4 and not col5:
+                service_headers_ok += 1
+            else:
+                issues.append(f"Row {row}: Service header has D={col4}, E={col5}")
+        else:
+            if col5:
+                expected = f"=E{row}*8"
+                if col4 == expected:
+                    tasks_ok += 1
+                else:
+                    issues.append(f"Row {row}: D={col4}, expected {expected}, E={col5}")
+
+    print(f"Service headers OK: {service_headers_ok}")
+    print(f"Tasks OK: {tasks_ok}")
+    if issues:
+        print(f"\\nIssues: {len(issues)}")
+        for issue in issues[:5]:
+            print(f"  {issue}")
+    else:
+        print("\\n✅ All verified!")