@ngocsangairvds/vsaf 3.1.27 → 3.2.1

package/tools/vds-scripts/scripts/validate_folder_structure.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+"""
+Folder Structure Consistency Validation Script
+Validates that all services follow identical folder and file structure.
+"""
+
+import os
+from pathlib import Path
+from typing import Dict, List, Set
+import json
+
+# Expected project structure
+EXPECTED_PROJECTS = ["INSURANCE", "LEP", "EKYC", "SAVING", "PLATFORM", "SECURITY"]
+
+# Expected folder structure for each project
+EXPECTED_FOLDERS = [
+    "analysis",
+    "business", 
+    "integrations",
+    "operations",
+    "overview",
+    "security"
+]
+
+# Expected files in each project root
+EXPECTED_FILES = [
+    "README.md"
+]
+
+def analyze_project_structure(project_path: Path) -> Dict:
+    """Analyze the folder structure of a project."""
+    
+    if not project_path.exists():
+        return {"error": f"Project path does not exist: {project_path}"}
+    
+    structure = {
+        "folders": [],
+        "files": [],
+        "subfolders": {}
+    }
+    
+    try:
+        # Get immediate folders and files
+        for item in project_path.iterdir():
+            if item.is_dir():
+                structure["folders"].append(item.name)
+                # Analyze subfolders
+                subfolders = []
+                try:
+                    for subitem in item.iterdir():
+                        if subitem.is_dir():
+                            subfolders.append(subitem.name)
+                except PermissionError:
+                    pass
+                structure["subfolders"][item.name] = sorted(subfolders)
+            elif item.is_file():
+                structure["files"].append(item.name)
+                
+        structure["folders"] = sorted(structure["folders"])
+        structure["files"] = sorted(structure["files"])
+        
+    except Exception as e:
+        structure["error"] = str(e)
+    
+    return structure
+
+def validate_folder_consistency() -> Dict:
+    """Validate folder structure consistency across all projects."""
+    
+    # Get the documentation root
+    script_dir = Path(__file__).parent
+    docs_root = script_dir.parent / "insurance-document"
+    
+    if not docs_root.exists():
+        docs_root = script_dir.parent.parent / "insurance-document"
+    
+    if not docs_root.exists():
+        return {"error": "Documentation root not found"}
+    
+    projects_root = docs_root / "projects"
+    if not projects_root.exists():
+        return {"error": "Projects directory not found"}
+    
+    print(f"🔍 Analyzing project structures at: {projects_root}")
+    
+    # Analyze each project
+    project_structures = {}
+    for project_name in EXPECTED_PROJECTS:
+        project_path = projects_root / project_name
+        if project_path.exists():
+            structure = analyze_project_structure(project_path)
+            project_structures[project_name] = structure
+        else:
+            project_structures[project_name] = {"error": "Project directory not found"}
+    
+    # Find common structure patterns
+    all_folders = set()
+    all_files = set()
+    
+    for project, structure in project_structures.items():
+        if "error" not in structure:
+            all_folders.update(structure.get("folders", []))
+            all_files.update(structure.get("files", []))
+    
+    # Check consistency
+    consistency_report = {
+        "total_projects": len(EXPECTED_PROJECTS),
+        "analyzed_projects": len([p for p in project_structures.values() if "error" not in p]),
+        "common_folders": sorted(list(all_folders)),
+        "common_files": sorted(list(all_files)),
+        "project_structures": project_structures,
+        "consistency_issues": [],
+        "missing_projects": []
+    }
+    
+    # Check for missing projects
+    for project_name in EXPECTED_PROJECTS:
+        if "error" in project_structures.get(project_name, {}):
+            consistency_report["missing_projects"].append(project_name)
+    
+    # Check folder consistency
+    reference_structure = None
+    reference_project = None
+    
+    for project, structure in project_structures.items():
+        if "error" not in structure:
+            if reference_structure is None:
+                reference_structure = structure
+                reference_project = project
+            else:
+                # Compare with reference
+                if set(structure["folders"]) != set(reference_structure["folders"]):
+                    consistency_report["consistency_issues"].append({
+                        "type": "folder_mismatch",
+                        "project": project,
+                        "reference": reference_project,
+                        "missing_folders": list(set(reference_structure["folders"]) - set(structure["folders"])),
+                        "extra_folders": list(set(structure["folders"]) - set(reference_structure["folders"]))
+                    })
+                
+                if set(structure["files"]) != set(reference_structure["files"]):
+                    consistency_report["consistency_issues"].append({
+                        "type": "file_mismatch", 
+                        "project": project,
+                        "reference": reference_project,
+                        "missing_files": list(set(reference_structure["files"]) - set(structure["files"])),
+                        "extra_files": list(set(structure["files"]) - set(reference_structure["files"]))
+                    })
+    
+    # Calculate consistency score
+    total_checks = len(EXPECTED_PROJECTS) * 2  # folders + files
+    failed_checks = len(consistency_report["consistency_issues"]) + len(consistency_report["missing_projects"])
+    consistency_score = max(0, (total_checks - failed_checks) / total_checks * 100)
+    
+    consistency_report["consistency_score"] = consistency_score
+    
+    return consistency_report
+
+def print_consistency_report(report: Dict):
+    """Print a formatted consistency report."""
+    
+    if "error" in report:
+        print(f"❌ {report['error']}")
+        return
+    
+    print("\n" + "="*60)
+    print("📁 FOLDER STRUCTURE CONSISTENCY REPORT")
+    print("="*60)
+    
+    print(f"\n📊 Summary:")
+    print(f"   Total Projects: {report['total_projects']}")
+    print(f"   Analyzed Projects: {report['analyzed_projects']}")
+    print(f"   Consistency Score: {report['consistency_score']:.1f}%")
+    
+    if report['consistency_score'] >= 95:
+        print("   ✅ Excellent consistency!")
+    elif report['consistency_score'] >= 80:
+        print("   ⚠️  Good consistency with minor issues")
+    else:
+        print("   ❌ Poor consistency - significant issues found")
+    
+    if report['missing_projects']:
+        print(f"\n❌ Missing Projects ({len(report['missing_projects'])}):")
+        for project in report['missing_projects']:
+            print(f"   • {project}")
+    
+    print(f"\n📂 Common Folder Structure:")
+    for folder in report['common_folders']:
+        print(f"   • {folder}/")
+    
+    print(f"\n📄 Common Files:")
+    for file in report['common_files']:
+        print(f"   • {file}")
+    
+    if report['consistency_issues']:
+        print(f"\n⚠️  Consistency Issues ({len(report['consistency_issues'])}):")
+        for issue in report['consistency_issues']:
+            if issue['type'] == 'folder_mismatch':
+                print(f"   • {issue['project']}: Folder mismatch with {issue['reference']}")
+                if issue['missing_folders']:
+                    print(f"     Missing: {', '.join(issue['missing_folders'])}")
+                if issue['extra_folders']:
+                    print(f"     Extra: {', '.join(issue['extra_folders'])}")
+            elif issue['type'] == 'file_mismatch':
+                print(f"   • {issue['project']}: File mismatch with {issue['reference']}")
+                if issue['missing_files']:
+                    print(f"     Missing: {', '.join(issue['missing_files'])}")
+                if issue['extra_files']:
+                    print(f"     Extra: {', '.join(issue['extra_files'])}")
+    
+    print(f"\n📋 Detailed Project Structures:")
+    for project, structure in report['project_structures'].items():
+        if "error" in structure:
+            print(f"   • {project}: ❌ {structure['error']}")
+        else:
+            print(f"   • {project}: ✅ {len(structure['folders'])} folders, {len(structure['files'])} files")
+
+if __name__ == "__main__":
+    print("🚀 Starting Folder Structure Consistency Validation...")
+    report = validate_folder_consistency()
+    print_consistency_report(report)
+    
+    # Save report to file
+    script_dir = Path(__file__).parent
+    report_file = script_dir / "folder_structure_report.json"
+    
+    with open(report_file, 'w') as f:
+        json.dump(report, f, indent=2)
+    
+    print(f"\n💾 Report saved to: {report_file}")
+    
+    # Exit with appropriate code
+    if "error" in report:
+        exit(1)
+    elif report['consistency_score'] < 80:
+        print(f"\n⚠️  Folder structure consistency below threshold: {report['consistency_score']:.1f}%")
+        exit(1)
+    else:
+        print(f"\n✅ Folder structure consistency validation passed: {report['consistency_score']:.1f}%")
+        exit(0)

package/tools/vds-scripts/scripts/validate_link_integrity.py

@@ -0,0 +1,272 @@
+#!/usr/bin/env python3
+"""
+Link Integrity Validation Script (Internal Links Only)
+Validates that all internal links in documentation resolve correctly.
+"""
+
+import os
+import re
+from pathlib import Path
+from typing import Dict, List, Set, Tuple
+import json
+import urllib.parse
+
+# Link patterns to match
+MARKDOWN_LINK_PATTERN = r'\[([^\]]*)\]\(([^)]+)\)'
+REFERENCE_LINK_PATTERN = r'\[([^\]]*)\]:\s*([^\s]+)'
+
+def extract_links_from_file(file_path: Path) -> List[Tuple[str, str, int]]:
+    """Extract all links from a markdown file."""
+    links = []
+    
+    try:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            content = f.read()
+            
+        lines = content.split('\n')
+        
+        for line_num, line in enumerate(lines, 1):
+            # Find markdown links [text](url)
+            for match in re.finditer(MARKDOWN_LINK_PATTERN, line):
+                text, url = match.groups()
+                links.append((text, url, line_num))
+            
+            # Find reference links [text]: url
+            for match in re.finditer(REFERENCE_LINK_PATTERN, line):
+                text, url = match.groups()
+                links.append((text, url, line_num))
+                
+    except Exception as e:
+        print(f"Error reading {file_path}: {e}")
+        
+    return links
+
+def categorize_link(url: str, base_path: Path) -> str:
+    """Categorize a link as internal, external, or anchor."""
+    
+    # Remove fragments (anchors)
+    clean_url = url.split('#')[0]
+    
+    if not clean_url:
+        return "anchor"
+    elif clean_url.startswith(('http://', 'https://')):
+        return "external"
+    elif clean_url.startswith('mailto:'):
+        return "email"
+    else:
+        return "internal"
+
+def validate_internal_link(url: str, source_file: Path, docs_root: Path) -> Dict:
+    """Validate an internal link."""
+    
+    # Remove anchor fragment
+    clean_url = url.split('#')[0]
+    
+    if not clean_url:
+        return {"valid": True, "type": "anchor", "message": "Anchor link"}
+    
+    # Resolve relative path
+    if clean_url.startswith('/'):
+        # Absolute path from docs root
+        target_path = docs_root / clean_url.lstrip('/')
+    else:
+        # Relative path from source file
+        target_path = source_file.parent / clean_url
+    
+    # Normalize path
+    try:
+        target_path = target_path.resolve()
+    except Exception as e:
+        return {"valid": False, "type": "internal", "message": f"Path resolution error: {e}"}
+    
+    # Check if target exists
+    if target_path.exists():
+        return {"valid": True, "type": "internal", "message": "File exists"}
+    else:
+        return {"valid": False, "type": "internal", "message": f"File not found: {target_path}"}
+
+def validate_links_in_file(file_path: Path, docs_root: Path) -> Dict:
+    """Validate all links in a single file."""
+    
+    links = extract_links_from_file(file_path)
+    
+    results = {
+        "file": str(file_path.relative_to(docs_root)),
+        "total_links": len(links),
+        "internal_links": 0,
+        "external_links": 0,
+        "valid_internal": 0,
+        "broken_internal": 0,
+        "link_results": []
+    }
+    
+    for text, url, line_num in links:
+        link_type = categorize_link(url, file_path)
+        
+        if link_type == "internal" or link_type == "anchor":
+            results["internal_links"] += 1
+            validation = validate_internal_link(url, file_path, docs_root)
+            
+            if validation["valid"]:
+                results["valid_internal"] += 1
+            else:
+                results["broken_internal"] += 1
+                
+        elif link_type == "external":
+            results["external_links"] += 1
+            validation = {"valid": True, "type": "external", "message": "External link (not validated)"}
+        elif link_type == "email":
+            validation = {"valid": True, "type": "email", "message": "Email link"}
+        else:
+            validation = {"valid": False, "type": "unknown", "message": "Unknown link type"}
+        
+        link_result = {
+            "text": text,
+            "url": url,
+            "line": line_num,
+            "type": link_type,
+            "valid": validation["valid"],
+            "message": validation["message"]
+        }
+        
+        results["link_results"].append(link_result)
+    
+    return results
+
+def validate_link_integrity(max_files: int = 50) -> Dict:
+    """Validate link integrity across documentation."""
+    
+    # Get the documentation root
+    script_dir = Path(__file__).parent
+    docs_root = script_dir.parent / "insurance-document"
+    
+    if not docs_root.exists():
+        docs_root = script_dir.parent.parent / "insurance-document"
+    
+    if not docs_root.exists():
+        return {"error": "Documentation root not found"}
+    
+    print(f"🔍 Scanning links in documentation at: {docs_root}")
+    
+    # Find markdown files (limit to avoid overwhelming)
+    md_files = list(docs_root.rglob("*.md"))[:max_files]
+    
+    print(f"📄 Found {len(md_files)} markdown files (limited to {max_files})")
+    
+    # Validate links in each file
+    all_results = []
+    total_links = 0
+    total_internal = 0
+    total_external = 0
+    valid_internal = 0
+    broken_internal = 0
+    
+    for i, md_file in enumerate(md_files, 1):
+        print(f"   Processing {i}/{len(md_files)}: {md_file.name}")
+        
+        file_results = validate_links_in_file(md_file, docs_root)
+        all_results.append(file_results)
+        
+        total_links += file_results["total_links"]
+        total_internal += file_results["internal_links"]
+        total_external += file_results["external_links"]
+        valid_internal += file_results["valid_internal"]
+        broken_internal += file_results["broken_internal"]
+    
+    # Calculate success rate for internal links only
+    internal_success_rate = (valid_internal / total_internal * 100) if total_internal > 0 else 100
+    
+    # Generate summary report
+    report = {
+        "total_files": len(md_files),
+        "total_links": total_links,
+        "internal_links": total_internal,
+        "external_links": total_external,
+        "valid_internal": valid_internal,
+        "broken_internal": broken_internal,
+        "internal_success_rate": internal_success_rate,
+        "file_results": all_results,
+        "broken_internal_summary": []
+    }
+    
+    # Collect broken internal links summary
+    for file_result in all_results:
+        for link in file_result["link_results"]:
+            if link["type"] in ["internal", "anchor"] and not link["valid"]:
+                report["broken_internal_summary"].append({
+                    "file": file_result["file"],
+                    "line": link["line"],
+                    "url": link["url"],
+                    "message": link["message"]
+                })
+    
+    return report
+
+def print_link_integrity_report(report: Dict):
+    """Print a formatted link integrity report."""
+    
+    if "error" in report:
+        print(f"❌ {report['error']}")
+        return
+    
+    print("\n" + "="*60)
+    print("🔗 LINK INTEGRITY VALIDATION REPORT (Internal Links)")
+    print("="*60)
+    
+    print(f"\n📊 Summary:")
+    print(f"   Files Analyzed: {report['total_files']}")
+    print(f"   Total Links: {report['total_links']}")
+    print(f"   Internal Links: {report['internal_links']}")
+    print(f"   External Links: {report['external_links']} (not validated)")
+    print(f"   Valid Internal Links: {report['valid_internal']}")
+    print(f"   Broken Internal Links: {report['broken_internal']}")
+    print(f"   Internal Success Rate: {report['internal_success_rate']:.1f}%")
+    
+    if report['internal_success_rate'] >= 95:
+        print("   ✅ Excellent internal link integrity!")
+    elif report['internal_success_rate'] >= 80:
+        print("   ⚠️  Good internal link integrity with some issues")
+    else:
+        print("   ❌ Poor internal link integrity - many broken links")
+    
+    if report['broken_internal'] > 0:
+        print(f"\n❌ Broken Internal Links ({len(report['broken_internal_summary'])}):")
+        for i, broken in enumerate(report['broken_internal_summary'][:10], 1):  # Show first 10
+            print(f"   {i}. {broken['file']}:{broken['line']}")
+            print(f"      URL: {broken['url']}")
+            print(f"      Issue: {broken['message']}")
+        
+        if len(report['broken_internal_summary']) > 10:
+            print(f"   ... and {len(report['broken_internal_summary']) - 10} more broken links")
+    
+    print(f"\n📁 Files with Most Links:")
+    sorted_files = sorted(report['file_results'], key=lambda x: x['total_links'], reverse=True)
+    for file_result in sorted_files[:5]:
+        status = "✅" if file_result['broken_internal'] == 0 else f"⚠️ {file_result['broken_internal']} broken"
+        print(f"   • {file_result['file']}: {file_result['total_links']} links ({status})")
+
+if __name__ == "__main__":
+    print("🚀 Starting Link Integrity Validation (Internal Links Only)...")
+    
+    # Run validation with limited scope for performance
+    report = validate_link_integrity(max_files=20)  # Limit to 20 files for demo
+    print_link_integrity_report(report)
+    
+    # Save report to file
+    script_dir = Path(__file__).parent
+    report_file = script_dir / "link_integrity_report.json"
+    
+    with open(report_file, 'w') as f:
+        json.dump(report, f, indent=2)
+    
+    print(f"\n💾 Report saved to: {report_file}")
+    
+    # Exit with appropriate code
+    if "error" in report:
+        exit(1)
+    elif report['internal_success_rate'] < 80:
+        print(f"\n⚠️  Internal link integrity below threshold: {report['internal_success_rate']:.1f}%")
+        exit(1)
+    else:
+        print(f"\n✅ Internal link integrity validation passed: {report['internal_success_rate']:.1f}%")
+        exit(0)

package/tools/vds-scripts/scripts/vds_sh_helpers.sh

@@ -0,0 +1,180 @@
+#!/usr/bin/env bash
+# vds_sh_helpers.sh — General-purpose helpers for VDS shell scripts.
+#
+# Provides workspace root resolution plus a canonical uv invocation contract for
+# maintained shell scripts under WHO-project/vds-scripts.
+#
+# Canonical contract:
+#   - resolve the vds-scripts workspace root
+#   - invoke scripts/worktree_uv.sh
+#   - bind the workspace explicitly with --directory <root> for non-root callers
+#   - target uv packages by their [project].name value when using --package
+#
+# Callers may continue to pass workspace member directory names (e.g. vds_cli,
+# markdown_orchestrator); this helper resolves those to uv project names.
+
+set -euo pipefail
+
+# ---------------------------------------------------------------------------
+# vds_resolve_scripts_root
+#
+# Walk upward from this helper location until a pyproject.toml containing the
+# vds-scripts workspace marker is found.
+#
+# Outputs the absolute path to the vds-scripts workspace root (no trailing /).
+# ---------------------------------------------------------------------------
+vds_resolve_scripts_root() {
+  local dir candidate depth
+  dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+  candidate="$dir"
+  depth=0
+
+  while [[ "$candidate" != "/" && $depth -lt 20 ]]; do
+    if [[ -f "$candidate/pyproject.toml" ]] && grep -q "vds-scripts-workspace" "$candidate/pyproject.toml" 2>/dev/null; then
+      printf '%s\n' "$candidate"
+      return 0
+    fi
+    candidate="$(dirname "$candidate")"
+    (( depth++ )) || true
+  done
+
+  local fallback
+  fallback="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+  if [[ -f "$fallback/scripts/worktree_uv.sh" ]]; then
+    printf '%s\n' "$fallback"
+    return 0
+  fi
+
+  echo "vds_sh_helpers: cannot locate vds-scripts workspace root" >&2
+  return 1
+}
+
+# ---------------------------------------------------------------------------
+# resolve_insurance_docs_root [base]
+#
+# Locate the insurance-document directory, optionally relative to [base].
+# Preserved for backward compatibility with scripts that already call it.
+# ---------------------------------------------------------------------------
+resolve_insurance_docs_root() {
+  local base="${1:-$(vds_resolve_scripts_root)}"
+
+  if [[ -n "${VDS_INSURANCE_DOCS_ROOT:-}" ]]; then
+    printf '%s\n' "${VDS_INSURANCE_DOCS_ROOT}"
+    return 0
+  fi
+
+  if [[ -d "$base/WHO-project/insurance-document" ]]; then
+    printf '%s\n' "$base/WHO-project/insurance-document"
+    return 0
+  fi
+
+  local match=""
+  match="$(find "$base" -maxdepth 2 -type d -path "$base/*-project/insurance-document" 2>/dev/null | sort | head -n 1)"
+  if [[ -n "$match" ]]; then
+    printf '%s\n' "$match"
+    return 0
+  fi
+
+  printf '%s\n' "$base/WHO-project/insurance-document"
+}
+
+# ---------------------------------------------------------------------------
+# vds_uv_project_name <member-or-package>
+#
+# Resolve a workspace member directory name (e.g. vds_cli) to the uv package
+# name declared in that member's pyproject.toml (e.g. vds-cli). If the input is
+# already not a member directory, return it unchanged to allow explicit package
+# names from callers.
+#
+# Intentionally avoids a host-Python/tomllib dependency so scripts remain
+# portable across shells where `python3` may be older than 3.11.
+# ---------------------------------------------------------------------------
+vds_uv_project_name() {
+  local member_or_package="${1:?vds_uv_project_name: member or package is required}"
+  local root member_pyproject line
+  root="$(vds_resolve_scripts_root)"
+  member_pyproject="$root/$member_or_package/pyproject.toml"
+
+  if [[ ! -f "$member_pyproject" ]]; then
+    printf '%s\n' "$member_or_package"
+    return 0
+  fi
+
+  line="$(awk '
+    BEGIN { in_project = 0 }
+    /^\[project\]/ { in_project = 1; next }
+    /^\[/ && in_project { exit }
+    in_project && $0 ~ /^[[:space:]]*name[[:space:]]*=/ { print; exit }
+  ' "$member_pyproject")"
+
+  if [[ -z "$line" ]]; then
+    echo "vds_sh_helpers: missing [project].name in $member_pyproject" >&2
+    return 1
+  fi
+
+  line="${line#*=}"
+  line="$(printf '%s' "$line" | sed 's/^[[:space:]]*//; s/[[:space:]]*$//')"
+  line="${line#\"}"
+  line="${line%\"}"
+  printf '%s\n' "$line"
+}
+
+# ---------------------------------------------------------------------------
+# vds_worktree_uv <uv args...>
+#
+# Invoke scripts/worktree_uv.sh from the resolved workspace root.
+# ---------------------------------------------------------------------------
+vds_worktree_uv() {
+  local root wrapper
+  root="$(vds_resolve_scripts_root)"
+  wrapper="$root/scripts/worktree_uv.sh"
+
+  if [[ ! -x "$wrapper" ]]; then
+    echo "vds_sh_helpers: worktree_uv.sh not found or not executable at $wrapper" >&2
+    return 1
+  fi
+
+  "$wrapper" "$@"
+}
+
+# ---------------------------------------------------------------------------
+# vds_uv_run_package <member-or-package> [command args...]
+#
+# Run a command from a workspace package through the worktree wrapper, binding
+# the workspace explicitly with --directory for callers outside the root.
+# ---------------------------------------------------------------------------
+vds_uv_run_package() {
+  local member_or_package="${1:?vds_uv_run_package: member or package is required}"
+  shift
+  local root package_name
+  root="$(vds_resolve_scripts_root)"
+  package_name="$(vds_uv_project_name "$member_or_package")"
+
+  vds_worktree_uv run --directory "$root" --package "$package_name" "$@"
+}
+
+# ---------------------------------------------------------------------------
+# vds_uv_sync_package <member-or-package> [extra uv sync args...]
+#
+# Sync a single workspace package.
+# ---------------------------------------------------------------------------
+vds_uv_sync_package() {
+  local member_or_package="${1:?vds_uv_sync_package: member or package is required}"
+  shift
+  local root package_name
+  root="$(vds_resolve_scripts_root)"
+  package_name="$(vds_uv_project_name "$member_or_package")"
+
+  vds_worktree_uv sync --directory "$root" --package "$package_name" "$@"
+}
+
+# ---------------------------------------------------------------------------
+# vds_uv_sync_all [extra uv sync args...]
+#
+# Full workspace sync — syncs all members defined in the root pyproject.toml.
+# ---------------------------------------------------------------------------
+vds_uv_sync_all() {
+  local root
+  root="$(vds_resolve_scripts_root)"
+  vds_worktree_uv sync --directory "$root" --all-packages "$@"
+}