@ngocsangairvds/vsaf 3.2.13 → 3.2.15

package/tools/vds-scripts/audit_orchestrator/scripts/cli_integration_test.py

@@ -1,400 +0,0 @@
-#!/usr/bin/env python3
-"""Comprehensive CLI integration test script.
-
-Exercises all CLI features against real repositories to verify functionality.
-
-Usage:
-    uv run scripts/cli_integration_test.py                    # Use default test repo
-    uv run scripts/cli_integration_test.py --repo /path/to/repo  # Use specific repo
-    uv run scripts/cli_integration_test.py --quick            # Quick smoke test only
-"""
-
-from __future__ import annotations
-
-import argparse
-import subprocess
-import sys
-import tempfile
-import time
-from dataclasses import dataclass, field
-from pathlib import Path
-
-
-@dataclass
-class TestResult:
-    """Result of a CLI test."""
-
-    name: str
-    command: str
-    passed: bool
-    duration: float
-    output: str = ""
-    error: str = ""
-
-
-@dataclass
-class TestSuite:
-    """Collection of test results."""
-
-    results: list[TestResult] = field(default_factory=list)
-
-    def add(self, result: TestResult) -> None:
-        self.results.append(result)
-
-    @property
-    def passed_count(self) -> int:
-        return sum(1 for r in self.results if r.passed)
-
-    @property
-    def failed_count(self) -> int:
-        return sum(1 for r in self.results if not r.passed)
-
-    @property
-    def total_duration(self) -> float:
-        return sum(r.duration for r in self.results)
-
-
-def run_cli(
-    args: list[str],
-    cwd: Path,
-    timeout: int = 120,
-    expect_failure: bool = False,
-) -> TestResult:
-    """Run a CLI command and return the result."""
-    cmd = ["uv", "run", "vds-cli", "audit", *args]
-    cmd_str = " ".join(["vds-audit", *args])
-
-    start = time.monotonic()
-    try:
-        result = subprocess.run(
-            cmd,
-            cwd=cwd,
-            capture_output=True,
-            text=True,
-            timeout=timeout,
-        )
-        duration = time.monotonic() - start
-
-        passed = result.returncode != 0 if expect_failure else result.returncode == 0
-
-        return TestResult(
-            name=args[0] if args else "help",
-            command=cmd_str,
-            passed=passed,
-            duration=duration,
-            output=result.stdout,
-            error=result.stderr,
-        )
-    except subprocess.TimeoutExpired:
-        duration = time.monotonic() - start
-        return TestResult(
-            name=args[0] if args else "help",
-            command=cmd_str,
-            passed=False,
-            duration=duration,
-            error=f"Command timed out after {timeout}s",
-        )
-    except Exception as e:
-        duration = time.monotonic() - start
-        return TestResult(
-            name=args[0] if args else "help",
-            command=cmd_str,
-            passed=False,
-            duration=duration,
-            error=str(e),
-        )
-
-
-def test_help_commands(project_root: Path, suite: TestSuite) -> None:
-    """Test all help commands."""
-    print("\n📚 Testing help commands...")
-
-    # Main help
-    result = run_cli(["--help"], project_root)
-    result.name = "help (main)"
-    suite.add(result)
-
-    # Subcommand help
-    subcommands = [
-        "workflow",
-        "workflow-resume",
-        "workflow-pending",
-        "workflow-export",
-        "task-graph",
-        "list-pending",
-        "approve",
-        "approval-history",
-        "status",
-        "list-checks",
-        "validate-template",
-        "analyze-template",
-        "analyze-gaps",
-        "export-template",
-        "audit",
-        "extract-docs",
-        "list-materials",
-        "upload-results",
-        "export-debug-bundle",
-    ]
-
-    for cmd in subcommands:
-        result = run_cli([cmd, "--help"], project_root)
-        result.name = f"help ({cmd})"
-        suite.add(result)
-
-
-def test_status_command(project_root: Path, suite: TestSuite) -> None:
-    """Test status command."""
-    print("\n🔍 Testing status command...")
-
-    result = run_cli(["status"], project_root)
-    result.name = "status"
-    suite.add(result)
-
-
-def test_list_checks(project_root: Path, suite: TestSuite) -> None:
-    """Test list-checks command."""
-    print("\n📋 Testing list-checks command...")
-
-    result = run_cli(["list-checks"], project_root)
-    result.name = "list-checks"
-    suite.add(result)
-
-
-def test_validate_template(project_root: Path, suite: TestSuite) -> None:
-    """Test template validation."""
-    print("\n📄 Testing template validation...")
-
-    # Find a template file
-    template_dir = project_root / "templates"
-    templates = list(template_dir.glob("*.xlsx")) if template_dir.exists() else []
-
-    if templates:
-        template = templates[0]
-        # Use --template option, not positional argument
-        result = run_cli(["validate-template", "--template", str(template)], project_root)
-        result.name = f"validate-template ({template.name})"
-        suite.add(result)
-    else:
-        print("   ⚠️  No templates found, skipping validate-template test")
-
-
-def test_workflow_export(project_root: Path, suite: TestSuite) -> None:
-    """Test workflow visualization export."""
-    print("\n📊 Testing workflow export...")
-
-    # workflow-export requires OUTPUT_PATH as positional argument
-    with tempfile.NamedTemporaryFile(suffix=".mmd", delete=False) as f:
-        output_path = f.name
-
-    result = run_cli(["workflow-export", output_path], project_root)
-    result.name = "workflow-export"
-
-    # Verify file was created with mermaid content
-    if result.passed and Path(output_path).exists():
-        content = Path(output_path).read_text()
-        if "graph" not in content.lower() and "statediagram" not in content.lower():
-            result.passed = False
-            result.error = "Export doesn't contain expected mermaid content"
-        Path(output_path).unlink()
-    suite.add(result)
-
-
-def test_task_graph(project_root: Path, suite: TestSuite) -> None:
-    """Test task graph visualization."""
-    print("\n🗺️  Testing task graph...")
-
-    # task-graph requires --thread-id but may fail if thread doesn't exist
-    # This is expected behavior - test that it handles gracefully
-    with tempfile.NamedTemporaryFile(suffix=".mmd", delete=False) as f:
-        output_path = f.name
-
-    result = run_cli(
-        ["task-graph", "--thread-id", "test-thread-nonexistent", "-o", output_path],
-        project_root,
-    )
-    result.name = "task-graph"
-    # This command may fail if no tasks/workflow state exist, which is OK
-    # We just verify it doesn't crash unexpectedly
-    # Check both stdout and stderr for expected "no workflow/tasks" messages
-    combined_output = (result.output + result.error).lower()
-    expected_messages = [
-        "no tasks",
-        "no workflow state found",
-        "no workflow state",
-        "empty task graph",
-        "thread not found",
-    ]
-    if not result.passed and any(msg in combined_output for msg in expected_messages):
-        result.passed = True  # Expected behavior for nonexistent thread
-    suite.add(result)
-
-
-def test_workflow_pending(project_root: Path, suite: TestSuite) -> None:
-    """Test workflow pending list."""
-    print("\n⏳ Testing workflow-pending...")
-
-    result = run_cli(["workflow-pending"], project_root)
-    result.name = "workflow-pending"
-    suite.add(result)
-
-
-def test_list_pending(project_root: Path, suite: TestSuite) -> None:
-    """Test list-pending approvals."""
-    print("\n📝 Testing list-pending...")
-
-    result = run_cli(["list-pending"], project_root)
-    result.name = "list-pending"
-    suite.add(result)
-
-
-def test_analyze_template(project_root: Path, suite: TestSuite) -> None:
-    """Test template analysis."""
-    print("\n🔬 Testing analyze-template...")
-
-    template_dir = project_root / "templates"
-    templates = list(template_dir.glob("*.xlsx")) if template_dir.exists() else []
-
-    if templates:
-        template = templates[0]
-        # Use --template option, not positional argument
-        # LLM analysis can take 3-5 minutes for ~20 criteria (each LLM call ~10-15s)
-        result = run_cli(["analyze-template", "--template", str(template)], project_root, timeout=300)
-        result.name = f"analyze-template ({template.name})"
-        suite.add(result)
-    else:
-        print("   ⚠️  No templates found, skipping analyze-template test")
-
-
-def test_workflow_real_repo(
-    project_root: Path,
-    repo_path: Path,
-    suite: TestSuite,
-) -> None:
-    """Test workflow command against a real repository."""
-    print(f"\n🚀 Testing workflow against real repo: {repo_path.name}...")
-
-    # Run workflow with a unique thread ID
-    thread_id = f"cli-test-{int(time.time())}"
-
-    result = run_cli(
-        [
-            "workflow",
-            str(repo_path),
-            "--thread-id",
-            thread_id,
-        ],
-        project_root,
-        timeout=180,  # 3 minutes max
-    )
-    result.name = f"workflow ({repo_path.name})"
-    suite.add(result)
-
-
-def test_error_handling(project_root: Path, suite: TestSuite) -> None:
-    """Test error handling with invalid inputs."""
-    print("\n🛡️  Testing error handling...")
-
-    # Invalid repo path
-    result = run_cli(
-        ["workflow", "/nonexistent/path"],
-        project_root,
-        expect_failure=True,
-    )
-    result.name = "error: invalid repo path"
-    suite.add(result)
-
-    # Invalid template
-    result = run_cli(
-        ["validate-template", "/nonexistent/template.xlsx"],
-        project_root,
-        expect_failure=True,
-    )
-    result.name = "error: invalid template"
-    suite.add(result)
-
-
-def print_results(suite: TestSuite) -> None:
-    """Print test results summary."""
-    print("\n" + "=" * 70)
-    print("CLI Integration Test Results")
-    print("=" * 70)
-
-    for result in suite.results:
-        status = "✅" if result.passed else "❌"
-        print(f"{status} {result.name}: {result.duration:.2f}s")
-        if not result.passed and result.error:
-            for line in result.error.strip().split("\n")[-3:]:
-                print(f"   {line}")
-
-    print()
-    print("-" * 70)
-    print(f"Total: {suite.passed_count}/{len(suite.results)} passed in {suite.total_duration:.2f}s")
-
-    if suite.failed_count > 0:
-        print(f"❌ {suite.failed_count} tests failed")
-    else:
-        print("✅ All tests passed!")
-
-
-def main() -> int:
-    """Run CLI integration tests."""
-    parser = argparse.ArgumentParser(description="CLI integration tests")
-    parser.add_argument(
-        "--repo",
-        type=Path,
-        help="Path to test repository (defaults to vds-scripts parent directory)",
-    )
-    parser.add_argument(
-        "--quick",
-        action="store_true",
-        help="Quick smoke test (help commands only)",
-    )
-    args = parser.parse_args()
-
-    project_root = Path(__file__).parent.parent
-
-    # Default to using vds-scripts (parent of audit_orchestrator) as test target
-    # This is a proper git repo unlike audit_orchestrator subdirectory
-    default_repo = project_root.parent  # vds-scripts directory
-    repo_path = args.repo or default_repo
-
-    print("=" * 70)
-    print("VDS Audit Orchestrator - CLI Integration Tests")
-    print("=" * 70)
-    print(f"Project root: {project_root}")
-    print(f"Test repo: {repo_path}")
-    print()
-
-    suite = TestSuite()
-
-    # Always run help tests
-    test_help_commands(project_root, suite)
-
-    if args.quick:
-        print("\n⚡ Quick mode - skipping full tests")
-    else:
-        # Run all tests
-        test_status_command(project_root, suite)
-        test_list_checks(project_root, suite)
-        test_validate_template(project_root, suite)
-        test_workflow_export(project_root, suite)
-        test_task_graph(project_root, suite)
-        test_workflow_pending(project_root, suite)
-        test_list_pending(project_root, suite)
-        test_analyze_template(project_root, suite)
-        test_error_handling(project_root, suite)
-
-        # Only run workflow test if repo is a git repo
-        if (repo_path / ".git").exists():
-            test_workflow_real_repo(project_root, repo_path, suite)
-        else:
-            print(f"\n⚠️  {repo_path} is not a git repo, skipping workflow test")
-
-    print_results(suite)
-
-    return 0 if suite.failed_count == 0 else 1
-
-
-if __name__ == "__main__":
-    sys.exit(main())

package/tools/vds-scripts/audit_orchestrator/scripts/index_workspace.py

@@ -1,178 +0,0 @@
-#!/usr/bin/env python3
-"""Workspace symbol indexer for VDS Audit Orchestrator.
-
-Provides workspace-wide symbol search similar to LSP workspaceSymbol
-when the persistent LSP connection is not available.
-"""
-
-import ast
-import json
-from pathlib import Path
-from typing import Any
-
-
-def index_workspace(root_dir: Path = Path("src")) -> dict[str, Any]:
-    """Index all Python files in the workspace.
-
-    Args:
-        root_dir: Root directory to index (default: src/)
-
-    Returns:
-        Dictionary with symbols indexed by type and name.
-    """
-    # Get absolute path for relative calculations
-    root_dir = root_dir.resolve()
-    cwd = Path.cwd()
-
-    index = {
-        "classes": {},
-        "functions": {},
-        "methods": {},
-        "constants": {},
-        "imports": {},
-    }
-
-    for py_file in root_dir.rglob("*.py"):
-        try:
-            with open(py_file, encoding="utf-8") as f:
-                source = f.read()
-            tree = ast.parse(source, filename=str(py_file))
-
-            # Calculate relative path if possible, otherwise use absolute
-            try:
-                rel_path = py_file.relative_to(cwd)
-            except ValueError:
-                rel_path = py_file
-
-            for node in ast.walk(tree):
-                symbol_info = {
-                    "file": str(rel_path),
-                    "line": getattr(node, "lineno", 0),
-                }
-
-                if isinstance(node, ast.ClassDef):
-                    index["classes"][node.name] = [
-                        *index["classes"].get(node.name, []),
-                        symbol_info,
-                    ]
-
-                elif isinstance(node, ast.FunctionDef):
-                    # Check if it's a method (inside a class)
-                    if hasattr(node, "parent"):
-                        # Will be processed in class traversal
-                        continue
-
-                    index["functions"][node.name] = [
-                        *index["functions"].get(node.name, []),
-                        symbol_info,
-                    ]
-
-                elif isinstance(node, ast.AsyncFunctionDef):
-                    if not hasattr(node, "parent"):
-                        index["functions"][node.name] = [
-                            *index["functions"].get(node.name, []),
-                            {**symbol_info, "async": True},
-                        ]
-
-                elif isinstance(node, ast.Assign):
-                    for target in node.targets:
-                        if isinstance(target, ast.Name) and target.id.isupper():
-                            index["constants"][target.id] = [
-                                *index["constants"].get(target.id, []),
-                                symbol_info,
-                            ]
-
-        except (SyntaxError, UnicodeDecodeError):
-            # Skip files that can't be parsed
-            continue
-
-    return index
-
-
-def find_calls(root_dir: Path = Path("src")) -> dict[str, list[str]]:
-    """Build a call graph by analyzing function calls.
-
-    Args:
-        root_dir: Root directory to analyze
-
-    Returns:
-        Dictionary mapping function names to lists of functions they call.
-    """
-    call_graph = {}
-
-    for py_file in root_dir.rglob("*.py"):
-        try:
-            with open(py_file, encoding="utf-8") as f:
-                source = f.read()
-            tree = ast.parse(source, filename=str(py_file))
-
-            class_context = None
-
-            for node in ast.walk(tree):
-                if isinstance(node, ast.ClassDef):
-                    class_context = node.name
-
-                elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
-                    func_name = node.name
-                    if class_context:
-                        func_name = f"{class_context}.{node.name}"
-
-                    calls = []
-                    for child in ast.walk(node):
-                        if isinstance(child, ast.Call):
-                            if isinstance(child.func, ast.Name):
-                                calls.append(child.func.id)
-                            if isinstance(child.func, ast.Attribute) and isinstance(child.func.value, ast.Name):
-                                    calls.append(f"{child.func.value.id}.{child.func.attr}")
-
-                    if calls:
-                        call_graph[func_name] = calls
-
-        except (SyntaxError, UnicodeDecodeError):
-            continue
-
-    return call_graph
-
-
-def find_incoming_calls(call_graph: dict[str, list[str]], func_name: str) -> list[str]:
-    """Find all functions that call the given function.
-
-    Args:
-        call_graph: Call graph from find_calls()
-        func_name: Function to find callers for
-
-    Returns:
-        List of function names that call the given function.
-    """
-    callers = []
-    for caller, callees in call_graph.items():
-        if func_name in callees:
-            callers.append(caller)
-    return callers
-
-
-if __name__ == "__main__":
-    import sys
-
-    root = Path("src")
-    if len(sys.argv) > 1:
-        root = Path(sys.argv[1])
-
-    print(f"Indexing workspace: {root}")
-    index = index_workspace(root)
-    call_graph = find_calls(root)
-
-    print(f"\nFound {len(index['classes'])} classes")
-    print(f"Found {len(index['functions'])} functions")
-    print(f"Found {len(index['constants'])} constants")
-    print(f"Built call graph with {len(call_graph)} entries")
-
-    # Example: find incoming calls for 'analyze'
-    analyze_callers = find_incoming_calls(call_graph, "analyze")
-    print(f"\nFunctions calling 'analyze': {analyze_callers}")
-
-    # Save index to JSON
-    with open("workspace_index.json", "w") as f:
-        json.dump({"index": index, "call_graph": call_graph}, f, indent=2)
-
-    print("\nIndex saved to workspace_index.json")