@mseep/csv-editor 1.0.0

package/src/csv_editor/utils/validators.py

@@ -0,0 +1,205 @@
+"""Validation utilities for CSV Editor."""
+
+import os
+import re
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+import pandas as pd
+
+
+def validate_file_path(file_path: str, must_exist: bool = True) -> tuple[bool, str]:
+    """Validate a file path for security and existence."""
+    try:
+        # Convert to Path object
+        path = Path(file_path).resolve()
+
+        # Security: Check for path traversal attempts
+        if ".." in file_path or file_path.startswith("~"):
+            return False, "Path traversal not allowed"
+
+        # Check file existence if required
+        if must_exist and not path.exists():
+            return False, f"File not found: {file_path}"
+
+        # Check if it's a file (not directory)
+        if must_exist and not path.is_file():
+            return False, f"Not a file: {file_path}"
+
+        # Check file extension
+        valid_extensions = [".csv", ".tsv", ".txt", ".dat"]
+        if path.suffix.lower() not in valid_extensions:
+            return False, f"Invalid file extension. Supported: {valid_extensions}"
+
+        # Check file size (max 1GB)
+        if must_exist:
+            max_size = 1024 * 1024 * 1024  # 1GB
+            if path.stat().st_size > max_size:
+                return False, "File too large. Maximum size: 1GB"
+
+        return True, str(path)
+
+    except Exception as e:
+        return False, f"Error validating path: {e!s}"
+
+
+def validate_url(url: str) -> tuple[bool, str]:
+    """Validate a URL for CSV download."""
+    try:
+        parsed = urlparse(url)
+
+        # Check scheme
+        if parsed.scheme not in ["http", "https"]:
+            return False, "Only HTTP/HTTPS URLs are supported"
+
+        # Check if URL is valid
+        if not parsed.netloc:
+            return False, "Invalid URL format"
+
+        return True, url
+
+    except Exception as e:
+        return False, f"Invalid URL: {e!s}"
+
+
+def validate_column_name(column_name: str) -> tuple[bool, str]:
+    """Validate a column name."""
+    if not column_name or not isinstance(column_name, str):
+        return False, "Column name must be a non-empty string"
+
+    # Check for invalid characters
+    if re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", column_name):
+        return True, column_name
+    else:
+        return (
+            False,
+            "Column name must start with letter/underscore and contain only letters, numbers, underscores",
+        )
+
+
+def validate_dataframe(df: pd.DataFrame) -> dict[str, Any]:
+    """Validate a DataFrame for common issues."""
+    issues = {"errors": [], "warnings": [], "info": {}}
+
+    # Check if empty
+    if df.empty:
+        issues["errors"].append("DataFrame is empty")
+        return issues
+
+    # Check shape
+    issues["info"]["shape"] = df.shape
+    issues["info"]["memory_usage_mb"] = df.memory_usage(deep=True).sum() / (1024 * 1024)
+
+    # Check for duplicate columns
+    if df.columns.duplicated().any():
+        dupes = df.columns[df.columns.duplicated()].tolist()
+        issues["errors"].append(f"Duplicate column names: {dupes}")
+
+    # Check for completely null columns
+    null_cols = df.columns[df.isnull().all()].tolist()
+    if null_cols:
+        issues["warnings"].append(f"Completely null columns: {null_cols}")
+
+    # Check for mixed types in columns
+    for col in df.columns:
+        if df[col].dtype == "object":
+            # Try to infer if it's mixed types
+            unique_types = df[col].dropna().apply(type).unique()
+            if len(unique_types) > 1:
+                issues["warnings"].append(
+                    f"Column '{col}' has mixed types: {[t.__name__ for t in unique_types]}"
+                )
+
+    # Check for high cardinality in string columns
+    for col in df.select_dtypes(include=["object"]).columns:
+        unique_ratio = df[col].nunique() / len(df)
+        if unique_ratio > 0.9:
+            issues["info"][f"{col}_high_cardinality"] = True
+
+    # Check for potential datetime columns
+    for col in df.select_dtypes(include=["object"]).columns:
+        sample = df[col].dropna().head(100)
+        if sample.empty:
+            continue
+        try:
+            pd.to_datetime(sample, errors="raise")
+            issues["info"][f"{col}_potential_datetime"] = True
+        except:
+            pass
+
+    return issues
+
+
+def validate_expression(expression: str, allowed_vars: list[str]) -> tuple[bool, str]:
+    """Validate a calculation expression for safety."""
+    # Remove whitespace
+    expr = expression.replace(" ", "")
+
+    # Check for dangerous operations
+    dangerous_patterns = [
+        "__",
+        "import",
+        "exec",
+        "eval",
+        "compile",
+        "open",
+        "file",
+        "input",
+        "raw_input",
+        "globals",
+        "locals",
+    ]
+
+    for pattern in dangerous_patterns:
+        if pattern in expr.lower():
+            return False, f"Dangerous operation '{pattern}' not allowed"
+
+    # Check if only allowed variables and safe operations are used
+    # This is a simplified check - in production use ast module for proper parsing
+    safe_chars = set("0123456789+-*/().,<>=! ")
+    safe_functions = {"abs", "min", "max", "sum", "len", "round", "int", "float", "str"}
+
+    # Extract potential variable/function names
+    tokens = re.findall(r"[a-zA-Z_][a-zA-Z0-9_]*", expr)
+
+    for token in tokens:
+        if token not in allowed_vars and token not in safe_functions:
+            return False, f"Unknown variable or function: {token}"
+
+    return True, expression
+
+
+def validate_sql_query(query: str) -> tuple[bool, str]:
+    """Validate SQL query for safety (basic check)."""
+    query_lower = query.lower()
+
+    # Only allow SELECT queries
+    if not query_lower.strip().startswith("select"):
+        return False, "Only SELECT queries are allowed"
+
+    # Check for dangerous keywords
+    dangerous = ["drop", "delete", "insert", "update", "alter", "create", "exec", "execute"]
+    for keyword in dangerous:
+        if keyword in query_lower:
+            return False, f"Dangerous operation '{keyword}' not allowed"
+
+    return True, query
+
+
+def sanitize_filename(filename: str) -> str:
+    """Sanitize a filename for safe file operations."""
+    # Remove path components
+    filename = os.path.basename(filename)
+
+    # Remove/replace invalid characters
+    invalid_chars = '<>:"|?*'
+    for char in invalid_chars:
+        filename = filename.replace(char, "_")
+
+    # Limit length
+    name, ext = os.path.splitext(filename)
+    if len(name) > 100:
+        name = name[:100]
+
+    return name + ext

package/tests/README.md

@@ -0,0 +1,65 @@
+# CSV Editor Tests
+
+Comprehensive test suite for the CSV Editor MCP Server.
+
+## Running Tests
+
+### Run all tests
+```bash
+uv run pytest
+```
+
+### Run with coverage
+```bash
+uv run pytest --cov=src/csv_editor --cov-report=html
+```
+
+### Run specific test file
+```bash
+uv run pytest tests/test_basic.py
+```
+
+### Run integration tests
+```bash
+uv run pytest tests/test_integration.py
+```
+
+## Test Structure
+
+- **test_basic.py** - Unit tests for core functionality
+  - Validators
+  - Session management
+  - Basic operations
+
+- **test_integration.py** - Full integration tests
+  - Complete workflows
+  - All tool operations
+  - Export functionality
+
+- **conftest.py** - Pytest configuration and fixtures
+  - Session fixtures
+  - Sample data fixtures
+  - Event loop configuration
+
+## Writing New Tests
+
+All test files should:
+1. Start with `test_`
+2. Use pytest fixtures from conftest.py
+3. Mark async tests with `@pytest.mark.asyncio`
+4. Clean up sessions after tests
+
+Example:
+```python
+@pytest.mark.asyncio
+async def test_my_feature(test_session):
+    """Test description."""
+    from src.csv_editor.tools.analytics import get_statistics
+    
+    result = await get_statistics(
+        session_id=test_session,
+        columns=["price"]
+    )
+    
+    assert result["success"] == True
+```

package/tests/__init__.py

@@ -0,0 +1,7 @@
+"""Test suite for CSV Editor."""
+
+import sys
+from pathlib import Path
+
+# Add parent directory to path for imports
+sys.path.insert(0, str(Path(__file__).parent.parent))

package/tests/conftest.py

@@ -0,0 +1,50 @@
+"""Pytest configuration for CSV Editor tests."""
+
+import asyncio
+import sys
+from pathlib import Path
+
+import pytest
+
+# Add src to path
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+
+@pytest.fixture(scope="session")
+def event_loop():
+    """Create an event loop for the test session."""
+    loop = asyncio.get_event_loop_policy().new_event_loop()
+    yield loop
+    loop.close()
+
+
+@pytest.fixture
+def sample_csv_data():
+    """Provide sample CSV data for testing."""
+    return """name,age,salary,department
+Alice,30,60000,Engineering
+Bob,25,50000,Marketing
+Charlie,35,70000,Engineering
+Diana,28,55000,Sales"""
+
+
+@pytest.fixture
+async def test_session():
+    """Create a test session."""
+    from src.csv_editor.models import get_session_manager
+    from src.csv_editor.tools.io_operations import load_csv_from_content
+
+    # Create session with sample data
+    result = await load_csv_from_content(
+        content="""product,price,quantity
+Laptop,999.99,10
+Mouse,29.99,50
+Keyboard,79.99,25""",
+        delimiter=",",
+    )
+
+    yield result["session_id"]
+
+    # Cleanup
+    manager = get_session_manager()
+    manager.remove_session(result["session_id"])

package/tests/test_auto_save.py

@@ -0,0 +1,378 @@
+"""Tests for auto-save functionality."""
+
+import asyncio
+import shutil
+import tempfile
+from pathlib import Path
+
+import pandas as pd
+import pytest
+
+from src.csv_editor.models.auto_save import AutoSaveConfig, AutoSaveMode, AutoSaveStrategy
+from src.csv_editor.models.csv_session import CSVSession, SessionManager
+from src.csv_editor.models.data_models import ExportFormat
+
+
+@pytest.fixture
+def sample_df():
+    """Create a sample DataFrame for testing."""
+    return pd.DataFrame(
+        {
+            "name": ["Alice", "Bob", "Charlie"],
+            "age": [25, 30, 35],
+            "city": ["New York", "London", "Paris"],
+        }
+    )
+
+
+@pytest.fixture
+def temp_dir():
+    """Create a temporary directory for test files."""
+    temp_dir = tempfile.mkdtemp()
+    yield temp_dir
+    # Cleanup
+    shutil.rmtree(temp_dir, ignore_errors=True)
+
+
+@pytest.mark.asyncio
+async def test_auto_save_config_creation():
+    """Test creating auto-save configuration."""
+    config = AutoSaveConfig(
+        enabled=True,
+        mode=AutoSaveMode.AFTER_OPERATION,
+        strategy=AutoSaveStrategy.BACKUP,
+        interval_seconds=60,
+        max_backups=5,
+    )
+
+    assert config.enabled is True
+    assert config.mode == AutoSaveMode.AFTER_OPERATION
+    assert config.strategy == AutoSaveStrategy.BACKUP
+    assert config.interval_seconds == 60
+    assert config.max_backups == 5
+
+
+@pytest.mark.asyncio
+async def test_auto_save_config_from_dict():
+    """Test creating auto-save config from dictionary."""
+    config_dict = {
+        "enabled": True,
+        "mode": "periodic",
+        "strategy": "versioned",
+        "interval_seconds": 120,
+        "max_backups": 10,
+    }
+
+    config = AutoSaveConfig.from_dict(config_dict)
+
+    assert config.enabled is True
+    assert config.mode == AutoSaveMode.PERIODIC
+    assert config.strategy == AutoSaveStrategy.VERSIONED
+    assert config.interval_seconds == 120
+    assert config.max_backups == 10
+
+
+@pytest.mark.asyncio
+async def test_session_with_auto_save_disabled(sample_df):
+    """Test session with auto-save disabled."""
+    config = AutoSaveConfig(enabled=False)
+    session = CSVSession(auto_save_config=config)
+    session.load_data(sample_df)
+
+    # Perform an operation
+    session.record_operation("test_op", {"test": "data"})
+
+    # Auto-save should not trigger
+    result = await session.trigger_auto_save_if_needed()
+    assert result is None
+
+
+@pytest.mark.asyncio
+async def test_session_with_auto_save_after_operation(sample_df, temp_dir):
+    """Test auto-save after each operation."""
+    config = AutoSaveConfig(
+        enabled=True,
+        mode=AutoSaveMode.AFTER_OPERATION,
+        strategy=AutoSaveStrategy.BACKUP,
+        backup_dir=temp_dir,
+    )
+
+    session = CSVSession(auto_save_config=config)
+    session.load_data(sample_df)
+
+    # Perform an operation
+    session.record_operation("test_op", {"test": "data"})
+
+    # Trigger auto-save
+    result = await session.trigger_auto_save_if_needed()
+
+    assert result is not None
+    assert result["success"] is True
+    assert result["trigger"] == "after_operation"
+
+    # Check that backup file was created
+    backup_files = list(Path(temp_dir).glob(f"*{session.session_id}*"))
+    assert len(backup_files) == 1
+
+
+@pytest.mark.asyncio
+async def test_manual_save(sample_df, temp_dir):
+    """Test manual save trigger."""
+    config = AutoSaveConfig(
+        enabled=True,
+        mode=AutoSaveMode.DISABLED,
+        strategy=AutoSaveStrategy.BACKUP,
+        backup_dir=temp_dir,
+    )
+
+    session = CSVSession(auto_save_config=config)
+    session.load_data(sample_df)
+
+    # Trigger manual save
+    result = await session.manual_save()
+
+    assert result["success"] is True
+    assert result["trigger"] == "manual"
+
+    # Check that backup file was created
+    backup_files = list(Path(temp_dir).glob(f"*{session.session_id}*"))
+    assert len(backup_files) == 1
+
+
+@pytest.mark.asyncio
+async def test_versioned_save_strategy(sample_df, temp_dir):
+    """Test versioned save strategy."""
+    config = AutoSaveConfig(
+        enabled=True,
+        mode=AutoSaveMode.AFTER_OPERATION,
+        strategy=AutoSaveStrategy.VERSIONED,
+        backup_dir=temp_dir,
+    )
+
+    session = CSVSession(auto_save_config=config)
+    session.load_data(sample_df)
+
+    # Trigger multiple saves
+    for i in range(3):
+        session.record_operation(f"test_op_{i}", {"index": i})
+        await session.trigger_auto_save_if_needed()
+
+    # Check that versioned files were created
+    backup_files = sorted(Path(temp_dir).glob(f"version_{session.session_id}_v*.csv"))
+    assert len(backup_files) == 3
+    assert "v0001" in str(backup_files[0])
+    assert "v0002" in str(backup_files[1])
+    assert "v0003" in str(backup_files[2])
+
+
+@pytest.mark.asyncio
+async def test_max_backups_cleanup(sample_df, temp_dir):
+    """Test that old backups are cleaned up when max_backups is exceeded."""
+    config = AutoSaveConfig(
+        enabled=True,
+        mode=AutoSaveMode.AFTER_OPERATION,
+        strategy=AutoSaveStrategy.BACKUP,
+        backup_dir=temp_dir,
+        max_backups=3,
+    )
+
+    session = CSVSession(auto_save_config=config)
+    session.load_data(sample_df)
+
+    # Trigger more saves than max_backups
+    for i in range(5):
+        session.record_operation(f"test_op_{i}", {"index": i})
+        await session.trigger_auto_save_if_needed()
+        await asyncio.sleep(0.1)  # Small delay to ensure different timestamps
+
+    # Check that only max_backups files remain
+    backup_files = list(Path(temp_dir).glob(f"*{session.session_id}*"))
+    assert len(backup_files) <= config.max_backups
+
+
+@pytest.mark.asyncio
+async def test_periodic_save(sample_df, temp_dir):
+    """Test periodic auto-save."""
+    config = AutoSaveConfig(
+        enabled=True,
+        mode=AutoSaveMode.PERIODIC,
+        strategy=AutoSaveStrategy.BACKUP,
+        backup_dir=temp_dir,
+        interval_seconds=1,  # 1 second for testing
+    )
+
+    session = CSVSession(auto_save_config=config)
+    session.load_data(sample_df)
+
+    # Start periodic save
+    await session.auto_save_manager.start_periodic_save(session._save_callback)
+
+    # Wait for periodic save to trigger
+    await asyncio.sleep(1.5)
+
+    # Stop periodic save
+    await session.auto_save_manager.stop_periodic_save()
+
+    # Check that backup file was created
+    backup_files = list(Path(temp_dir).glob(f"*{session.session_id}*"))
+    assert len(backup_files) >= 1
+
+
+@pytest.mark.asyncio
+async def test_hybrid_mode(sample_df, temp_dir):
+    """Test hybrid mode (both periodic and after-operation)."""
+    config = AutoSaveConfig(
+        enabled=True,
+        mode=AutoSaveMode.HYBRID,
+        strategy=AutoSaveStrategy.BACKUP,
+        backup_dir=temp_dir,
+        interval_seconds=2,
+    )
+
+    session = CSVSession(auto_save_config=config)
+    session.load_data(sample_df)
+
+    # Start periodic save
+    await session.auto_save_manager.start_periodic_save(session._save_callback)
+
+    # Trigger operation-based save
+    session.record_operation("test_op", {"test": "data"})
+    result = await session.trigger_auto_save_if_needed()
+    assert result["success"] is True
+
+    # Wait for periodic save
+    await asyncio.sleep(2.5)
+
+    # Stop periodic save
+    await session.auto_save_manager.stop_periodic_save()
+
+    # Should have multiple backup files
+    backup_files = list(Path(temp_dir).glob(f"*{session.session_id}*"))
+    assert len(backup_files) >= 2
+
+
+@pytest.mark.asyncio
+async def test_different_export_formats(sample_df, temp_dir):
+    """Test auto-save with different export formats."""
+    formats = [ExportFormat.CSV, ExportFormat.JSON, ExportFormat.TSV]
+
+    for format in formats:
+        config = AutoSaveConfig(
+            enabled=True,
+            mode=AutoSaveMode.AFTER_OPERATION,
+            strategy=AutoSaveStrategy.BACKUP,
+            backup_dir=temp_dir,
+            format=format,
+        )
+
+        session = CSVSession(auto_save_config=config)
+        session.load_data(sample_df)
+
+        # Trigger save
+        session.record_operation("test_op", {"format": format.value})
+        result = await session.trigger_auto_save_if_needed()
+
+        assert result["success"] is True
+
+        # Check file was created with correct extension
+        pattern = f"*{session.session_id}*.{format.value}"
+        files = list(Path(temp_dir).glob(pattern))
+        assert len(files) == 1
+
+
+@pytest.mark.asyncio
+async def test_enable_disable_auto_save(sample_df, temp_dir):
+    """Test enabling and disabling auto-save."""
+    session = CSVSession()
+    session.load_data(sample_df)
+
+    # Initially disabled
+    assert session.auto_save_config.enabled is False
+
+    # Enable auto-save
+    config_dict = {
+        "enabled": True,
+        "mode": "after_operation",
+        "strategy": "backup",
+        "backup_dir": temp_dir,
+    }
+    result = await session.enable_auto_save(config_dict)
+    assert result["success"] is True
+    assert session.auto_save_config.enabled is True
+
+    # Trigger save
+    session.record_operation("test_op", {"test": "data"})
+    save_result = await session.trigger_auto_save_if_needed()
+    assert save_result["success"] is True
+
+    # Disable auto-save
+    result = await session.disable_auto_save()
+    assert result["success"] is True
+    assert session.auto_save_config.enabled is False
+
+    # Save should not trigger
+    session.record_operation("test_op_2", {"test": "data"})
+    save_result = await session.trigger_auto_save_if_needed()
+    assert save_result is None
+
+
+@pytest.mark.asyncio
+async def test_get_auto_save_status(sample_df):
+    """Test getting auto-save status."""
+    config = AutoSaveConfig(
+        enabled=True, mode=AutoSaveMode.AFTER_OPERATION, strategy=AutoSaveStrategy.BACKUP
+    )
+
+    session = CSVSession(auto_save_config=config)
+    session.load_data(sample_df)
+
+    status = session.get_auto_save_status()
+
+    assert status["enabled"] is True
+    assert status["mode"] == "after_operation"
+    assert status["strategy"] == "backup"
+    assert status["save_count"] == 0
+    assert status["periodic_active"] is False
+
+    # Trigger a save
+    await session.manual_save()
+
+    status = session.get_auto_save_status()
+    assert status["save_count"] == 1
+
+
+@pytest.mark.asyncio
+async def test_session_manager_cleanup(sample_df, temp_dir):
+    """Test that auto-save is stopped when session is removed."""
+    config = AutoSaveConfig(
+        enabled=True,
+        mode=AutoSaveMode.PERIODIC,
+        strategy=AutoSaveStrategy.BACKUP,
+        backup_dir=temp_dir,
+        interval_seconds=1,
+    )
+
+    manager = SessionManager()
+    session_id = manager.create_session()
+    session = manager.get_session(session_id)
+
+    # Enable auto-save
+    await session.enable_auto_save(config.to_dict())
+    session.load_data(sample_df)
+
+    # Start periodic save
+    await session.auto_save_manager.start_periodic_save(session._save_callback)
+
+    # Remove session (should stop auto-save)
+    removed = await manager.remove_session(session_id)
+    assert removed is True
+
+    # Verify periodic task was cancelled
+    assert (
+        session.auto_save_manager.periodic_task is None
+        or session.auto_save_manager.periodic_task.done()
+    )
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])