claude-code-workflow 6.1.4 → 6.2.2

package/codex-lens/src/codexlens/storage/migrations/migration_005_cleanup_unused_fields.py

@@ -0,0 +1,196 @@
+"""
+Migration 005: Remove unused and redundant database fields.
+
+This migration removes four problematic fields identified by Gemini analysis:
+
+1. **semantic_metadata.keywords** (deprecated - replaced by file_keywords table)
+   - Data: Migrated to normalized file_keywords table in migration 001
+   - Impact: Column now redundant, remove to prevent sync issues
+
+2. **symbols.token_count** (unused - always NULL)
+   - Data: Never populated, always NULL
+   - Impact: No data loss, just removes unused column
+
+3. **symbols.symbol_type** (redundant - duplicates kind)
+   - Data: Redundant with symbols.kind field
+   - Impact: No data loss, kind field contains same information
+
+4. **subdirs.direct_files** (unused - never displayed)
+   - Data: Never used in queries or display logic
+   - Impact: No data loss, just removes unused column
+
+Schema changes use table recreation pattern (SQLite best practice):
+- Create new table without deprecated columns
+- Copy data from old table
+- Drop old table
+- Rename new table
+- Recreate indexes
+"""
+
+import logging
+from sqlite3 import Connection
+
+log = logging.getLogger(__name__)
+
+
+def upgrade(db_conn: Connection):
+    """Remove unused and redundant fields from schema.
+
+    Note: Transaction management is handled by MigrationManager.
+    This migration should NOT start its own transaction.
+
+    Args:
+        db_conn: The SQLite database connection.
+    """
+    cursor = db_conn.cursor()
+
+    # Step 1: Remove semantic_metadata.keywords (if column exists)
+    log.info("Checking semantic_metadata.keywords column...")
+
+    cursor.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='semantic_metadata'"
+    )
+    if cursor.fetchone():
+        # Check if keywords column exists
+        cursor.execute("PRAGMA table_info(semantic_metadata)")
+        columns = {row[1] for row in cursor.fetchall()}
+
+        if "keywords" in columns:
+            log.info("Removing semantic_metadata.keywords column...")
+            cursor.execute("""
+                CREATE TABLE semantic_metadata_new (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    file_id INTEGER NOT NULL UNIQUE,
+                    summary TEXT,
+                    purpose TEXT,
+                    llm_tool TEXT,
+                    generated_at REAL,
+                    FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE
+                )
+            """)
+
+            cursor.execute("""
+                INSERT INTO semantic_metadata_new (id, file_id, summary, purpose, llm_tool, generated_at)
+                SELECT id, file_id, summary, purpose, llm_tool, generated_at
+                FROM semantic_metadata
+            """)
+
+            cursor.execute("DROP TABLE semantic_metadata")
+            cursor.execute("ALTER TABLE semantic_metadata_new RENAME TO semantic_metadata")
+
+            # Recreate index
+            cursor.execute(
+                "CREATE INDEX IF NOT EXISTS idx_semantic_file ON semantic_metadata(file_id)"
+            )
+            log.info("Removed semantic_metadata.keywords column")
+        else:
+            log.info("semantic_metadata.keywords column does not exist, skipping")
+    else:
+        log.info("semantic_metadata table does not exist, skipping")
+
+    # Step 2: Remove symbols.token_count and symbols.symbol_type (if columns exist)
+    log.info("Checking symbols.token_count and symbols.symbol_type columns...")
+
+    cursor.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='symbols'"
+    )
+    if cursor.fetchone():
+        # Check if token_count or symbol_type columns exist
+        cursor.execute("PRAGMA table_info(symbols)")
+        columns = {row[1] for row in cursor.fetchall()}
+
+        if "token_count" in columns or "symbol_type" in columns:
+            log.info("Removing symbols.token_count and symbols.symbol_type columns...")
+            cursor.execute("""
+                CREATE TABLE symbols_new (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    file_id INTEGER NOT NULL,
+                    name TEXT NOT NULL,
+                    kind TEXT,
+                    start_line INTEGER,
+                    end_line INTEGER,
+                    FOREIGN KEY (file_id) REFERENCES files(id) ON DELETE CASCADE
+                )
+            """)
+
+            cursor.execute("""
+                INSERT INTO symbols_new (id, file_id, name, kind, start_line, end_line)
+                SELECT id, file_id, name, kind, start_line, end_line
+                FROM symbols
+            """)
+
+            cursor.execute("DROP TABLE symbols")
+            cursor.execute("ALTER TABLE symbols_new RENAME TO symbols")
+
+            # Recreate indexes
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_symbols_file ON symbols(file_id)")
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_symbols_name ON symbols(name)")
+            log.info("Removed symbols.token_count and symbols.symbol_type columns")
+        else:
+            log.info("symbols.token_count/symbol_type columns do not exist, skipping")
+    else:
+        log.info("symbols table does not exist, skipping")
+
+    # Step 3: Remove subdirs.direct_files (if column exists)
+    log.info("Checking subdirs.direct_files column...")
+
+    cursor.execute(
+        "SELECT name FROM sqlite_master WHERE type='table' AND name='subdirs'"
+    )
+    if cursor.fetchone():
+        # Check if direct_files column exists
+        cursor.execute("PRAGMA table_info(subdirs)")
+        columns = {row[1] for row in cursor.fetchall()}
+
+        if "direct_files" in columns:
+            log.info("Removing subdirs.direct_files column...")
+            cursor.execute("""
+                CREATE TABLE subdirs_new (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    name TEXT NOT NULL UNIQUE,
+                    index_path TEXT NOT NULL,
+                    files_count INTEGER DEFAULT 0,
+                    last_updated REAL
+                )
+            """)
+
+            cursor.execute("""
+                INSERT INTO subdirs_new (id, name, index_path, files_count, last_updated)
+                SELECT id, name, index_path, files_count, last_updated
+                FROM subdirs
+            """)
+
+            cursor.execute("DROP TABLE subdirs")
+            cursor.execute("ALTER TABLE subdirs_new RENAME TO subdirs")
+
+            # Recreate index
+            cursor.execute("CREATE INDEX IF NOT EXISTS idx_subdirs_name ON subdirs(name)")
+            log.info("Removed subdirs.direct_files column")
+        else:
+            log.info("subdirs.direct_files column does not exist, skipping")
+    else:
+        log.info("subdirs table does not exist, skipping")
+
+    log.info("Migration 005 completed successfully")
+
+    # Vacuum to reclaim space (outside transaction, optional)
+    # Note: VACUUM cannot run inside a transaction, so we skip it here
+    # The caller can run VACUUM separately if desired
+
+
+def downgrade(db_conn: Connection):
+    """Restore removed fields (data will be lost for keywords, token_count, symbol_type, direct_files).
+
+    This is a placeholder - true downgrade is not feasible as data is lost.
+    The migration is designed to be one-way since removed fields are unused/redundant.
+
+    Args:
+        db_conn: The SQLite database connection.
+    """
+    log.warning(
+        "Migration 005 downgrade not supported - removed fields are unused/redundant. "
+        "Data cannot be restored."
+    )
+    raise NotImplementedError(
+        "Migration 005 downgrade not supported - this is a one-way migration"
+    )

package/codex-lens/src/codexlens/storage/path_mapper.py

@@ -0,0 +1,274 @@
+"""Path mapping utilities for source paths and index paths.
+
+This module provides bidirectional mapping between source code directories
+and their corresponding index storage locations.
+
+Storage Structure:
+    ~/.codexlens/
+    ├── registry.db                    # Global mapping table
+    └── indexes/
+        └── D/
+            └── Claude_dms3/
+                ├── _index.db          # Root directory index
+                └── src/
+                    └── _index.db      # src/ directory index
+"""
+
+import platform
+from pathlib import Path
+from typing import Optional
+
+
+class PathMapper:
+    """Bidirectional mapping tool for source paths ↔ index paths.
+
+    Handles cross-platform path normalization and conversion between
+    source code directories and their index storage locations.
+
+    Attributes:
+        DEFAULT_INDEX_ROOT: Default root directory for all indexes
+        INDEX_DB_NAME: Standard name for index database files
+        index_root: Configured index root directory
+    """
+
+    DEFAULT_INDEX_ROOT = Path.home() / ".codexlens" / "indexes"
+    INDEX_DB_NAME = "_index.db"
+
+    def __init__(self, index_root: Optional[Path] = None):
+        """Initialize PathMapper with optional custom index root.
+
+        Args:
+            index_root: Custom index root directory. If None, uses DEFAULT_INDEX_ROOT.
+        """
+        self.index_root = (index_root or self.DEFAULT_INDEX_ROOT).resolve()
+
+    def source_to_index_dir(self, source_path: Path) -> Path:
+        """Convert source directory to its index directory path.
+
+        Maps a source code directory to where its index data should be stored.
+        The mapping preserves the directory structure but normalizes paths
+        for cross-platform compatibility.
+
+        Args:
+            source_path: Source directory path to map
+
+        Returns:
+            Index directory path under index_root
+
+        Examples:
+            >>> mapper = PathMapper()
+            >>> mapper.source_to_index_dir(Path("D:/Claude_dms3/src"))
+            PosixPath('/home/user/.codexlens/indexes/D/Claude_dms3/src')
+
+            >>> mapper.source_to_index_dir(Path("/home/user/project"))
+            PosixPath('/home/user/.codexlens/indexes/home/user/project')
+        """
+        source_path = source_path.resolve()
+        normalized = self.normalize_path(source_path)
+        return self.index_root / normalized
+
+    def source_to_index_db(self, source_path: Path) -> Path:
+        """Convert source directory to its index database file path.
+
+        Maps a source directory to the full path of its index database file,
+        including the standard INDEX_DB_NAME.
+
+        Args:
+            source_path: Source directory path to map
+
+        Returns:
+            Full path to the index database file
+
+        Examples:
+            >>> mapper = PathMapper()
+            >>> mapper.source_to_index_db(Path("D:/Claude_dms3/src"))
+            PosixPath('/home/user/.codexlens/indexes/D/Claude_dms3/src/_index.db')
+        """
+        index_dir = self.source_to_index_dir(source_path)
+        return index_dir / self.INDEX_DB_NAME
+
+    def index_to_source(self, index_path: Path) -> Path:
+        """Convert index path back to original source path.
+
+        Performs reverse mapping from an index storage location to the
+        original source directory. Handles both directory paths and
+        database file paths.
+
+        Args:
+            index_path: Index directory or database file path
+
+        Returns:
+            Original source directory path
+
+        Raises:
+            ValueError: If index_path is not under index_root
+
+        Examples:
+            >>> mapper = PathMapper()
+            >>> mapper.index_to_source(
+            ...     Path("~/.codexlens/indexes/D/Claude_dms3/src/_index.db")
+            ... )
+            WindowsPath('D:/Claude_dms3/src')
+
+            >>> mapper.index_to_source(
+            ...     Path("~/.codexlens/indexes/D/Claude_dms3/src")
+            ... )
+            WindowsPath('D:/Claude_dms3/src')
+        """
+        index_path = index_path.resolve()
+
+        # Remove _index.db if present
+        if index_path.name == self.INDEX_DB_NAME:
+            index_path = index_path.parent
+
+        # Verify path is under index_root
+        try:
+            relative = index_path.relative_to(self.index_root)
+        except ValueError:
+            raise ValueError(
+                f"Index path {index_path} is not under index root {self.index_root}"
+            )
+
+        # Convert normalized path back to source path
+        normalized_str = str(relative).replace("\\", "/")
+        return self.denormalize_path(normalized_str)
+
+    def get_project_root(self, source_path: Path) -> Path:
+        """Find the project root directory (topmost indexed directory).
+
+        Walks up the directory tree to find the highest-level directory
+        that has an index database.
+
+        Args:
+            source_path: Source directory to start from
+
+        Returns:
+            Project root directory path. Returns source_path itself if
+            no parent index is found.
+
+        Examples:
+            >>> mapper = PathMapper()
+            >>> mapper.get_project_root(Path("D:/Claude_dms3/src/codexlens"))
+            WindowsPath('D:/Claude_dms3')
+        """
+        source_path = source_path.resolve()
+        current = source_path
+        project_root = source_path
+
+        # Walk up the tree
+        while current.parent != current:  # Stop at filesystem root
+            parent_index_db = self.source_to_index_db(current.parent)
+            if parent_index_db.exists():
+                project_root = current.parent
+                current = current.parent
+            else:
+                break
+
+        return project_root
+
+    def get_relative_depth(self, source_path: Path, project_root: Path) -> int:
+        """Calculate directory depth relative to project root.
+
+        Args:
+            source_path: Target directory path
+            project_root: Project root directory path
+
+        Returns:
+            Number of directory levels from project_root to source_path
+
+        Raises:
+            ValueError: If source_path is not under project_root
+
+        Examples:
+            >>> mapper = PathMapper()
+            >>> mapper.get_relative_depth(
+            ...     Path("D:/Claude_dms3/src/codexlens"),
+            ...     Path("D:/Claude_dms3")
+            ... )
+            2
+        """
+        source_path = source_path.resolve()
+        project_root = project_root.resolve()
+
+        try:
+            relative = source_path.relative_to(project_root)
+            # Count path components
+            return len(relative.parts)
+        except ValueError:
+            raise ValueError(
+                f"Source path {source_path} is not under project root {project_root}"
+            )
+
+    def normalize_path(self, path: Path) -> str:
+        """Normalize path to cross-platform storage format.
+
+        Converts OS-specific paths to a standardized format for storage:
+        - Windows: Removes drive colons (D: → D)
+        - Unix: Removes leading slash
+        - Uses forward slashes throughout
+
+        Args:
+            path: Path to normalize
+
+        Returns:
+            Normalized path string
+
+        Examples:
+            >>> mapper = PathMapper()
+            >>> mapper.normalize_path(Path("D:/path/to/dir"))
+            'D/path/to/dir'
+
+            >>> mapper.normalize_path(Path("/home/user/path"))
+            'home/user/path'
+        """
+        path = path.resolve()
+        path_str = str(path)
+
+        # Handle Windows paths with drive letters
+        if platform.system() == "Windows" and len(path.parts) > 0:
+            # Convert D:\path\to\dir → D/path/to/dir
+            drive = path.parts[0].replace(":", "")  # D: → D
+            rest = Path(*path.parts[1:]) if len(path.parts) > 1 else Path()
+            normalized = f"{drive}/{rest}".replace("\\", "/")
+            return normalized.rstrip("/")
+
+        # Handle Unix paths
+        # /home/user/path → home/user/path
+        return path_str.lstrip("/").replace("\\", "/")
+
+    def denormalize_path(self, normalized: str) -> Path:
+        """Convert normalized path back to OS-specific path.
+
+        Reverses the normalization process to restore OS-native path format:
+        - Windows: Adds drive colons (D → D:)
+        - Unix: Adds leading slash
+
+        Args:
+            normalized: Normalized path string
+
+        Returns:
+            OS-specific Path object
+
+        Examples:
+            >>> mapper = PathMapper()
+            >>> mapper.denormalize_path("D/path/to/dir")  # On Windows
+            WindowsPath('D:/path/to/dir')
+
+            >>> mapper.denormalize_path("home/user/path")  # On Unix
+            PosixPath('/home/user/path')
+        """
+        parts = normalized.split("/")
+
+        # Handle Windows paths
+        if platform.system() == "Windows" and len(parts) > 0:
+            # Check if first part is a drive letter
+            if len(parts[0]) == 1 and parts[0].isalpha():
+                # D/path/to/dir → D:/path/to/dir
+                drive = f"{parts[0]}:"
+                if len(parts) > 1:
+                    return Path(drive) / Path(*parts[1:])
+                return Path(drive)
+
+        # Handle Unix paths or relative paths
+        # home/user/path → /home/user/path
+        return Path("/") / Path(*parts)