claude-code-workflow 6.2.4 → 6.2.5

package/codex-lens/src/codexlens/storage/migration_manager.py

@@ -1,154 +0,0 @@
-"""
-Manages database schema migrations.
-
-This module provides a framework for applying versioned migrations to the SQLite
-database. Migrations are discovered from the `codexlens.storage.migrations`
-package and applied sequentially. The database schema version is tracked using
-the `user_version` pragma.
-"""
-
-import importlib
-import logging
-import pkgutil
-from pathlib import Path
-from sqlite3 import Connection
-from typing import List, NamedTuple
-
-log = logging.getLogger(__name__)
-
-
-class Migration(NamedTuple):
-    """Represents a single database migration."""
-
-    version: int
-    name: str
-    upgrade: callable
-
-
-def discover_migrations() -> List[Migration]:
-    """
-    Discovers and returns a sorted list of database migrations.
-
-    Migrations are expected to be in the `codexlens.storage.migrations` package,
-    with filenames in the format `migration_XXX_description.py`, where XXX is
-    the version number. Each migration module must contain an `upgrade` function
-    that takes a `sqlite3.Connection` object as its argument.
-
-    Returns:
-        A list of Migration objects, sorted by version.
-    """
-    import codexlens.storage.migrations
-
-    migrations = []
-    package_path = Path(codexlens.storage.migrations.__file__).parent
-    
-    for _, name, _ in pkgutil.iter_modules([str(package_path)]):
-        if name.startswith("migration_"):
-            try:
-                version = int(name.split("_")[1])
-                module = importlib.import_module(f"codexlens.storage.migrations.{name}")
-                if hasattr(module, "upgrade"):
-                    migrations.append(
-                        Migration(version=version, name=name, upgrade=module.upgrade)
-                    )
-                else:
-                    log.warning(f"Migration {name} is missing 'upgrade' function.")
-            except (ValueError, IndexError) as e:
-                log.warning(f"Could not parse migration name {name}: {e}")
-            except ImportError as e:
-                log.warning(f"Could not import migration {name}: {e}")
-
-    migrations.sort(key=lambda m: m.version)
-    return migrations
-
-
-class MigrationManager:
-    """
-    Manages the application of migrations to a database.
-    """
-
-    def __init__(self, db_conn: Connection):
-        """
-        Initializes the MigrationManager.
-
-        Args:
-            db_conn: The SQLite database connection.
-        """
-        self.db_conn = db_conn
-        self.migrations = discover_migrations()
-
-    def get_current_version(self) -> int:
-        """
-        Gets the current version of the database schema.
-
-        Returns:
-            The current schema version number.
-        """
-        return self.db_conn.execute("PRAGMA user_version").fetchone()[0]
-
-    def set_version(self, version: int):
-        """
-        Sets the database schema version.
-
-        Args:
-            version: The version number to set.
-        """
-        self.db_conn.execute(f"PRAGMA user_version = {version}")
-        log.info(f"Database schema version set to {version}")
-
-    def apply_migrations(self):
-        """
-        Applies all pending migrations to the database.
-
-        This method checks the current database version and applies all
-        subsequent migrations in order. Each migration is applied within
-        a transaction, unless the migration manages its own transactions.
-        """
-        current_version = self.get_current_version()
-        log.info(f"Current database schema version: {current_version}")
-
-        for migration in self.migrations:
-            if migration.version > current_version:
-                log.info(f"Applying migration {migration.version}: {migration.name}...")
-                try:
-                    # Check if a transaction is already in progress
-                    in_transaction = self.db_conn.in_transaction
-
-                    # Only start transaction if not already in one
-                    if not in_transaction:
-                        self.db_conn.execute("BEGIN")
-
-                    migration.upgrade(self.db_conn)
-                    self.set_version(migration.version)
-
-                    # Only commit if we started the transaction and it's still active
-                    if not in_transaction and self.db_conn.in_transaction:
-                        self.db_conn.execute("COMMIT")
-
-                    log.info(
-                        f"Successfully applied migration {migration.version}: {migration.name}"
-                    )
-                except Exception as e:
-                    log.error(
-                        f"Failed to apply migration {migration.version}: {migration.name}. Error: {e}",
-                        exc_info=True,
-                    )
-                    # Try to rollback if transaction is active
-                    try:
-                        if self.db_conn.in_transaction:
-                            self.db_conn.execute("ROLLBACK")
-                    except Exception:
-                        pass  # Ignore rollback errors
-                    raise
-
-        latest_migration_version = self.migrations[-1].version if self.migrations else 0
-        if current_version < latest_migration_version:
-            # This case can be hit if migrations were applied but the loop was exited
-            # and set_version was not called for the last one for some reason.
-            # To be safe, we explicitly set the version to the latest known migration.
-            final_version = self.get_current_version()
-            if final_version != latest_migration_version:
-                 log.warning(f"Database version ({final_version}) is not the latest migration version ({latest_migration_version}). This may indicate a problem.")
-
-        log.info("All pending migrations applied successfully.")
-

package/codex-lens/src/codexlens/storage/migrations/__init__.py

@@ -1 +0,0 @@
-# This file makes the 'migrations' directory a Python package.

package/codex-lens/src/codexlens/storage/migrations/migration_001_normalize_keywords.py

@@ -1,123 +0,0 @@
-"""
-Migration 001: Normalize keywords into separate tables.
-
-This migration introduces two new tables, `keywords` and `file_keywords`, to
-store semantic keywords in a normalized fashion. It then migrates the existing
-keywords from the `semantic_data` JSON blob in the `files` table into these
-new tables. This is intended to speed up keyword-based searches significantly.
-"""
-
-import json
-import logging
-from sqlite3 import Connection
-
-log = logging.getLogger(__name__)
-
-
-def upgrade(db_conn: Connection):
-    """
-    Applies the migration to normalize keywords.
-
-    - Creates `keywords` and `file_keywords` tables.
-    - Creates indexes for efficient querying.
-    - Migrates data from `files.semantic_data` to the new tables.
-
-    Args:
-        db_conn: The SQLite database connection.
-    """
-    cursor = db_conn.cursor()
-
-    log.info("Creating 'keywords' and 'file_keywords' tables...")
-    # Create a table to store unique keywords
-    cursor.execute(
-        """
-        CREATE TABLE IF NOT EXISTS keywords (
-            id INTEGER PRIMARY KEY,
-            keyword TEXT NOT NULL UNIQUE
-        )
-        """
-    )
-
-    # Create a join table to link files and keywords (many-to-many)
-    cursor.execute(
-        """
-        CREATE TABLE IF NOT EXISTS file_keywords (
-            file_id INTEGER NOT NULL,
-            keyword_id INTEGER NOT NULL,
-            PRIMARY KEY (file_id, keyword_id),
-            FOREIGN KEY (file_id) REFERENCES files (id) ON DELETE CASCADE,
-            FOREIGN KEY (keyword_id) REFERENCES keywords (id) ON DELETE CASCADE
-        )
-        """
-    )
-    
-    log.info("Creating indexes for new keyword tables...")
-    cursor.execute("CREATE INDEX IF NOT EXISTS idx_keywords_keyword ON keywords (keyword)")
-    cursor.execute("CREATE INDEX IF NOT EXISTS idx_file_keywords_file_id ON file_keywords (file_id)")
-    cursor.execute("CREATE INDEX IF NOT EXISTS idx_file_keywords_keyword_id ON file_keywords (keyword_id)")
-
-    log.info("Migrating existing keywords from 'semantic_metadata' table...")
-
-    # Check if semantic_metadata table exists before querying
-    cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='semantic_metadata'")
-    if not cursor.fetchone():
-        log.info("No 'semantic_metadata' table found, skipping data migration.")
-        return
-
-    # Check if 'keywords' column exists in semantic_metadata table
-    # (current schema may already use normalized tables without this column)
-    cursor.execute("PRAGMA table_info(semantic_metadata)")
-    columns = {row[1] for row in cursor.fetchall()}
-    if "keywords" not in columns:
-        log.info("No 'keywords' column in semantic_metadata table, skipping data migration.")
-        return
-
-    cursor.execute("SELECT file_id, keywords FROM semantic_metadata WHERE keywords IS NOT NULL AND keywords != ''")
-
-    files_to_migrate = cursor.fetchall()
-    if not files_to_migrate:
-        log.info("No existing files with semantic metadata to migrate.")
-        return
-
-    log.info(f"Found {len(files_to_migrate)} files with semantic metadata to migrate.")
-
-    for file_id, keywords_json in files_to_migrate:
-        if not keywords_json:
-            continue
-        try:
-            keywords = json.loads(keywords_json)
-
-            if not isinstance(keywords, list):
-                log.warning(f"Keywords for file_id {file_id} is not a list, skipping.")
-                continue
-
-            for keyword in keywords:
-                if not isinstance(keyword, str):
-                    log.warning(f"Non-string keyword '{keyword}' found for file_id {file_id}, skipping.")
-                    continue
-
-                keyword = keyword.strip()
-                if not keyword:
-                    continue
-
-                # Get or create keyword_id
-                cursor.execute("INSERT OR IGNORE INTO keywords (keyword) VALUES (?)", (keyword,))
-                cursor.execute("SELECT id FROM keywords WHERE keyword = ?", (keyword,))
-                keyword_id_result = cursor.fetchone()
-
-                if keyword_id_result:
-                    keyword_id = keyword_id_result[0]
-                    # Link file to keyword
-                    cursor.execute(
-                        "INSERT OR IGNORE INTO file_keywords (file_id, keyword_id) VALUES (?, ?)",
-                        (file_id, keyword_id),
-                    )
-                else:
-                    log.error(f"Failed to retrieve or create keyword_id for keyword: {keyword}")
-
-        except json.JSONDecodeError as e:
-            log.warning(f"Could not parse keywords for file_id {file_id}: {e}")
-        except Exception as e:
-            log.error(f"An unexpected error occurred during migration for file_id {file_id}: {e}", exc_info=True)
-
-    log.info("Finished migrating keywords.")

package/codex-lens/src/codexlens/storage/migrations/migration_002_add_token_metadata.py

@@ -1,48 +0,0 @@
-"""
-Migration 002: Add token_count and symbol_type to symbols table.
-
-This migration adds token counting metadata to symbols for accurate chunk
-splitting and performance optimization. It also adds symbol_type for better
-filtering in searches.
-"""
-
-import logging
-from sqlite3 import Connection
-
-log = logging.getLogger(__name__)
-
-
-def upgrade(db_conn: Connection):
-    """
-    Applies the migration to add token metadata to symbols.
-
-    - Adds token_count column to symbols table
-    - Adds symbol_type column to symbols table (for future use)
-    - Creates index on symbol_type for efficient filtering
-    - Backfills existing symbols with NULL token_count (to be calculated lazily)
-
-    Args:
-        db_conn: The SQLite database connection.
-    """
-    cursor = db_conn.cursor()
-
-    log.info("Adding token_count column to symbols table...")
-    try:
-        cursor.execute("ALTER TABLE symbols ADD COLUMN token_count INTEGER")
-        log.info("Successfully added token_count column.")
-    except Exception as e:
-        # Column might already exist
-        log.warning(f"Could not add token_count column (might already exist): {e}")
-
-    log.info("Adding symbol_type column to symbols table...")
-    try:
-        cursor.execute("ALTER TABLE symbols ADD COLUMN symbol_type TEXT")
-        log.info("Successfully added symbol_type column.")
-    except Exception as e:
-        # Column might already exist
-        log.warning(f"Could not add symbol_type column (might already exist): {e}")
-
-    log.info("Creating index on symbol_type for efficient filtering...")
-    cursor.execute("CREATE INDEX IF NOT EXISTS idx_symbols_type ON symbols(symbol_type)")
-
-    log.info("Migration 002 completed successfully.")

package/codex-lens/src/codexlens/storage/migrations/migration_004_dual_fts.py

@@ -1,232 +0,0 @@
-"""
-Migration 004: Add dual FTS tables for exact and fuzzy matching.
-
-This migration introduces two FTS5 tables:
-- files_fts_exact: Uses unicode61 tokenizer for exact token matching
-- files_fts_fuzzy: Uses trigram tokenizer (or extended unicode61) for substring/fuzzy matching
-
-Both tables are synchronized with the files table via triggers for automatic updates.
-"""
-
-import logging
-from sqlite3 import Connection
-
-from codexlens.storage.sqlite_utils import check_trigram_support, get_sqlite_version
-
-log = logging.getLogger(__name__)
-
-
-def upgrade(db_conn: Connection):
-    """
-    Applies the migration to add dual FTS tables.
-
-    - Drops old files_fts table and triggers
-    - Creates files_fts_exact with unicode61 tokenizer
-    - Creates files_fts_fuzzy with trigram or extended unicode61 tokenizer
-    - Creates synchronized triggers for both tables
-    - Rebuilds FTS indexes from files table
-
-    Args:
-        db_conn: The SQLite database connection.
-    """
-    cursor = db_conn.cursor()
-
-    try:
-        # Check trigram support
-        has_trigram = check_trigram_support(db_conn)
-        version = get_sqlite_version(db_conn)
-        log.info(f"SQLite version: {'.'.join(map(str, version))}")
-
-        if has_trigram:
-            log.info("Trigram tokenizer available, using for fuzzy FTS table")
-            fuzzy_tokenizer = "trigram"
-        else:
-            log.warning(
-                f"Trigram tokenizer not available (requires SQLite >= 3.34), "
-                f"using extended unicode61 tokenizer for fuzzy matching"
-            )
-            fuzzy_tokenizer = "unicode61 tokenchars '_-.'"
-
-        # Start transaction
-        cursor.execute("BEGIN TRANSACTION")
-
-        # Check if files table has 'name' column (v2 schema doesn't have it)
-        cursor.execute("PRAGMA table_info(files)")
-        columns = {row[1] for row in cursor.fetchall()}
-        
-        if 'name' not in columns:
-            log.info("Adding 'name' column to files table (v2 schema upgrade)...")
-            # Add name column
-            cursor.execute("ALTER TABLE files ADD COLUMN name TEXT")
-            # Populate name from path (extract filename from last '/')
-            # Use Python to do the extraction since SQLite doesn't have reverse()
-            cursor.execute("SELECT rowid, path FROM files")
-            rows = cursor.fetchall()
-            for rowid, path in rows:
-                # Extract filename from path
-                name = path.split('/')[-1] if '/' in path else path
-                cursor.execute("UPDATE files SET name = ? WHERE rowid = ?", (name, rowid))
-            
-        # Rename 'path' column to 'full_path' if needed
-        if 'path' in columns and 'full_path' not in columns:
-            log.info("Renaming 'path' to 'full_path' (v2 schema upgrade)...")
-            # Check if indexed_at column exists in v2 schema
-            has_indexed_at = 'indexed_at' in columns
-            has_mtime = 'mtime' in columns
-            
-            # SQLite doesn't support RENAME COLUMN before 3.25, so use table recreation
-            cursor.execute("""
-                CREATE TABLE files_new (
-                    id INTEGER PRIMARY KEY AUTOINCREMENT,
-                    name TEXT NOT NULL,
-                    full_path TEXT NOT NULL UNIQUE,
-                    content TEXT,
-                    language TEXT,
-                    mtime REAL,
-                    indexed_at TEXT
-                )
-            """)
-            
-            # Build INSERT statement based on available columns
-            # Note: v2 schema has no rowid (path is PRIMARY KEY), so use NULL for AUTOINCREMENT
-            if has_indexed_at and has_mtime:
-                cursor.execute("""
-                    INSERT INTO files_new (name, full_path, content, language, mtime, indexed_at)
-                    SELECT name, path, content, language, mtime, indexed_at FROM files
-                """)
-            elif has_indexed_at:
-                cursor.execute("""
-                    INSERT INTO files_new (name, full_path, content, language, indexed_at)
-                    SELECT name, path, content, language, indexed_at FROM files
-                """)
-            elif has_mtime:
-                cursor.execute("""
-                    INSERT INTO files_new (name, full_path, content, language, mtime)
-                    SELECT name, path, content, language, mtime FROM files
-                """)
-            else:
-                cursor.execute("""
-                    INSERT INTO files_new (name, full_path, content, language)
-                    SELECT name, path, content, language FROM files
-                """)
-            
-            cursor.execute("DROP TABLE files")
-            cursor.execute("ALTER TABLE files_new RENAME TO files")
-
-        log.info("Dropping old FTS triggers and table...")
-        # Drop old triggers
-        cursor.execute("DROP TRIGGER IF EXISTS files_ai")
-        cursor.execute("DROP TRIGGER IF EXISTS files_ad")
-        cursor.execute("DROP TRIGGER IF EXISTS files_au")
-
-        # Drop old FTS table
-        cursor.execute("DROP TABLE IF EXISTS files_fts")
-
-        # Create exact FTS table (unicode61 with underscores/hyphens/dots as token chars)
-        # Note: tokenchars includes '.' to properly tokenize qualified names like PortRole.FLOW
-        log.info("Creating files_fts_exact table with unicode61 tokenizer...")
-        cursor.execute(
-            """
-            CREATE VIRTUAL TABLE files_fts_exact USING fts5(
-                name, full_path UNINDEXED, content,
-                content='files',
-                content_rowid='id',
-                tokenize="unicode61 tokenchars '_-.'"
-            )
-            """
-        )
-
-        # Create fuzzy FTS table (trigram or extended unicode61)
-        log.info(f"Creating files_fts_fuzzy table with {fuzzy_tokenizer} tokenizer...")
-        cursor.execute(
-            f"""
-            CREATE VIRTUAL TABLE files_fts_fuzzy USING fts5(
-                name, full_path UNINDEXED, content,
-                content='files',
-                content_rowid='id',
-                tokenize="{fuzzy_tokenizer}"
-            )
-            """
-        )
-
-        # Create synchronized triggers for files_fts_exact
-        log.info("Creating triggers for files_fts_exact...")
-        cursor.execute(
-            """
-            CREATE TRIGGER files_exact_ai AFTER INSERT ON files BEGIN
-                INSERT INTO files_fts_exact(rowid, name, full_path, content)
-                VALUES(new.id, new.name, new.full_path, new.content);
-            END
-            """
-        )
-        cursor.execute(
-            """
-            CREATE TRIGGER files_exact_ad AFTER DELETE ON files BEGIN
-                INSERT INTO files_fts_exact(files_fts_exact, rowid, name, full_path, content)
-                VALUES('delete', old.id, old.name, old.full_path, old.content);
-            END
-            """
-        )
-        cursor.execute(
-            """
-            CREATE TRIGGER files_exact_au AFTER UPDATE ON files BEGIN
-                INSERT INTO files_fts_exact(files_fts_exact, rowid, name, full_path, content)
-                VALUES('delete', old.id, old.name, old.full_path, old.content);
-                INSERT INTO files_fts_exact(rowid, name, full_path, content)
-                VALUES(new.id, new.name, new.full_path, new.content);
-            END
-            """
-        )
-
-        # Create synchronized triggers for files_fts_fuzzy
-        log.info("Creating triggers for files_fts_fuzzy...")
-        cursor.execute(
-            """
-            CREATE TRIGGER files_fuzzy_ai AFTER INSERT ON files BEGIN
-                INSERT INTO files_fts_fuzzy(rowid, name, full_path, content)
-                VALUES(new.id, new.name, new.full_path, new.content);
-            END
-            """
-        )
-        cursor.execute(
-            """
-            CREATE TRIGGER files_fuzzy_ad AFTER DELETE ON files BEGIN
-                INSERT INTO files_fts_fuzzy(files_fts_fuzzy, rowid, name, full_path, content)
-                VALUES('delete', old.id, old.name, old.full_path, old.content);
-            END
-            """
-        )
-        cursor.execute(
-            """
-            CREATE TRIGGER files_fuzzy_au AFTER UPDATE ON files BEGIN
-                INSERT INTO files_fts_fuzzy(files_fts_fuzzy, rowid, name, full_path, content)
-                VALUES('delete', old.id, old.name, old.full_path, old.content);
-                INSERT INTO files_fts_fuzzy(rowid, name, full_path, content)
-                VALUES(new.id, new.name, new.full_path, new.content);
-            END
-            """
-        )
-
-        # Rebuild FTS indexes from files table
-        log.info("Rebuilding FTS indexes from files table...")
-        cursor.execute("INSERT INTO files_fts_exact(files_fts_exact) VALUES('rebuild')")
-        cursor.execute("INSERT INTO files_fts_fuzzy(files_fts_fuzzy) VALUES('rebuild')")
-
-        # Commit transaction
-        cursor.execute("COMMIT")
-        log.info("Migration 004 completed successfully")
-
-        # Vacuum to reclaim space (outside transaction)
-        try:
-            log.info("Running VACUUM to reclaim space...")
-            cursor.execute("VACUUM")
-        except Exception as e:
-            log.warning(f"VACUUM failed (non-critical): {e}")
-
-    except Exception as e:
-        log.error(f"Migration 004 failed: {e}")
-        try:
-            cursor.execute("ROLLBACK")
-        except Exception:
-            pass
-        raise