claude-code-workflow 6.1.4 → 6.2.2

package/codex-lens/src/codexlens/storage/migration_manager.py

@@ -0,0 +1,154 @@
+"""
+Manages database schema migrations.
+
+This module provides a framework for applying versioned migrations to the SQLite
+database. Migrations are discovered from the `codexlens.storage.migrations`
+package and applied sequentially. The database schema version is tracked using
+the `user_version` pragma.
+"""
+
+import importlib
+import logging
+import pkgutil
+from pathlib import Path
+from sqlite3 import Connection
+from typing import List, NamedTuple
+
+log = logging.getLogger(__name__)
+
+
+class Migration(NamedTuple):
+    """Represents a single database migration."""
+
+    version: int
+    name: str
+    upgrade: callable
+
+
+def discover_migrations() -> List[Migration]:
+    """
+    Discovers and returns a sorted list of database migrations.
+
+    Migrations are expected to be in the `codexlens.storage.migrations` package,
+    with filenames in the format `migration_XXX_description.py`, where XXX is
+    the version number. Each migration module must contain an `upgrade` function
+    that takes a `sqlite3.Connection` object as its argument.
+
+    Returns:
+        A list of Migration objects, sorted by version.
+    """
+    import codexlens.storage.migrations
+
+    migrations = []
+    package_path = Path(codexlens.storage.migrations.__file__).parent
+    
+    for _, name, _ in pkgutil.iter_modules([str(package_path)]):
+        if name.startswith("migration_"):
+            try:
+                version = int(name.split("_")[1])
+                module = importlib.import_module(f"codexlens.storage.migrations.{name}")
+                if hasattr(module, "upgrade"):
+                    migrations.append(
+                        Migration(version=version, name=name, upgrade=module.upgrade)
+                    )
+                else:
+                    log.warning(f"Migration {name} is missing 'upgrade' function.")
+            except (ValueError, IndexError) as e:
+                log.warning(f"Could not parse migration name {name}: {e}")
+            except ImportError as e:
+                log.warning(f"Could not import migration {name}: {e}")
+
+    migrations.sort(key=lambda m: m.version)
+    return migrations
+
+
+class MigrationManager:
+    """
+    Manages the application of migrations to a database.
+    """
+
+    def __init__(self, db_conn: Connection):
+        """
+        Initializes the MigrationManager.
+
+        Args:
+            db_conn: The SQLite database connection.
+        """
+        self.db_conn = db_conn
+        self.migrations = discover_migrations()
+
+    def get_current_version(self) -> int:
+        """
+        Gets the current version of the database schema.
+
+        Returns:
+            The current schema version number.
+        """
+        return self.db_conn.execute("PRAGMA user_version").fetchone()[0]
+
+    def set_version(self, version: int):
+        """
+        Sets the database schema version.
+
+        Args:
+            version: The version number to set.
+        """
+        self.db_conn.execute(f"PRAGMA user_version = {version}")
+        log.info(f"Database schema version set to {version}")
+
+    def apply_migrations(self):
+        """
+        Applies all pending migrations to the database.
+
+        This method checks the current database version and applies all
+        subsequent migrations in order. Each migration is applied within
+        a transaction, unless the migration manages its own transactions.
+        """
+        current_version = self.get_current_version()
+        log.info(f"Current database schema version: {current_version}")
+
+        for migration in self.migrations:
+            if migration.version > current_version:
+                log.info(f"Applying migration {migration.version}: {migration.name}...")
+                try:
+                    # Check if a transaction is already in progress
+                    in_transaction = self.db_conn.in_transaction
+
+                    # Only start transaction if not already in one
+                    if not in_transaction:
+                        self.db_conn.execute("BEGIN")
+
+                    migration.upgrade(self.db_conn)
+                    self.set_version(migration.version)
+
+                    # Only commit if we started the transaction and it's still active
+                    if not in_transaction and self.db_conn.in_transaction:
+                        self.db_conn.execute("COMMIT")
+
+                    log.info(
+                        f"Successfully applied migration {migration.version}: {migration.name}"
+                    )
+                except Exception as e:
+                    log.error(
+                        f"Failed to apply migration {migration.version}: {migration.name}. Error: {e}",
+                        exc_info=True,
+                    )
+                    # Try to rollback if transaction is active
+                    try:
+                        if self.db_conn.in_transaction:
+                            self.db_conn.execute("ROLLBACK")
+                    except Exception:
+                        pass  # Ignore rollback errors
+                    raise
+
+        latest_migration_version = self.migrations[-1].version if self.migrations else 0
+        if current_version < latest_migration_version:
+            # This case can be hit if migrations were applied but the loop was exited
+            # and set_version was not called for the last one for some reason.
+            # To be safe, we explicitly set the version to the latest known migration.
+            final_version = self.get_current_version()
+            if final_version != latest_migration_version:
+                 log.warning(f"Database version ({final_version}) is not the latest migration version ({latest_migration_version}). This may indicate a problem.")
+
+        log.info("All pending migrations applied successfully.")
+

package/codex-lens/src/codexlens/storage/migrations/__init__.py

@@ -0,0 +1 @@
+# This file makes the 'migrations' directory a Python package.

package/codex-lens/src/codexlens/storage/migrations/migration_001_normalize_keywords.py

@@ -0,0 +1,123 @@
+"""
+Migration 001: Normalize keywords into separate tables.
+
+This migration introduces two new tables, `keywords` and `file_keywords`, to
+store semantic keywords in a normalized fashion. It then migrates the existing
+keywords from the `semantic_data` JSON blob in the `files` table into these
+new tables. This is intended to speed up keyword-based searches significantly.
+"""
+
+import json
+import logging
+from sqlite3 import Connection
+
+log = logging.getLogger(__name__)
+
+
+def upgrade(db_conn: Connection):
+    """
+    Applies the migration to normalize keywords.
+
+    - Creates `keywords` and `file_keywords` tables.
+    - Creates indexes for efficient querying.
+    - Migrates data from `files.semantic_data` to the new tables.
+
+    Args:
+        db_conn: The SQLite database connection.
+    """
+    cursor = db_conn.cursor()
+
+    log.info("Creating 'keywords' and 'file_keywords' tables...")
+    # Create a table to store unique keywords
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS keywords (
+            id INTEGER PRIMARY KEY,
+            keyword TEXT NOT NULL UNIQUE
+        )
+        """
+    )
+
+    # Create a join table to link files and keywords (many-to-many)
+    cursor.execute(
+        """
+        CREATE TABLE IF NOT EXISTS file_keywords (
+            file_id INTEGER NOT NULL,
+            keyword_id INTEGER NOT NULL,
+            PRIMARY KEY (file_id, keyword_id),
+            FOREIGN KEY (file_id) REFERENCES files (id) ON DELETE CASCADE,
+            FOREIGN KEY (keyword_id) REFERENCES keywords (id) ON DELETE CASCADE
+        )
+        """
+    )
+    
+    log.info("Creating indexes for new keyword tables...")
+    cursor.execute("CREATE INDEX IF NOT EXISTS idx_keywords_keyword ON keywords (keyword)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS idx_file_keywords_file_id ON file_keywords (file_id)")
+    cursor.execute("CREATE INDEX IF NOT EXISTS idx_file_keywords_keyword_id ON file_keywords (keyword_id)")
+
+    log.info("Migrating existing keywords from 'semantic_metadata' table...")
+
+    # Check if semantic_metadata table exists before querying
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='semantic_metadata'")
+    if not cursor.fetchone():
+        log.info("No 'semantic_metadata' table found, skipping data migration.")
+        return
+
+    # Check if 'keywords' column exists in semantic_metadata table
+    # (current schema may already use normalized tables without this column)
+    cursor.execute("PRAGMA table_info(semantic_metadata)")
+    columns = {row[1] for row in cursor.fetchall()}
+    if "keywords" not in columns:
+        log.info("No 'keywords' column in semantic_metadata table, skipping data migration.")
+        return
+
+    cursor.execute("SELECT file_id, keywords FROM semantic_metadata WHERE keywords IS NOT NULL AND keywords != ''")
+
+    files_to_migrate = cursor.fetchall()
+    if not files_to_migrate:
+        log.info("No existing files with semantic metadata to migrate.")
+        return
+
+    log.info(f"Found {len(files_to_migrate)} files with semantic metadata to migrate.")
+
+    for file_id, keywords_json in files_to_migrate:
+        if not keywords_json:
+            continue
+        try:
+            keywords = json.loads(keywords_json)
+
+            if not isinstance(keywords, list):
+                log.warning(f"Keywords for file_id {file_id} is not a list, skipping.")
+                continue
+
+            for keyword in keywords:
+                if not isinstance(keyword, str):
+                    log.warning(f"Non-string keyword '{keyword}' found for file_id {file_id}, skipping.")
+                    continue
+
+                keyword = keyword.strip()
+                if not keyword:
+                    continue
+
+                # Get or create keyword_id
+                cursor.execute("INSERT OR IGNORE INTO keywords (keyword) VALUES (?)", (keyword,))
+                cursor.execute("SELECT id FROM keywords WHERE keyword = ?", (keyword,))
+                keyword_id_result = cursor.fetchone()
+
+                if keyword_id_result:
+                    keyword_id = keyword_id_result[0]
+                    # Link file to keyword
+                    cursor.execute(
+                        "INSERT OR IGNORE INTO file_keywords (file_id, keyword_id) VALUES (?, ?)",
+                        (file_id, keyword_id),
+                    )
+                else:
+                    log.error(f"Failed to retrieve or create keyword_id for keyword: {keyword}")
+
+        except json.JSONDecodeError as e:
+            log.warning(f"Could not parse keywords for file_id {file_id}: {e}")
+        except Exception as e:
+            log.error(f"An unexpected error occurred during migration for file_id {file_id}: {e}", exc_info=True)
+
+    log.info("Finished migrating keywords.")

package/codex-lens/src/codexlens/storage/migrations/migration_002_add_token_metadata.py

@@ -0,0 +1,48 @@
+"""
+Migration 002: Add token_count and symbol_type to symbols table.
+
+This migration adds token counting metadata to symbols for accurate chunk
+splitting and performance optimization. It also adds symbol_type for better
+filtering in searches.
+"""
+
+import logging
+from sqlite3 import Connection
+
+log = logging.getLogger(__name__)
+
+
+def upgrade(db_conn: Connection):
+    """
+    Applies the migration to add token metadata to symbols.
+
+    - Adds token_count column to symbols table
+    - Adds symbol_type column to symbols table (for future use)
+    - Creates index on symbol_type for efficient filtering
+    - Backfills existing symbols with NULL token_count (to be calculated lazily)
+
+    Args:
+        db_conn: The SQLite database connection.
+    """
+    cursor = db_conn.cursor()
+
+    log.info("Adding token_count column to symbols table...")
+    try:
+        cursor.execute("ALTER TABLE symbols ADD COLUMN token_count INTEGER")
+        log.info("Successfully added token_count column.")
+    except Exception as e:
+        # Column might already exist
+        log.warning(f"Could not add token_count column (might already exist): {e}")
+
+    log.info("Adding symbol_type column to symbols table...")
+    try:
+        cursor.execute("ALTER TABLE symbols ADD COLUMN symbol_type TEXT")
+        log.info("Successfully added symbol_type column.")
+    except Exception as e:
+        # Column might already exist
+        log.warning(f"Could not add symbol_type column (might already exist): {e}")
+
+    log.info("Creating index on symbol_type for efficient filtering...")
+    cursor.execute("CREATE INDEX IF NOT EXISTS idx_symbols_type ON symbols(symbol_type)")
+
+    log.info("Migration 002 completed successfully.")

package/codex-lens/src/codexlens/storage/migrations/migration_004_dual_fts.py

@@ -0,0 +1,232 @@
+"""
+Migration 004: Add dual FTS tables for exact and fuzzy matching.
+
+This migration introduces two FTS5 tables:
+- files_fts_exact: Uses unicode61 tokenizer for exact token matching
+- files_fts_fuzzy: Uses trigram tokenizer (or extended unicode61) for substring/fuzzy matching
+
+Both tables are synchronized with the files table via triggers for automatic updates.
+"""
+
+import logging
+from sqlite3 import Connection
+
+from codexlens.storage.sqlite_utils import check_trigram_support, get_sqlite_version
+
+log = logging.getLogger(__name__)
+
+
+def upgrade(db_conn: Connection):
+    """
+    Applies the migration to add dual FTS tables.
+
+    - Drops old files_fts table and triggers
+    - Creates files_fts_exact with unicode61 tokenizer
+    - Creates files_fts_fuzzy with trigram or extended unicode61 tokenizer
+    - Creates synchronized triggers for both tables
+    - Rebuilds FTS indexes from files table
+
+    Args:
+        db_conn: The SQLite database connection.
+    """
+    cursor = db_conn.cursor()
+
+    try:
+        # Check trigram support
+        has_trigram = check_trigram_support(db_conn)
+        version = get_sqlite_version(db_conn)
+        log.info(f"SQLite version: {'.'.join(map(str, version))}")
+
+        if has_trigram:
+            log.info("Trigram tokenizer available, using for fuzzy FTS table")
+            fuzzy_tokenizer = "trigram"
+        else:
+            log.warning(
+                f"Trigram tokenizer not available (requires SQLite >= 3.34), "
+                f"using extended unicode61 tokenizer for fuzzy matching"
+            )
+            fuzzy_tokenizer = "unicode61 tokenchars '_-.'"
+
+        # Start transaction
+        cursor.execute("BEGIN TRANSACTION")
+
+        # Check if files table has 'name' column (v2 schema doesn't have it)
+        cursor.execute("PRAGMA table_info(files)")
+        columns = {row[1] for row in cursor.fetchall()}
+        
+        if 'name' not in columns:
+            log.info("Adding 'name' column to files table (v2 schema upgrade)...")
+            # Add name column
+            cursor.execute("ALTER TABLE files ADD COLUMN name TEXT")
+            # Populate name from path (extract filename from last '/')
+            # Use Python to do the extraction since SQLite doesn't have reverse()
+            cursor.execute("SELECT rowid, path FROM files")
+            rows = cursor.fetchall()
+            for rowid, path in rows:
+                # Extract filename from path
+                name = path.split('/')[-1] if '/' in path else path
+                cursor.execute("UPDATE files SET name = ? WHERE rowid = ?", (name, rowid))
+            
+        # Rename 'path' column to 'full_path' if needed
+        if 'path' in columns and 'full_path' not in columns:
+            log.info("Renaming 'path' to 'full_path' (v2 schema upgrade)...")
+            # Check if indexed_at column exists in v2 schema
+            has_indexed_at = 'indexed_at' in columns
+            has_mtime = 'mtime' in columns
+            
+            # SQLite doesn't support RENAME COLUMN before 3.25, so use table recreation
+            cursor.execute("""
+                CREATE TABLE files_new (
+                    id INTEGER PRIMARY KEY AUTOINCREMENT,
+                    name TEXT NOT NULL,
+                    full_path TEXT NOT NULL UNIQUE,
+                    content TEXT,
+                    language TEXT,
+                    mtime REAL,
+                    indexed_at TEXT
+                )
+            """)
+            
+            # Build INSERT statement based on available columns
+            # Note: v2 schema has no rowid (path is PRIMARY KEY), so use NULL for AUTOINCREMENT
+            if has_indexed_at and has_mtime:
+                cursor.execute("""
+                    INSERT INTO files_new (name, full_path, content, language, mtime, indexed_at)
+                    SELECT name, path, content, language, mtime, indexed_at FROM files
+                """)
+            elif has_indexed_at:
+                cursor.execute("""
+                    INSERT INTO files_new (name, full_path, content, language, indexed_at)
+                    SELECT name, path, content, language, indexed_at FROM files
+                """)
+            elif has_mtime:
+                cursor.execute("""
+                    INSERT INTO files_new (name, full_path, content, language, mtime)
+                    SELECT name, path, content, language, mtime FROM files
+                """)
+            else:
+                cursor.execute("""
+                    INSERT INTO files_new (name, full_path, content, language)
+                    SELECT name, path, content, language FROM files
+                """)
+            
+            cursor.execute("DROP TABLE files")
+            cursor.execute("ALTER TABLE files_new RENAME TO files")
+
+        log.info("Dropping old FTS triggers and table...")
+        # Drop old triggers
+        cursor.execute("DROP TRIGGER IF EXISTS files_ai")
+        cursor.execute("DROP TRIGGER IF EXISTS files_ad")
+        cursor.execute("DROP TRIGGER IF EXISTS files_au")
+
+        # Drop old FTS table
+        cursor.execute("DROP TABLE IF EXISTS files_fts")
+
+        # Create exact FTS table (unicode61 with underscores/hyphens/dots as token chars)
+        # Note: tokenchars includes '.' to properly tokenize qualified names like PortRole.FLOW
+        log.info("Creating files_fts_exact table with unicode61 tokenizer...")
+        cursor.execute(
+            """
+            CREATE VIRTUAL TABLE files_fts_exact USING fts5(
+                name, full_path UNINDEXED, content,
+                content='files',
+                content_rowid='id',
+                tokenize="unicode61 tokenchars '_-.'"
+            )
+            """
+        )
+
+        # Create fuzzy FTS table (trigram or extended unicode61)
+        log.info(f"Creating files_fts_fuzzy table with {fuzzy_tokenizer} tokenizer...")
+        cursor.execute(
+            f"""
+            CREATE VIRTUAL TABLE files_fts_fuzzy USING fts5(
+                name, full_path UNINDEXED, content,
+                content='files',
+                content_rowid='id',
+                tokenize="{fuzzy_tokenizer}"
+            )
+            """
+        )
+
+        # Create synchronized triggers for files_fts_exact
+        log.info("Creating triggers for files_fts_exact...")
+        cursor.execute(
+            """
+            CREATE TRIGGER files_exact_ai AFTER INSERT ON files BEGIN
+                INSERT INTO files_fts_exact(rowid, name, full_path, content)
+                VALUES(new.id, new.name, new.full_path, new.content);
+            END
+            """
+        )
+        cursor.execute(
+            """
+            CREATE TRIGGER files_exact_ad AFTER DELETE ON files BEGIN
+                INSERT INTO files_fts_exact(files_fts_exact, rowid, name, full_path, content)
+                VALUES('delete', old.id, old.name, old.full_path, old.content);
+            END
+            """
+        )
+        cursor.execute(
+            """
+            CREATE TRIGGER files_exact_au AFTER UPDATE ON files BEGIN
+                INSERT INTO files_fts_exact(files_fts_exact, rowid, name, full_path, content)
+                VALUES('delete', old.id, old.name, old.full_path, old.content);
+                INSERT INTO files_fts_exact(rowid, name, full_path, content)
+                VALUES(new.id, new.name, new.full_path, new.content);
+            END
+            """
+        )
+
+        # Create synchronized triggers for files_fts_fuzzy
+        log.info("Creating triggers for files_fts_fuzzy...")
+        cursor.execute(
+            """
+            CREATE TRIGGER files_fuzzy_ai AFTER INSERT ON files BEGIN
+                INSERT INTO files_fts_fuzzy(rowid, name, full_path, content)
+                VALUES(new.id, new.name, new.full_path, new.content);
+            END
+            """
+        )
+        cursor.execute(
+            """
+            CREATE TRIGGER files_fuzzy_ad AFTER DELETE ON files BEGIN
+                INSERT INTO files_fts_fuzzy(files_fts_fuzzy, rowid, name, full_path, content)
+                VALUES('delete', old.id, old.name, old.full_path, old.content);
+            END
+            """
+        )
+        cursor.execute(
+            """
+            CREATE TRIGGER files_fuzzy_au AFTER UPDATE ON files BEGIN
+                INSERT INTO files_fts_fuzzy(files_fts_fuzzy, rowid, name, full_path, content)
+                VALUES('delete', old.id, old.name, old.full_path, old.content);
+                INSERT INTO files_fts_fuzzy(rowid, name, full_path, content)
+                VALUES(new.id, new.name, new.full_path, new.content);
+            END
+            """
+        )
+
+        # Rebuild FTS indexes from files table
+        log.info("Rebuilding FTS indexes from files table...")
+        cursor.execute("INSERT INTO files_fts_exact(files_fts_exact) VALUES('rebuild')")
+        cursor.execute("INSERT INTO files_fts_fuzzy(files_fts_fuzzy) VALUES('rebuild')")
+
+        # Commit transaction
+        cursor.execute("COMMIT")
+        log.info("Migration 004 completed successfully")
+
+        # Vacuum to reclaim space (outside transaction)
+        try:
+            log.info("Running VACUUM to reclaim space...")
+            cursor.execute("VACUUM")
+        except Exception as e:
+            log.warning(f"VACUUM failed (non-critical): {e}")
+
+    except Exception as e:
+        log.error(f"Migration 004 failed: {e}")
+        try:
+            cursor.execute("ROLLBACK")
+        except Exception:
+            pass
+        raise