claude-code-workflow 6.3.18 → 6.3.20

package/codex-lens/src/codexlens/watcher/incremental_indexer.py

@@ -0,0 +1,369 @@
+"""Incremental indexer for processing file changes."""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional
+
+from codexlens.config import Config
+from codexlens.parsers.factory import ParserFactory
+from codexlens.storage.dir_index import DirIndexStore
+from codexlens.storage.global_index import GlobalSymbolIndex
+from codexlens.storage.path_mapper import PathMapper
+from codexlens.storage.registry import RegistryStore
+
+from .events import ChangeType, FileEvent, IndexResult
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class FileIndexResult:
+    """Result of indexing a single file."""
+    path: Path
+    symbols_count: int
+    success: bool
+    error: Optional[str] = None
+
+
+class IncrementalIndexer:
+    """Incremental indexer for processing file change events.
+    
+    Processes file events (create, modify, delete, move) and updates
+    the corresponding index databases incrementally.
+    
+    Reuses existing infrastructure:
+    - ParserFactory for symbol extraction
+    - DirIndexStore for per-directory storage
+    - GlobalSymbolIndex for cross-file symbols
+    - PathMapper for source-to-index path conversion
+    
+    Example:
+        indexer = IncrementalIndexer(registry, mapper, config)
+        result = indexer.process_changes([
+            FileEvent(Path("foo.py"), ChangeType.MODIFIED, time.time()),
+        ])
+        print(f"Indexed {result.files_indexed} files")
+    """
+    
+    def __init__(
+        self,
+        registry: RegistryStore,
+        mapper: PathMapper,
+        config: Optional[Config] = None,
+    ) -> None:
+        """Initialize incremental indexer.
+        
+        Args:
+            registry: Global project registry
+            mapper: Path mapper for source-to-index conversion
+            config: CodexLens configuration (uses defaults if None)
+        """
+        self.registry = registry
+        self.mapper = mapper
+        self.config = config or Config()
+        self.parser_factory = ParserFactory(self.config)
+        
+        self._global_index: Optional[GlobalSymbolIndex] = None
+        self._dir_stores: dict[Path, DirIndexStore] = {}
+        self._lock = __import__("threading").RLock()
+    
+    def _get_global_index(self, index_root: Path, source_root: Optional[Path] = None) -> Optional[GlobalSymbolIndex]:
+        """Get or create global symbol index.
+
+        Args:
+            index_root: Root directory containing the global symbol index DB
+            source_root: Source directory root for looking up project_id from registry
+        """
+        if not self.config.global_symbol_index_enabled:
+            return None
+
+        if self._global_index is None:
+            global_db_path = index_root / GlobalSymbolIndex.DEFAULT_DB_NAME
+            if global_db_path.exists():
+                # Get project_id from registry using source_root
+                project_id = 0  # Default fallback
+                if source_root:
+                    project_info = self.registry.get_project(source_root)
+                    if project_info:
+                        project_id = project_info.id
+                self._global_index = GlobalSymbolIndex(global_db_path, project_id=project_id)
+
+        return self._global_index
+    
+    def _get_dir_store(self, dir_path: Path) -> Optional[DirIndexStore]:
+        """Get DirIndexStore for a directory, if indexed."""
+        with self._lock:
+            if dir_path in self._dir_stores:
+                return self._dir_stores[dir_path]
+            
+            index_db = self.mapper.source_to_index_db(dir_path)
+            if not index_db.exists():
+                logger.debug("No index found for directory: %s", dir_path)
+                return None
+            
+            # Get index root for global index
+            source_root = self.mapper.get_project_root(dir_path) or dir_path
+            index_root = self.mapper.source_to_index_dir(source_root)
+            global_index = self._get_global_index(index_root, source_root=source_root)
+            
+            store = DirIndexStore(
+                index_db,
+                config=self.config,
+                global_index=global_index,
+            )
+            self._dir_stores[dir_path] = store
+            return store
+    
+    def process_changes(self, events: List[FileEvent]) -> IndexResult:
+        """Process a batch of file change events.
+        
+        Args:
+            events: List of file events to process
+            
+        Returns:
+            IndexResult with statistics
+        """
+        result = IndexResult()
+        
+        for event in events:
+            try:
+                if event.change_type == ChangeType.CREATED:
+                    file_result = self._index_file(event.path)
+                    if file_result.success:
+                        result.files_indexed += 1
+                        result.symbols_added += file_result.symbols_count
+                    else:
+                        result.errors.append(file_result.error or f"Failed to index: {event.path}")
+                
+                elif event.change_type == ChangeType.MODIFIED:
+                    file_result = self._index_file(event.path)
+                    if file_result.success:
+                        result.files_indexed += 1
+                        result.symbols_added += file_result.symbols_count
+                    else:
+                        result.errors.append(file_result.error or f"Failed to index: {event.path}")
+                
+                elif event.change_type == ChangeType.DELETED:
+                    self._remove_file(event.path)
+                    result.files_removed += 1
+                
+                elif event.change_type == ChangeType.MOVED:
+                    # Remove from old location, add at new location
+                    if event.old_path:
+                        self._remove_file(event.old_path)
+                        result.files_removed += 1
+                    file_result = self._index_file(event.path)
+                    if file_result.success:
+                        result.files_indexed += 1
+                        result.symbols_added += file_result.symbols_count
+                    else:
+                        result.errors.append(file_result.error or f"Failed to index: {event.path}")
+                
+            except Exception as exc:
+                error_msg = f"Error processing {event.path}: {type(exc).__name__}: {exc}"
+                logger.error(error_msg)
+                result.errors.append(error_msg)
+        
+        return result
+    
+    def _index_file(self, path: Path) -> FileIndexResult:
+        """Index a single file.
+
+        Args:
+            path: Path to the file to index
+
+        Returns:
+            FileIndexResult with status
+        """
+        path = Path(path).resolve()
+
+        # Check if file exists
+        if not path.exists():
+            return FileIndexResult(
+                path=path,
+                symbols_count=0,
+                success=False,
+                error=f"File not found: {path}",
+            )
+
+        # Check if language is supported
+        language = self.config.language_for_path(path)
+        if not language:
+            return FileIndexResult(
+                path=path,
+                symbols_count=0,
+                success=False,
+                error=f"Unsupported language for: {path}",
+            )
+
+        # Get directory store
+        dir_path = path.parent
+        store = self._get_dir_store(dir_path)
+        if store is None:
+            return FileIndexResult(
+                path=path,
+                symbols_count=0,
+                success=False,
+                error=f"Directory not indexed: {dir_path}",
+            )
+
+        # Read file content with fallback encodings
+        try:
+            content = path.read_text(encoding="utf-8")
+        except UnicodeDecodeError:
+            logger.debug("UTF-8 decode failed for %s, using fallback with errors='ignore'", path)
+            try:
+                content = path.read_text(encoding="utf-8", errors="ignore")
+            except Exception as exc:
+                return FileIndexResult(
+                    path=path,
+                    symbols_count=0,
+                    success=False,
+                    error=f"Failed to read file: {exc}",
+                )
+        except Exception as exc:
+            return FileIndexResult(
+                path=path,
+                symbols_count=0,
+                success=False,
+                error=f"Failed to read file: {exc}",
+            )
+
+        # Parse symbols
+        try:
+            parser = self.parser_factory.get_parser(language)
+            indexed_file = parser.parse(content, path)
+        except Exception as exc:
+            error_msg = f"Failed to parse {path}: {type(exc).__name__}: {exc}"
+            logger.error(error_msg)
+            return FileIndexResult(
+                path=path,
+                symbols_count=0,
+                success=False,
+                error=error_msg,
+            )
+
+        # Update store with retry logic for transient database errors
+        max_retries = 3
+        for attempt in range(max_retries):
+            try:
+                store.add_file(
+                    name=path.name,
+                    full_path=str(path),
+                    content=content,
+                    language=language,
+                    symbols=indexed_file.symbols,
+                    relationships=indexed_file.relationships,
+                )
+
+                # Update merkle root
+                store.update_merkle_root()
+
+                logger.debug("Indexed file: %s (%d symbols)", path, len(indexed_file.symbols))
+
+                return FileIndexResult(
+                    path=path,
+                    symbols_count=len(indexed_file.symbols),
+                    success=True,
+                )
+
+            except __import__("sqlite3").OperationalError as exc:
+                # Transient database errors (e.g., database locked)
+                if attempt < max_retries - 1:
+                    import time
+                    wait_time = 0.1 * (2 ** attempt)  # Exponential backoff
+                    logger.debug("Database operation failed (attempt %d/%d), retrying in %.2fs: %s",
+                                attempt + 1, max_retries, wait_time, exc)
+                    time.sleep(wait_time)
+                    continue
+                else:
+                    error_msg = f"Failed to store {path} after {max_retries} attempts: {exc}"
+                    logger.error(error_msg)
+                    return FileIndexResult(
+                        path=path,
+                        symbols_count=0,
+                        success=False,
+                        error=error_msg,
+                    )
+            except Exception as exc:
+                error_msg = f"Failed to store {path}: {type(exc).__name__}: {exc}"
+                logger.error(error_msg)
+                return FileIndexResult(
+                    path=path,
+                    symbols_count=0,
+                    success=False,
+                    error=error_msg,
+                )
+
+        # Should never reach here
+        return FileIndexResult(
+            path=path,
+            symbols_count=0,
+            success=False,
+            error="Unexpected error in indexing loop",
+        )
+    
+    def _remove_file(self, path: Path) -> bool:
+        """Remove a file from the index.
+
+        Args:
+            path: Path to the file to remove
+
+        Returns:
+            True if removed successfully
+        """
+        path = Path(path).resolve()
+        dir_path = path.parent
+
+        store = self._get_dir_store(dir_path)
+        if store is None:
+            logger.debug("Cannot remove file, directory not indexed: %s", dir_path)
+            return False
+
+        # Retry logic for transient database errors
+        max_retries = 3
+        for attempt in range(max_retries):
+            try:
+                store.remove_file(str(path))
+                store.update_merkle_root()
+                logger.debug("Removed file from index: %s", path)
+                return True
+
+            except __import__("sqlite3").OperationalError as exc:
+                # Transient database errors (e.g., database locked)
+                if attempt < max_retries - 1:
+                    import time
+                    wait_time = 0.1 * (2 ** attempt)  # Exponential backoff
+                    logger.debug("Database operation failed (attempt %d/%d), retrying in %.2fs: %s",
+                                attempt + 1, max_retries, wait_time, exc)
+                    time.sleep(wait_time)
+                    continue
+                else:
+                    logger.error("Failed to remove %s after %d attempts: %s", path, max_retries, exc)
+                    return False
+            except Exception as exc:
+                logger.error("Failed to remove %s: %s", path, exc)
+                return False
+
+        # Should never reach here
+        return False
+    
+    def close(self) -> None:
+        """Close all open stores."""
+        with self._lock:
+            for store in self._dir_stores.values():
+                try:
+                    store.close()
+                except Exception:
+                    pass
+            self._dir_stores.clear()
+            
+            if self._global_index:
+                try:
+                    self._global_index.close()
+                except Exception:
+                    pass
+                self._global_index = None

package/codex-lens/src/codexlens/watcher/manager.py

@@ -0,0 +1,255 @@
+"""Watcher manager for coordinating file watching and incremental indexing."""
+
+from __future__ import annotations
+
+import json
+import logging
+import signal
+import threading
+import time
+from pathlib import Path
+from typing import Callable, List, Optional
+
+from codexlens.config import Config
+from codexlens.storage.path_mapper import PathMapper
+from codexlens.storage.registry import RegistryStore
+
+from .events import FileEvent, IndexResult, PendingQueueStatus, WatcherConfig, WatcherStats
+from .file_watcher import FileWatcher
+from .incremental_indexer import IncrementalIndexer
+
+logger = logging.getLogger(__name__)
+
+
+class WatcherManager:
+    """High-level manager for file watching and incremental indexing.
+    
+    Coordinates FileWatcher and IncrementalIndexer with:
+    - Lifecycle management (start/stop)
+    - Signal handling (SIGINT/SIGTERM)
+    - Statistics tracking
+    - Graceful shutdown
+    """
+    
+    def __init__(
+        self,
+        root_path: Path,
+        config: Optional[Config] = None,
+        watcher_config: Optional[WatcherConfig] = None,
+        on_indexed: Optional[Callable[[IndexResult], None]] = None,
+        on_queue_change: Optional[Callable[[PendingQueueStatus], None]] = None,
+    ) -> None:
+        self.root_path = Path(root_path).resolve()
+        self.config = config or Config()
+        self.watcher_config = watcher_config or WatcherConfig()
+        self.on_indexed = on_indexed
+        self.on_queue_change = on_queue_change
+
+        self._registry: Optional[RegistryStore] = None
+        self._mapper: Optional[PathMapper] = None
+        self._watcher: Optional[FileWatcher] = None
+        self._indexer: Optional[IncrementalIndexer] = None
+
+        self._running = False
+        self._stop_event = threading.Event()
+        self._lock = threading.RLock()
+
+        # Statistics
+        self._stats = WatcherStats()
+        self._original_sigint = None
+        self._original_sigterm = None
+
+        # Index history for tracking recent results
+        self._index_history: List[IndexResult] = []
+        self._max_history_size = 10
+    
+    def _handle_changes(self, events: List[FileEvent]) -> None:
+        """Handle file change events from watcher."""
+        if not self._indexer or not events:
+            return
+
+        logger.info("Processing %d file changes", len(events))
+        result = self._indexer.process_changes(events)
+
+        # Update stats
+        self._stats.events_processed += len(events)
+        self._stats.last_event_time = time.time()
+
+        # Save to history
+        self._index_history.append(result)
+        if len(self._index_history) > self._max_history_size:
+            self._index_history.pop(0)
+
+        if result.files_indexed > 0 or result.files_removed > 0:
+            logger.info(
+                "Indexed %d files, removed %d files, %d errors",
+                result.files_indexed, result.files_removed, len(result.errors)
+            )
+
+        # Output JSON for TypeScript backend parsing
+        result_data = {
+            "files_indexed": result.files_indexed,
+            "files_removed": result.files_removed,
+            "symbols_added": result.symbols_added,
+            "symbols_removed": result.symbols_removed,
+            "files_success": result.files_success[:20],  # Limit output
+            "files_failed": result.files_failed[:20],
+            "errors": result.errors[:10],
+            "timestamp": result.timestamp
+        }
+        print(f"[INDEX_RESULT] {json.dumps(result_data)}", flush=True)
+
+        if self.on_indexed:
+            try:
+                self.on_indexed(result)
+            except Exception as exc:
+                logger.error("Error in on_indexed callback: %s", exc)
+    
+    def _signal_handler(self, signum, frame) -> None:
+        """Handle shutdown signals."""
+        logger.info("Received signal %d, stopping...", signum)
+        self.stop()
+    
+    def _install_signal_handlers(self) -> None:
+        """Install signal handlers for graceful shutdown."""
+        try:
+            self._original_sigint = signal.signal(signal.SIGINT, self._signal_handler)
+            if hasattr(signal, 'SIGTERM'):
+                self._original_sigterm = signal.signal(signal.SIGTERM, self._signal_handler)
+        except (ValueError, OSError):
+            # Signal handling not available (e.g., not main thread)
+            pass
+    
+    def _restore_signal_handlers(self) -> None:
+        """Restore original signal handlers."""
+        try:
+            if self._original_sigint is not None:
+                signal.signal(signal.SIGINT, self._original_sigint)
+            if self._original_sigterm is not None and hasattr(signal, 'SIGTERM'):
+                signal.signal(signal.SIGTERM, self._original_sigterm)
+        except (ValueError, OSError):
+            pass
+    
+    def start(self) -> None:
+        """Start watching and indexing."""
+        with self._lock:
+            if self._running:
+                logger.warning("WatcherManager already running")
+                return
+            
+            # Validate path
+            if not self.root_path.exists():
+                raise ValueError(f"Root path does not exist: {self.root_path}")
+            
+            # Initialize components
+            self._registry = RegistryStore()
+            self._registry.initialize()
+            self._mapper = PathMapper()
+            
+            self._indexer = IncrementalIndexer(
+                self._registry, self._mapper, self.config
+            )
+            
+            self._watcher = FileWatcher(
+                self.root_path, self.watcher_config, self._handle_changes
+            )
+
+            # Always register queue change callback for stdout output (TypeScript backend)
+            # The wrapper prints [QUEUE_STATUS] JSON and optionally calls on_queue_change
+            self._watcher.register_queue_change_callback(self._on_queue_change_wrapper)
+
+            # Install signal handlers
+            self._install_signal_handlers()
+            
+            # Start watcher
+            self._running = True
+            self._stats.is_running = True
+            self._stop_event.clear()
+            self._watcher.start()
+            
+            logger.info("WatcherManager started for: %s", self.root_path)
+    
+    def stop(self) -> None:
+        """Stop watching and clean up."""
+        with self._lock:
+            if not self._running:
+                return
+            
+            self._running = False
+            self._stats.is_running = False
+            self._stop_event.set()
+            
+            # Stop watcher
+            if self._watcher:
+                self._watcher.stop()
+                self._watcher = None
+            
+            # Close indexer
+            if self._indexer:
+                self._indexer.close()
+                self._indexer = None
+            
+            # Close registry
+            if self._registry:
+                self._registry.close()
+                self._registry = None
+            
+            # Restore signal handlers
+            self._restore_signal_handlers()
+            
+            logger.info("WatcherManager stopped")
+    
+    def wait(self) -> None:
+        """Block until stopped."""
+        try:
+            while self._running:
+                self._stop_event.wait(timeout=1.0)
+        except KeyboardInterrupt:
+            logger.info("Interrupted, stopping...")
+            self.stop()
+    
+    @property
+    def is_running(self) -> bool:
+        """Check if manager is running."""
+        return self._running
+    
+    def get_stats(self) -> WatcherStats:
+        """Get runtime statistics."""
+        return WatcherStats(
+            files_watched=self._stats.files_watched,
+            events_processed=self._stats.events_processed,
+            last_event_time=self._stats.last_event_time,
+            is_running=self._running,
+        )
+
+    def _on_queue_change_wrapper(self, status: PendingQueueStatus) -> None:
+        """Wrapper for queue change callback with JSON output."""
+        # Output JSON for TypeScript backend parsing
+        status_data = {
+            "file_count": status.file_count,
+            "files": status.files,
+            "countdown_seconds": status.countdown_seconds,
+            "last_event_time": status.last_event_time
+        }
+        print(f"[QUEUE_STATUS] {json.dumps(status_data)}", flush=True)
+
+        if self.on_queue_change:
+            try:
+                self.on_queue_change(status)
+            except Exception as exc:
+                logger.error("Error in on_queue_change callback: %s", exc)
+
+    def flush_now(self) -> None:
+        """Immediately flush pending queue (manual trigger)."""
+        if self._watcher:
+            self._watcher.flush_now()
+
+    def get_pending_queue_status(self) -> Optional[PendingQueueStatus]:
+        """Get current pending queue status."""
+        if self._watcher:
+            return self._watcher.get_pending_queue_status()
+        return None
+
+    def get_index_history(self, limit: int = 5) -> List[IndexResult]:
+        """Get recent index history."""
+        return self._index_history[-limit:]

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-code-workflow",
-  "version": "6.3.18",
+  "version": "6.3.20",
   "description": "JSON-driven multi-agent development framework with intelligent CLI orchestration (Gemini/Qwen/Codex), context-first architecture, and automated workflow execution",
   "type": "module",
   "main": "ccw/src/index.js",
@@ -13,6 +13,7 @@
     "start": "node ccw/bin/ccw.js",
     "test": "node --experimental-strip-types --test ccw/tests/*.test.js",
     "test:visual": "node --experimental-strip-types --test ccw/tests/visual/**/*.visual.test.ts",
+    "test:e2e": "node --experimental-strip-types --test ccw/tests/e2e/*.e2e.test.ts",
     "prepublishOnly": "npm run build && echo 'Ready to publish @dyw/claude-code-workflow'"
   },
   "keywords": [
@@ -40,6 +41,7 @@
     "glob": "^10.3.0",
     "gradient-string": "^2.0.2",
     "inquirer": "^9.2.0",
+    "jsonwebtoken": "^9.0.3",
     "open": "^9.1.0",
     "ora": "^7.0.0",
     "zod": "^4.1.13"
@@ -79,6 +81,7 @@
     "@types/better-sqlite3": "^7.6.12",
     "@types/gradient-string": "^1.1.6",
     "@types/inquirer": "^9.0.9",
+    "@types/jsonwebtoken": "^9.0.10",
     "@types/node": "^25.0.1",
     "pixelmatch": "^7.1.0",
     "playwright": "^1.57.0",